From 2ee10a09155b3b4c55761ee41d4d8e3ac135aa94 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 12 Sep 2026 06:14:07 +0100 Subject: [PATCH 1/2] feat: draft regression tests from confirmed findings (phase 1) Assisted-by: Claude:claude-fable-5-1 --- .../brief/04-surfaces/07-review.md | 95 +- .../brief/04-surfaces/08-draft-tests.md | 205 +++++ .../brief/05-internals/01-packages.md | 28 +- .../brief/05-internals/02-schemas.md | 70 ++ ...ne-append-primitive-and-one-review-verb.md | 147 ++++ .../itd-9-regression-test-drafting.md | 19 +- ...09120417480624-regression-test-drafting.md | 827 ++++++++++++++++++ .abcd/work/CONTEXT.md | 11 +- .abcd/work/DECISIONS.md | 25 + CHANGELOG.md | 29 + README.md | 18 +- docs/README.md | 2 +- docs/how-to/draft-regression-tests.md | 171 ++++ docs/reference/session-directory.md | 47 +- examples/sample-session/tests.jsonl | 6 + internal/analyze/analyze_test.go | 56 +- internal/analyze/emit.go | 2 +- internal/analyze/ingest.go | 153 ++-- internal/drafttests/drafttests.go | 440 ++++++++++ internal/drafttests/drafttests_test.go | 398 +++++++++ internal/drafttests/emit.go | 199 +++++ internal/drafttests/emit_test.go | 272 ++++++ internal/drafttests/ingest.go | 422 +++++++++ internal/drafttests/ingest_test.go | 469 ++++++++++ internal/drafttests/render.go | 135 +++ internal/drafttests/render_test.go | 292 +++++++ internal/drafttests/review.go | 486 ++++++++++ internal/drafttests/review_test.go | 584 +++++++++++++ internal/drafttests/testdata/answer.json | 11 + internal/drafttests/testdata/edit.json | 1 + internal/drafttests/testdata/findings.jsonl | 9 + internal/drafttests/testdata/manifest.json | 11 + internal/drafttests/testdata/tests.md | 38 + internal/drafttests/testdata/timeline.jsonl | 13 + internal/drafttests/window.go | 83 ++ internal/review/review.go | 300 +++---- internal/review/review_test.go | 178 ++-- internal/session/records.go | 275 ++++++ internal/session/records_test.go | 322 +++++++ internal/session/session.go | 33 +- 40 files changed, 6429 insertions(+), 453 deletions(-) create mode 100644 .abcd/development/brief/04-surfaces/08-draft-tests.md create mode 100644 .abcd/development/decisions/adrs/0001-one-append-primitive-and-one-review-verb.md rename .abcd/development/intents/{drafts => planned}/itd-9-regression-test-drafting.md (79%) create mode 100644 .abcd/development/specs/open/spc-2609120417480624-regression-test-drafting.md create mode 100644 docs/how-to/draft-regression-tests.md create mode 100644 examples/sample-session/tests.jsonl create mode 100644 internal/drafttests/drafttests.go create mode 100644 internal/drafttests/drafttests_test.go create mode 100644 internal/drafttests/emit.go create mode 100644 internal/drafttests/emit_test.go create mode 100644 internal/drafttests/ingest.go create mode 100644 internal/drafttests/ingest_test.go create mode 100644 internal/drafttests/render.go create mode 100644 internal/drafttests/render_test.go create mode 100644 internal/drafttests/review.go create mode 100644 internal/drafttests/review_test.go create mode 100644 internal/drafttests/testdata/answer.json create mode 100644 internal/drafttests/testdata/edit.json create mode 100644 internal/drafttests/testdata/findings.jsonl create mode 100644 internal/drafttests/testdata/manifest.json create mode 100644 internal/drafttests/testdata/tests.md create mode 100644 internal/drafttests/testdata/timeline.jsonl create mode 100644 internal/drafttests/window.go create mode 100644 internal/session/records.go create mode 100644 internal/session/records_test.go diff --git a/.abcd/development/brief/04-surfaces/07-review.md b/.abcd/development/brief/04-surfaces/07-review.md index 2ad6af3..a6c3034 100644 --- a/.abcd/development/brief/04-surfaces/07-review.md +++ b/.abcd/development/brief/04-surfaces/07-review.md @@ -1,21 +1,44 @@ # `testimony review` -Records a human's verdict on each candidate finding. A finding is born -`unverified` ([`06-analyze.md`](06-analyze.md)); review is where a person -confirms, rejects, or marks it a duplicate. The retained human verdict is the -precision measure the method stands on, so a verdict is stored as a separate -*appended* record in `findings.jsonl`, never by rewriting the finding in place — -the birth state and the full decision history both survive. +The pipeline's one human-decision verb, across both record families. `-kind +findings` (the default) records a verdict on each candidate finding: a finding is +born `unverified` ([`06-analyze.md`](06-analyze.md)), and review is where a person +confirms, rejects, or marks it a duplicate. `-kind tests` records an accept / edit +/ reject decision on each drafted regression test +([`08-draft-tests.md`](08-draft-tests.md)). One verb rather than two is an +architecture-shaping choice +([ADR 0001](../../decisions/adrs/0001-one-append-primitive-and-one-review-verb.md)). + +The retained human decision is the precision measure the method stands on, so +both families store it as a separate *appended* record — in `findings.jsonl` and +`tests.jsonl` respectively — never by rewriting the machine record in place: the +birth state and the full decision history both survive. Both append through the +one shared primitive, `session.AppendRecord`; the walks and the vocabularies stay +per-kind, because `accepted | edited | rejected` (where `edited` carries a payload +no verdict ever does) is genuinely a different vocabulary from +`confirmed | rejected | duplicate`. ## Flags | Flag | Default | Meaning | |---|---|---| | `-session` | (required) | session directory | -| `-finding` | *(interactive)* | non-interactive: the finding to judge (`F-NNN`) | -| `-verdict` | *(interactive)* | non-interactive: `confirmed` \| `rejected` \| `duplicate-of-F-NNN` | +| `-kind` | `findings` | which record family to review: `findings` or `tests` | +| `-finding` | *(interactive)* | non-interactive: the finding to judge (`F-NNN`), `-kind findings` only | +| `-verdict` | *(interactive)* | non-interactive: `confirmed` \| `rejected` \| `duplicate-of-F-NNN`, `-kind findings` only | +| `-test` | *(interactive)* | non-interactive: the draft to decide (`T-NNN`), `-kind tests` only | +| `-decision` | *(interactive)* | non-interactive: `accepted` \| `edited` \| `rejected`, `-kind tests` only | +| `-edit` | *(off)* | with `-decision edited`: the replacement fields as a JSON object at `FILE` (or `-` for stdin) | + +`-finding`/`-verdict` are refused with `-kind tests`, and +`-test`/`-decision`/`-edit` with `-kind findings`, at exit 2 — a flag that belongs +to the other record family is a wrong invocation, not a silently ignored one. +`review.Run` refuses the same pairings itself, so the rule is a property of the +API rather than of one caller's invariants: a run that recorded nothing while +exiting 0 would let a script believe the decision landed. +`-kind findings` is byte-for-byte the findings-side behaviour described below. -## Behaviour +## Behaviour — `-kind findings` (default) - Loads the findings and any existing verdicts (hinting to run `analyze -ingest` first when there is no `findings.jsonl`) and computes each @@ -45,3 +68,57 @@ the birth state and the full decision history both survive. The rendered verdicts appear in [`report`](04-report.md)'s Findings section, grouped by effective status. + +## Behaviour — `-kind tests` + +- Loads the drafts and any existing decisions (hinting to run + `draft-tests -ingest` first when there is no `tests.jsonl`) and computes each + draft's effective status: every draft starts `proposed`, decision records apply + in file order, and the last one for a draft wins. A decision naming an unknown + draft is ignored for display, and one whose value is outside the closed enum is + ignored rather than applied — a draft in an unrenderable status would otherwise + vanish from both the walk and the render. +- **Interactive** (`review -session DIR -kind tests`): walks the `proposed` drafts + in id order. For each it prints the id, the source finding with its type, + severity and clock, the title, the numbered steps, the expected and observed + behaviour, and the participant's quote, then prompts + `[a]ccept [e]dit [r]eject [s]kip [q]uit`. `e` asks for each editable field in + turn showing the current value, a blank answer keeping it, and reads `steps` one + per line until a blank line. A pass that changes nothing prints + `no changes; recorded as accepted.` and records `accepted` — an `edited` + decision with an empty `edit` records a change that did not happen and is not + representable. Each decision appends a record stamped with today's date. The + source finding supplies only the type and the clock: if `findings.jsonl` is + absent or unreadable those degrade to placeholders rather than blocking a + decision, because the draft itself carries the substance. +- **Interactive mode is gated on stdin being a character device**, exactly as the + findings walk is, so CI never blocks. +- **Non-interactive** (`review -session DIR -kind tests -test T-001 -decision + accepted`): validates that the draft exists and the decision parses, appends one + record, and prints a one-line confirmation. `-decision edited` requires + `-edit FILE` (or `-` for stdin), a JSON object holding the replacement fields, + read through `session.OpenFileNoFollowRead` and decoded with + `DisallowUnknownFields` and the same field rules as the interactive edit. Every + interactive path in this repository has a non-interactive twin; making `edited` + the one exception would put the only lossy decision out of reach of a script or + an agent host. +- **The `edit` object is a closed four-field subset** (`title`, `steps`, + `expected`, `observed`). An `edit` naming `id`, `finding`, `session`, + `severity`, or `rationale_quote` is a hard error, not a silently dropped key, so + no human edit can re-point a draft at a different finding or session; the only + way to change the link is to reject the draft and ingest a new one. An `edited` + draft's rendered fields are the last `edited` decision's `edit` applied over the + draft, computed at render time — the draft line is never rewritten. +- **Under-lock target check.** `AppendDecision` re-reads the current drafts under + its exclusive lock and refuses if the targeted id has vanished or now names a + different draft — the mechanism `AppendVerdict` uses, via + `drafttests.SameIdentity`. The walk snapshots the drafts once and then blocks on + the operator, a concurrent `draft-tests -ingest` may truncate-and-rewrite in + that gap (permitted until the first decision exists), and draft ids restart at + `T-001`, so without the re-check a decision would silently attach to a different + draft. +- A decision may be appended even when one already exists (append-only + correction; the latest wins). No existing line is ever touched. + +The accepted and edited drafts are what +[`draft-tests -render`](08-draft-tests.md) puts in the test plan. diff --git a/.abcd/development/brief/04-surfaces/08-draft-tests.md b/.abcd/development/brief/04-surfaces/08-draft-tests.md new file mode 100644 index 0000000..2fb8a79 --- /dev/null +++ b/.abcd/development/brief/04-surfaces/08-draft-tests.md @@ -0,0 +1,205 @@ +# `testimony draft-tests` + +The regression-test drafting layer, in the same host-delegated halves as +[`analyze`](06-analyze.md) plus a render mode. The repository's oracle is +host-delegated: the CLI never calls a model, holds no keys, and adds no network +dependency. So `draft-tests` first *emits* a self-contained drafting request — a +versioned rubric, the session context, and, for each **confirmed** finding, that +finding's record plus its event window from `timeline.jsonl` — that any agent host +(or a human) runs to produce a JSON answer; then `draft-tests -ingest` validates +that answer field-by-field and writes `tests.jsonl`. Ingest is the sole validation +boundary: every draft lands `status: "proposed"` regardless of what the answer +claims. `draft-tests -render` renders the accepted drafts as Markdown test-case +blocks a docs-as-code test plan can hold. + +Eligibility is the whole point of the step: it sits *downstream* of verification, +so only evidence a human already vouched for can become a test. The human +accept / edit / reject pass is [`review -kind tests`](07-review.md) — one +human-decision verb for the whole pipeline +([ADR 0001](../../decisions/adrs/0001-one-append-primitive-and-one-review-verb.md)). +See the schemas page ([`../05-internals/02-schemas.md`](../05-internals/02-schemas.md)). + +## Flags + +| Flag | Default | Meaning | +|---|---|---| +| `-session` | (required) | session directory | +| `-window` | `10` | emit mode: the event-window half-width in seconds | +| `-out` | *(stdout)* | emit/render mode: write to `FILE` instead of stdout | +| `-ingest` | *(off)* | ingest mode: validate the answer JSON at `FILE` (or `-` for stdin) into `tests.jsonl` | +| `-render` | *(off)* | render mode: write Markdown test cases for the accepted drafts | + +`draft-tests` runs in exactly one mode, extending `analyze`'s emit-or-ingest rule +by one: emit (neither `-ingest` nor `-render`), ingest (`-ingest`), or render +(`-render`). `-ingest` combines with neither `-out` nor `-render`. Emit reads +`manifest.json`, `findings.jsonl`, and `timeline.jsonl`; ingest reads +`manifest.json` and `findings.jsonl` only (drafts are validated against the +*findings*, never re-derived from the timeline); render reads `manifest.json`, +`findings.jsonl`, and `tests.jsonl`. Emit hints to run `merge` first when the +timeline is missing (reusing `analyze.LoadTimeline`, so a duplicated entry id or an +unknown `src` is refused there too); every mode hints to run `analyze -ingest` +first when there is no `findings.jsonl`, and ingest, render and +`review -kind tests` hint to run `draft-tests -ingest` first when there is no +`tests.jsonl`. + +## Eligibility + +A finding is eligible iff its **effective status is `confirmed`** and its `mode` +is not `B`. Effective status is not recomputed here: `drafttests` calls +`analyze.Load` and `analyze.EffectiveStatus`, the single helper `review` and +`report` already use, so "later verdicts override earlier ones" holds for free — +a finding confirmed then later rejected is not eligible, and one rejected then +later confirmed is. `duplicate` and `unverified` are never eligible, including a +`duplicate` whose target is confirmed: the canonical finding carries the +evidence. `mode: "B"` (reference capture, itd-4) is excluded because a design +preference has nothing to regress against; nothing produces Mode B today, so the +exclusion is a guard rather than a live filter. `type` is deliberately **not** +filtered — the acceptance criterion names a confirmed finding without +qualification, the request carries each finding's `type` so the model can +calibrate, and a draft with nothing to regress against is what the reject verb is +for. + +## Behaviour — emit (default) + +- Writes a single, self-contained prompt so that an agent given only this text can + answer. In order: the rubric version header + (`Testimony regression-test drafting rubric: testimony-testdraft/v1`), the + proposal stance, the per-field instructions, the rubric body (the field + definitions and the hard constraints ingest enforces), the session context + (session, app, participant, ordered tasks), each eligible finding's own JSON + line followed by its event window, and the required output shape with a worked + example. +- The rubric version is a package constant; it pins the drafting scheme so drafts + are comparable across sessions and future revisions are explicit. +- **The event window** is the only material the model may reconstruct steps from. + `Window(entries, f, window)` returns every entry whose time falls in `[lo, hi]`, + where `lo` is the earliest cited evidence entry's start minus `window` and `hi` + the latest cited entry's end (`timeline.SpeechEnd`) plus `window`. Speech and + event entries are both included: the utterances carry the *expected* behaviour + and the events carry the *steps*. Evidence ids are matched in their + `session.SafeText` form, the form the request shows and `analyze` validates + against. A finding whose evidence resolves to no entry — reachable only via a + hand-edited `findings.jsonl` — falls back to `[f.T - window, f.T + window]`. +- **`-window` defaults to 10 seconds**, not `report`'s 2.5: `report`'s window + joins an event to the utterance it accompanies, whereas a repro needs the + lead-up and the aftermath. On the bundled sample, F-001's evidence spans + `ev-003` (19.2 s), `utt-004` (22–28 s) and `ev-004` (24.1 s); at 2.5 s the + window is [16.7, 30.5] and excludes `utt-003` at 16.0 s — the one utterance + stating the expected behaviour — while at 10 s it is [9.2, 38.0] and holds the + whole click-type-click-click repro. Negative values are legitimate (they + narrow the window), matching `report`; only finiteness is required. +- **Escaping** follows the rule `analyze`'s emitter established: a value rendered + as prose or a list item *outside* any code fence goes through + `session.SafeInline` (the manifest fields, and each per-finding header's id and + type), so an attacker-authored manifest or findings file cannot survive as an + active link or an image beacon when a saved `request.md` is previewed; a value + rendered *inside* a fence goes through `session.SafeText` only, which strips the + terminal-control and Trojan-Source bytes `json.Marshal` passes through. There is + one shared home for the escape set and this adds no second one. +- Emit mutates nothing in the session directory. `-out FILE` writes the prompt to + a file instead of stdout. + +## Behaviour — ingest (`-ingest FILE`) + +- Loads `manifest.json` and `findings.jsonl`, computes effective status, and + builds the eligible set. If it is empty, the refusal comes **before** a byte of + the answer is read: every draft would fail the same rule, and the operator + should read the one fact that explains them. +- Reads the answer from `FILE` (or stdin when `-`), bounded by + `session.MaxAnswerBytes`. Accepts a top-level object with a `tests` array + (optionally carrying a `rubric`, which must be a known version) or a bare array. +- Decodes each draft with unknown fields disallowed — a closed output shape — and + runs every schema rule + ([`../05-internals/02-schemas.md`](../05-internals/02-schemas.md)): id format + and uniqueness; a `finding` whose effective status is `confirmed` and whose mode + is not `B`; a `session` equal to the manifest's; a non-empty `title` of at most + 200 characters; non-empty `steps` of at most 32 non-empty entries; non-empty + `expected` and `observed`; a `rationale_quote` **equal** to the source finding's + `quote`; and a `severity` **equal** to the source finding's. +- `severity` decodes through a `*int` so an absent value is reported as *missing* + rather than as a mismatch against a number the answer never gave (the + `rawFinding.T` precedent). +- Validation is transactional and exhaustive: all errors across all drafts are + reported at once, each naming the draft (its id when well-formed, otherwise its + position *in the answer*), the field, and the offending value; on any error + nothing is written and the command exits non-zero. +- On success it forces `status: "proposed"` on every draft, pre-flights the + serialised set against `session.MaxJSONLLine` per line and + `session.MaxJSONLBytes` in total, and commits through `session.CommitRecords`. + Nothing is ever born accepted. +- To protect the retained human record, ingest refuses to overwrite a + `tests.jsonl` that already holds decision records. The guard counts any + `kind:"decision"` line, including one whose value is outside the closed enum, so + a foreign-valued human decision is never silently truncated. +- An answer with no drafts (a bare `[]`, `{"tests":[]}`, or a truncated file) is + refused rather than written: the commit replaces the file whole, so an empty + answer would otherwise erase a prior good `tests.jsonl` and report success. + +### Why quote and severity are equality-checked + +`analyze -ingest` already proved the finding's quote is a byte-for-byte substring +of a cited evidence utterance, and a human then confirmed *that* finding. Letting +the drafting model re-derive a quote from the same utterance would let it +substitute different words for the evidence the human vouched for. Requiring +equality makes the drafting step structurally incapable of introducing new +evidence: it can only carry forward what is already on the record. A mismatch is +also the cheapest available signal that the model linked the draft to the wrong +finding. Severity is the same argument for triage order: it is a human product, +bounded to `1..4` at analysis time, so the draft restates it and ingest refuses +any restatement that disagrees. + +## Behaviour — render (`-render`) + +- Writes one Markdown test-case block per draft whose effective status is + `accepted` or `edited`, in id order, with the last `edited` decision's `edit` + applied over the draft. `proposed` and `rejected` drafts are omitted: a proposal + is not a test, and a rejected draft is retained in `tests.jsonl` for the record, + not for the plan. +- Each block names the source finding (id, type, severity, clock) and the session, + so a failing test leads back to the evidence that motivated it. The clock is + rendered `[MM:SS]` from the *finding's* `t`, with a leading `-` for a negative + time, matching `report`. +- Every inserted value is escaped through the shared home for the escape set — + `session.SafeInline` in prose, and the backtick-stripping code-span form inside + a code span, where a backslash escape does not apply — so an attacker-authored + draft cannot forge Markdown structure, an active link, or an image beacon in a + document the operator pastes into their own repository. +- Output goes to stdout by default; `-out FILE` writes it and prints + `wrote `. The rendered plan gets **no** session constant: it is a hand-off + artefact whose destination the operator chooses, and Testimony never writes into + the application's repository. + +Why a mode on `draft-tests` rather than a section in `report.md`: `report.md` is +the session record, written at a fixed path, and would have to carry the section +whether or not anyone accepted a draft; the test plan is a different artefact with +a different audience and lives wherever the operator's docs-as-code plan lives. +`report` keeps its single mode and single output path. + +## Loud staging + +Both refusals write nothing and exit 1 (a well-formed invocation whose work +cannot be done), naming the counts so the operator can see *why* they are empty: + +``` +testimony: no confirmed findings to draft tests from (5 findings: 0 confirmed, 2 unverified, 1 duplicate, 1 rejected); confirm one with `testimony review -session sessions/x` first +testimony: no accepted test drafts to render (3 drafts: 0 accepted, 0 edited, 2 proposed, 1 rejected); accept one with `testimony review -session sessions/x -kind tests` first +``` + +The first applies to emit **and** ingest: with no eligible finding there is +nothing a draft could legally reference. The second keeps `-out FILE` from +truncating an existing test plan into an empty document, which is the same +reasoning behind `analyze -ingest`'s empty-answer refusal. Both wrap a package +sentinel (`drafttests.ErrNoConfirmedFindings`, `drafttests.ErrNoAcceptedDrafts`) +so a caller can tell a staged-empty session from a genuine failure. + +## Deferred + +Keyframe extraction stays out of scope, for the reason +[`analyze`](06-analyze.md) defers it: it needs local video and a multimodal pass, +neither of which fits CI or the local-only privacy boundary, and Mode B (itd-4) is +where that channel gets built. The seam is left open — `Window` returns whole +timeline entries, so a later revision can attach a keyframe reference to an entry +without changing the prompt contract. The open question the intent raised — whether +the window alone yields a followable repro — is answered by the 10-second default +with `edited` in the decision vocabulary as the recorded repair when it is not +enough; the edited-decision rate on real sessions is what would show it wrong. diff --git a/.abcd/development/brief/05-internals/01-packages.md b/.abcd/development/brief/05-internals/01-packages.md index 255b72c..f07b8c3 100644 --- a/.abcd/development/brief/05-internals/01-packages.md +++ b/.abcd/development/brief/05-internals/01-packages.md @@ -5,8 +5,8 @@ entrypoint that calls `cli.Run` and exits with its return code. - **`internal/cli`** — the command-line interface: usage text, one `flag.FlagSet` per subcommand (`demo`, `record`, `transcribe`, `merge`, - `report`, `analyze`, `review`, `version`, `help`), and dispatch into the - other packages. Holds the `Version` variable stamped by the release + `report`, `analyze`, `draft-tests`, `review`, `version`, `help`), and dispatch + into the other packages. Holds the `Version` variable stamped by the release process. Errors print as `testimony: ` and map to exit codes (1 failure, 2 usage). - **`internal/demo`** — the instrumented demo app: an embedded single-page @@ -18,8 +18,18 @@ entrypoint that calls `cli.Run` and exits with its return code. the microphone recorder — and, with `-video`, the screen recorder — as ffmpeg subprocesses, and stops them cleanly on Ctrl+C. - **`internal/session`** — the on-disk layout of a session: the `Manifest` - schema, well-known file-name constants, and generic JSONL read/write - helpers (`ReadJSONL[T]`/`WriteJSONL[T]`) used by every other package. + schema, well-known file-name constants, the shared size caps + (`MaxJSONLLine`/`MaxJSONLBytes`/`MaxAnswerBytes`), and generic JSONL read/write + helpers (`ReadJSONL[T]`/`WriteJSONL[T]`) used by every other package. It also + holds the two dangerous session-file writes, each written once: + `AppendRecord` (one appended record — the no-follow open, the exclusive lock, + the two size pre-flights, the newline framing over an unterminated last line, + an optional under-lock `Verify`, the partial-write rollback, and the returned + Close error) and `CommitRecords` (a guarded whole-file replacement, buffered + before the truncate and rolled back to empty on a short write). `findings.jsonl` + and `tests.jsonl` both hold a machine record plus appended human records, so + both write through these rather than `WriteJSONL` + ([ADR 0001](../../decisions/adrs/0001-one-append-primitive-and-one-review-verb.md)). - **`internal/timeline`** — the data model of the merged record: `Utterance`, `Word`, `Interaction`, and `Entry` types; `BuildEntries` (normalise both streams to session-relative seconds and sort); `EventsNear` (the join-window @@ -39,7 +49,15 @@ entrypoint that calls `cli.Run` and exits with its return code. self-contained, host-delegated analysis request (a versioned rubric plus the session's timeline) and is the sole validation boundary for the model's answer, writing validated findings to `findings.jsonl`. +- **`internal/drafttests`** — the regression-test drafting layer: emits a + self-contained, host-delegated drafting request (a versioned rubric plus each + confirmed finding and its event window, via `Window`), is the sole validation + boundary for the model's answer (writing proposed drafts to `tests.jsonl`), + records the human accept / edit / reject decision as an appended record, and + renders the accepted drafts as a Markdown test plan. Imports `analyze` (for + `Load`, `EffectiveStatus`, and `LoadTimeline`), `session`, and `timeline`. - **`internal/review`** — records human verdicts on candidate findings, each appended to `findings.jsonl` as a separate, non-destructive record rather than an in-place rewrite, so a finding's birth state and full verdict - history both survive. + history both survive. Its `-kind` dispatch sends the tests side to + `internal/drafttests`, so the pipeline has one human-decision verb. diff --git a/.abcd/development/brief/05-internals/02-schemas.md b/.abcd/development/brief/05-internals/02-schemas.md index 96ba56e..2349ba7 100644 --- a/.abcd/development/brief/05-internals/02-schemas.md +++ b/.abcd/development/brief/05-internals/02-schemas.md @@ -14,6 +14,7 @@ sessions// transcript.jsonl # word-aligned utterances (session-relative seconds) timeline.jsonl # merged, session-relative timeline findings.jsonl # analysis findings + appended verdicts (written by analyze/review) + tests.jsonl # regression-test drafts + appended decisions (written by draft-tests/review) report.md # human-readable session report ``` @@ -128,3 +129,72 @@ shared session) is ignored, so its finding keeps `unverified` and still appears in the report and review queue rather than vanishing into an unrendered group. Design and rationale: [`../01-product/04-analysis.md`](../01-product/04-analysis.md). + +## `tests.jsonl` — test drafts plus appended decisions + +The drafting layer's output, written by +[`draft-tests -ingest`](../04-surfaces/08-draft-tests.md) and +[`review -kind tests`](../04-surfaces/07-review.md). Two record kinds share the +file, one per line. A draft line carries no `kind`; a decision line is +discriminated by `kind: "decision"`. Decisions are **appended, never in-place +rewrites**, so the drafted proposal and the human's decision both survive and a +draft's link to its source finding and session is unreachable by any later write. +Ingest decodes each draft with unknown fields disallowed — the shape is closed — +and is the sole validation boundary; `status` is forced to `"proposed"` on ingest +whatever the answer JSON claims. + +**Draft record** (`drafttests.Draft`): + +| Field | Type | Required | Notes | +|---|---|---|---| +| `id` | string | yes | `^T-\d{3}$` (T-NNN, zero-padded); unique within the file | +| `finding` | string | yes | an existing finding id whose **effective status is `confirmed`** and whose `mode` is not `B` | +| `session` | string | yes | equal to the manifest's `session`, so the link survives the line being copied out of the directory | +| `title` | string | yes | non-empty, at most 200 characters | +| `steps` | []string | yes | non-empty, at most 32 entries, every entry non-empty; in time order | +| `expected` | string | yes | non-empty; the behaviour the participant expected | +| `observed` | string | yes | non-empty; what the system did | +| `rationale_quote` | string | yes | **equal** to the source finding's `quote` (compared in `SafeText` form) | +| `severity` | int | yes | **equal** to the source finding's `severity` | +| `status` | string | yes | always `"proposed"` on ingest (the model is never trusted) | + +```json +{"id":"T-001","finding":"F-001","session":"sample-session", + "title":"Saving gives no confirmation", + "steps":["Open #general in the settings prototype.","Click the Save button ([data-testid=save-btn])."], + "expected":"The save is confirmed on screen.","observed":"Nothing visibly changes.", + "rationale_quote":"I clicked save and nothing happened","severity":3,"status":"proposed"} +``` + +The reference field is named after the kind of record it points at — as the +verdict's `finding` and the decision's `test` are — so one convention covers all +three rather than a second spelling of the same idea. + +**Decision record** (`drafttests.Decision`): + +| Field | Type | Required | Notes | +|---|---|---|---| +| `kind` | string | yes | literal `"decision"` — the discriminator | +| `test` | string | yes | an existing draft id in the file | +| `decision` | string | yes | one of `accepted \| edited \| rejected` | +| `at` | string | yes | ISO date `YYYY-MM-DD` | +| `edit` | object | when edited | a **closed** subset of `{title, steps, expected, observed}` with at least one member, each held to the draft's own rule for that field | + +```json +{"kind":"decision","test":"T-001","decision":"accepted","at":"2026-09-12"} +{"kind":"decision","test":"T-003","decision":"edited","at":"2026-09-12","edit":{"title":"Saving a display name gives no confirmation","steps":["Open #general.","Click Save."]}} +``` + +`edit` is decoded with `DisallowUnknownFields`, so one naming `id`, `finding`, +`session`, `severity`, or `rationale_quote` is a hard error rather than a silently +dropped key: there is no path through which a human edit can re-point a draft at +a different finding or session. The only way to change the link is to reject the +draft and ingest a new one. + +Effective status: every draft starts `proposed`; decision records apply in file +order and the last one for a draft wins. A decision naming an unknown draft, or +carrying a value outside the closed enum, is ignored rather than applied — a draft +in an unrenderable status would otherwise vanish from both the review walk and the +render. An `edited` draft's rendered fields are the last `edited` decision's +`edit` applied over the draft, computed at render time. Design and rationale: +[`../04-surfaces/08-draft-tests.md`](../04-surfaces/08-draft-tests.md). diff --git a/.abcd/development/decisions/adrs/0001-one-append-primitive-and-one-review-verb.md b/.abcd/development/decisions/adrs/0001-one-append-primitive-and-one-review-verb.md new file mode 100644 index 0000000..105ecad --- /dev/null +++ b/.abcd/development/decisions/adrs/0001-one-append-primitive-and-one-review-verb.md @@ -0,0 +1,147 @@ +--- +status: accepted +date: 2026-09-12 +decision-makers: the maintainer +--- + +# One append primitive, and one review verb, across record families + +## Context and Problem Statement + +Testimony's pipeline produces two session artefacts that hold a machine record +plus appended human records: `findings.jsonl` (candidate findings plus verdicts) +and, with the regression-test drafting layer (itd-9), `tests.jsonl` (test drafts +plus decisions). Both are written the same dangerous way — a no-follow open, an +exclusive advisory lock, two size pre-flights against the read-side invariants, a +newline framing over a possibly unterminated last line, a partial-write rollback, +and a returned `Close` error — and both are guarded against a re-ingest +destroying the human record the file exists to hold. + +Two questions arrived together. Where does that write logic live once the second +artefact exists? And does the human decision on a drafted test belong to +`testimony review`, or to a verb of its own? + +The retained human decision is the precision measure the whole method stands on. +Losing one to a subtly divergent second copy of a lock-and-rollback sequence, or +to a decision attached to the wrong record, is the failure mode that matters most +here — not a missing feature. + +## Decision Drivers + +- The human-decision record must never be silently lost, truncated, or + misattributed; every write path that can do so is the same hazard. +- Two copies of a subtle write path drift apart. The comments in the original + single copy already argue this at length for each of its guards. +- The two record families have genuinely different *vocabularies* + (`confirmed | rejected | duplicate` against + `accepted | edited | rejected`, where `edited` carries a replacement payload no + verdict ever does) and genuinely the same *mechanics*. +- The operator learns one pipeline, and the pipeline's shape is already one verb + per step. +- There is exactly one moment when the extraction costs one refactor rather than + two implementations plus a later unification: the moment the second + implementation is written. + +## Considered Options + +1. **Extract two write primitives into `internal/session`; keep one review verb + with a `-kind` selector.** +2. **Copy the write logic into the new package; add a separate `accept-tests` + verb.** +3. **Generalise the whole review machinery over a `Subject` interface, so one + walk serves both families.** +4. **Add a `review -file FILE` flag, letting the path select the record family.** +5. **Defer the extraction: ship the duplicate now, unify later.** + +## Decision Outcome + +Chosen option: **option 1**. + +`internal/session` gains `AppendRecord` (one appended record, with an optional +under-lock `Verify` hook the caller supplies) and `CommitRecords` (a guarded +whole-file replacement). Both are written once and shared: `review.AppendVerdict` +and `drafttests.AppendDecision` append through the first; +`analyze.commitFindings` and `drafttests.commitDrafts` replace through the +second. The per-record *labelling* stays with the callers, because only they can +name a record by its own id or by its position in an answer, and the guard and +verify closures stay with the callers, because only they know what makes their +file's human record worth protecting. + +The human decision stays one verb: `testimony review -kind findings|tests`. The +walks and the vocabularies stay per-kind — `-kind tests` does not reuse the +findings walk — because the prompt-shaped code is about a hundred lines and the +vocabulary difference is real. A flag that belongs to the other record family is +refused as a wrong invocation rather than silently ignored. + +### Consequences + +- Good: the lock, the size pre-flights, the framing, the rollback, and the + returned `Close` error exist in one place, so a fix or a hardening reaches both + artefacts at once. The extraction is behaviour-preserving: both callers' error + strings are unchanged, and their existing assertions pass untouched. +- Good: the regression tests that pinned the two write paths move to + `internal/session` with the code they exercise, so the primitive is tested where + it lives rather than twice through its callers. +- Good: one human-decision verb keeps the surface the intent's own wording asks + for — the decision retained *as the verdicts already are*. +- Bad: `internal/session` grows beyond a pure layout package; it holds two + behaviours, not only a schema. The alternative is worse, and the two behaviours + are precisely the ones every session-file writer needs. +- Bad: `review` gains a mode selector, so its flag surface is larger and the + cross-family refusals have to be stated and tested. +- Neutral: the shared primitives do not fix a caller's own pre-flight duty. Each + caller still holds its records to the line and total caps before committing, + because only it can attribute an oversized record to something the operator can + count to. + +### Confirmation + +The extraction is confirmed by the existing assertions in `internal/review` and +`internal/analyze` passing with their expected strings unmodified, and by the +moved tests in `internal/session` covering the newline framing over an +unterminated file, both size pre-flights, the partial-write rollback on each side, +the `Verify` refusal, and the `Guard` refusal. The one-verb decision is confirmed +by the CLI's exit-2 table refusing each flag against the wrong `-kind`. + +## Pros and Cons of the Options + +### Option 2 — copy the write logic; add `accept-tests` + +- Good: each package reads standalone, with no shared abstraction to understand. +- Bad: two copies of a lock-frame-rollback sequence. Every guard in it exists + because of a specific failure already reasoned about once; a copy inherits the + reasoning only until someone edits one side. +- Bad: a separate verb splits the human surface in two, so the operator learns + where a decision lives per record family rather than learning one verb. + +### Option 3 — generalise review over a `Subject` interface + +- Good: one walk, one prompt loop, one place to improve the interaction. +- Bad: the vocabularies do not unify. `edited` carries a payload, prompts for four + fields, and can collapse to `accepted`; no verdict does any of that. The + interface would buy less than the prompt-shaped code it costs, and it would + couple two surfaces that are free to diverge. + +### Option 4 — `review -file FILE` + +- Good: no new enum; the file name carries the choice. +- Bad: the operator names a path where they mean a record family, and the file + name does not determine the vocabulary — a renamed or copied file would silently + select the wrong walk. + +### Option 5 — defer the extraction + +- Good: the drafting layer ships sooner. +- Bad: the cost rises rather than falls. Deferring buys two implementations plus a + unification, against one refactor taken while both call sites are in hand and + both test suites are green. + +## More Information + +- Surfaces: [`../../brief/04-surfaces/07-review.md`](../../brief/04-surfaces/07-review.md), + [`../../brief/04-surfaces/08-draft-tests.md`](../../brief/04-surfaces/08-draft-tests.md). +- Packages and schemas: + [`../../brief/05-internals/01-packages.md`](../../brief/05-internals/01-packages.md), + [`../../brief/05-internals/02-schemas.md`](../../brief/05-internals/02-schemas.md). +- Intent and spec: `itd-9` (regression-test drafting) and + `spc-2609120417480624`. diff --git a/.abcd/development/intents/drafts/itd-9-regression-test-drafting.md b/.abcd/development/intents/planned/itd-9-regression-test-drafting.md similarity index 79% rename from .abcd/development/intents/drafts/itd-9-regression-test-drafting.md rename to .abcd/development/intents/planned/itd-9-regression-test-drafting.md index 5c748fc..253fc42 100644 --- a/.abcd/development/intents/drafts/itd-9-regression-test-drafting.md +++ b/.abcd/development/intents/planned/itd-9-regression-test-drafting.md @@ -1,8 +1,8 @@ --- id: itd-9 slug: regression-test-drafting -spec_id: null -kind: null +spec_id: spc-2609120417480624 +kind: standalone suggested_kind: null reclassification_history: [] builds_on: [] @@ -37,6 +37,17 @@ Keeping the human in the loop mirrors the stance the rest of the pipeline takes. - Drafting from unverified or rejected findings; only human-confirmed findings are eligible. - Reference-capture findings (itd-4), which are design preferences rather than defects and have nothing to regress against. +## Scope Conditions + +- The session holds a `findings.jsonl` with at least one finding whose current + verdict is `confirmed`; a session with none is staged loudly rather than + drafted from. +- `timeline.jsonl` is present, so each confirmed finding's event window + resolves — the reproduction steps are reconstructed from it. +- The host that answers the drafting request is the operator's chosen model, as + for `analyze`: the CLI emits the request, never calls a model, holds no keys, + and adds no network dependency. + ## Acceptance Criteria - **Given** a finding whose status is `confirmed`, **when** the drafting step runs, **then** a test case draft is produced containing reproduction steps from the event window, the expected and observed behaviour, and the participant's quote. @@ -52,3 +63,7 @@ Keeping the human in the loop mirrors the stance the rest of the pipeline takes. ## Audit Notes _Empty. Populated by intent-fidelity-reviewer when intent moves to shipped/._ + +## Grounds + +- pursued: a confirmed finding's event window plus the participant's own words is enough for a host model to draft a followable regression test, and a human accept/edit/reject pass keeps every draft a proposal; what would show it wrong is drafts whose steps a developer cannot follow from the window alone, which the edited-decision rate on real sessions would reveal diff --git a/.abcd/development/specs/open/spc-2609120417480624-regression-test-drafting.md b/.abcd/development/specs/open/spc-2609120417480624-regression-test-drafting.md new file mode 100644 index 0000000..4ac73d3 --- /dev/null +++ b/.abcd/development/specs/open/spc-2609120417480624-regression-test-drafting.md @@ -0,0 +1,827 @@ +--- +id: spc-2609120417480624 +slug: regression-test-drafting +intent: itd-9 +origin: researcher-authored +production_mode: hand-written +--- +# regression-test-drafting + +## Summary + +`testimony draft-tests` adds the drafting step that turns a **confirmed** +finding into a proposed regression test case, and `testimony review -kind tests` +adds the human accept / edit / reject pass over each draft. The oracle stays +**host-delegated**, exactly as for `analyze`: the CLI never calls a model, holds +no keys, and adds no network dependency. So `draft-tests -session DIR` *emits* a +single self-contained drafting request — a versioned rubric, the session +context, and, for each confirmed finding, that finding's record plus its **event +window** from `timeline.jsonl` — on stdout or to `-out FILE`; any agent host or +human runs it and saves the JSON answer. `draft-tests -session DIR -ingest FILE` +is the **validation boundary**: it checks the answer field-by-field against the +draft schema — every draft must name a finding whose *current* status is +`confirmed`, carry that finding's `quote` byte-for-byte and its `severity` +unchanged, claim the session it actually sits in, and list at least one step — +and writes `tests.jsonl` with every draft forced to `status: proposed`. +`review -kind tests` then walks the proposed drafts (or takes one decision +non-interactively) and records `accepted | edited | rejected` as **appended, +non-destructive records**, so the drafted proposal and the human's decision both +survive, and `edited` carries the human's replacement fields without ever +rewriting the draft line. `draft-tests -session DIR -render` renders the +accepted drafts as Markdown test-case blocks a docs-as-code test plan can hold. + +Eligibility is the whole point of the step: it sits *downstream* of verification, +so only evidence a human already vouched for can become a test. A session with +no confirmed finding is **staged loudly** — `draft-tests` refuses, names the +finding count by status, exits non-zero, and writes nothing. + +Every test in this slice is fixture-based and hermetic; stdlib only. + +## Design + +### Eligibility — which findings are drafted from + +A finding is eligible iff its **effective status is `confirmed`** and its `mode` +is not `B`. Effective status is not recomputed here: `draft-tests` calls +`analyze.Load(dir)` and `analyze.EffectiveStatus(findings, verdicts)`, the single +helper `review` and `report` already use. That is what makes "later verdicts +override earlier ones" true for free — verdict records apply in file order and +the last one for a finding wins, so a finding confirmed then later rejected is +**not** eligible, and one rejected then later confirmed **is**. `duplicate` and +`unverified` are never eligible, including a `duplicate` whose target is +confirmed: the canonical finding is the one that carries the evidence. + +`mode: "B"` is excluded per the intent's out-of-scope bullet (reference-capture +findings are design preferences with nothing to regress against). Nothing +produces Mode B today, so the exclusion is a guard, not a live filter. + +`type` is **not** filtered. AC1 says *a finding whose status is `confirmed`* +without qualification, so filtering out `preference`/`idea` would fail the +criterion as written. The request carries each finding's `type` so the model can +calibrate (a `preference` usually yields a weak "expected behaviour"), and the +human rejects a draft that has nothing to regress against — which is precisely +what the reject verb is for. + +### CLI surface + +``` +testimony draft-tests -session DIR [-window 10] [-out FILE] emit the drafting request (stdout default) +testimony draft-tests -session DIR -ingest FILE validate answer JSON → tests.jsonl (FILE may be "-" for stdin) +testimony draft-tests -session DIR -render [-out FILE] render accepted drafts as Markdown test cases (stdout default) + +testimony review -session DIR -kind tests interactive accept/edit/reject walk +testimony review -session DIR -kind tests -test T-NNN -decision accepted|rejected +testimony review -session DIR -kind tests -test T-NNN -decision edited -edit FILE +``` + +`draft-tests` runs in exactly one mode, mirroring `analyze`'s emit-or-ingest +rule and extending it by one: emit (neither `-ingest` nor `-render`), ingest +(`-ingest`), or render (`-render`). `-ingest` combines with neither `-out` nor +`-render`. `-out` pairs with emit or render. Every flag follows the CLI's +existing exit-2 gauntlet: `-session` required; an explicitly-empty `-out`, +`-ingest`, or `-kind` refused as a wrong invocation (the unset-shell-variable +case); a non-finite `-window` refused (the `report -window` precedent); no +positional arguments (`rejectArgs`). + +| Flag | Default | Meaning | +|---|---|---| +| `-session` | *(required)* | session directory | +| `-window` | `10` | emit mode: the event-window half-width in seconds | +| `-out` | *(stdout)* | emit/render mode: write to `FILE` instead of stdout | +| `-ingest` | *(off)* | ingest mode: validate the answer JSON at `FILE` (or `-` for stdin) into `tests.jsonl` | +| `-render` | *(off)* | render mode: write Markdown test cases for the accepted drafts | + +`review` gains three flags and one mode selector: + +| Flag | Default | Meaning | +|---|---|---| +| `-kind` | `findings` | which record family to review: `findings` or `tests` | +| `-test` | *(interactive)* | non-interactive: the draft to decide (`T-NNN`), `-kind tests` only | +| `-decision` | *(interactive)* | non-interactive: `accepted`, `edited`, or `rejected`, `-kind tests` only | +| `-edit` | *(off)* | with `-decision edited`: the replacement fields as a JSON object at `FILE` (or `-` for stdin) | + +`-finding`/`-verdict` are refused with `-kind tests`, and `-test`/`-decision`/ +`-edit` with `-kind findings`, at exit 2 — a flag that belongs to the other +record family is a wrong invocation, not a silently ignored one. `-kind +findings` is byte-for-byte the behaviour `review` has today. + +Reads by mode: emit reads `manifest.json`, `timeline.jsonl`, and +`findings.jsonl`; ingest reads `manifest.json` and `findings.jsonl` only (drafts +are validated against the *findings*, never re-derived from the timeline); +render reads `manifest.json`, `findings.jsonl`, and `tests.jsonl`. Emit hints to +run `merge` first when the timeline is missing (reusing `analyze`'s +`loadTimeline`, so a duplicated entry id or an unknown `src` is refused there +too); every mode hints to run `analyze -ingest` first when there is no +`findings.jsonl`, and ingest/render/`review -kind tests` hint to run +`draft-tests -ingest` first when there is no `tests.jsonl`. + +### The event window + +The reproduction steps are the expensive part, and the window is the only thing +the model is allowed to reconstruct them from. For finding `f`: + +```go +// Window returns the timeline entries around f, in time order: every entry +// whose time falls in [lo, hi], where lo and hi span f's cited evidence +// entries widened by window on both sides. +func Window(entries []timeline.Entry, f analyze.Finding, window float64) []timeline.Entry +``` + +`lo = min(e.T) - window` and `hi = max(timeline.SpeechEnd(e)) + window` over the +entries `f.Evidence` resolves (ids matched in their `session.SafeText` form, the +form the request shows and `analyze` already validates against). A finding whose +evidence resolves to no entry — impossible after `analyze -ingest`, reachable via +a hand-edited `findings.jsonl` — falls back to `[f.T - window, f.T + window]`. +Speech and event entries are both included: the utterances around the moment are +what carry the *expected* behaviour, and the events are what carry the *steps*. + +`-window` defaults to **10 seconds**, not `report`'s 2.5: `report`'s window +joins an event to the utterance it accompanies, whereas a repro needs the lead-up +and the aftermath. The bundled sample shows why. F-001 cites `utt-004` (22–28 s), +`ev-003` (19.2 s) and `ev-004` (24.1 s). At 2.5 s the window is [16.7, 30.5] and +excludes `utt-003` at 16.0 s — "Now I expect this save button to confirm +somehow", the one utterance in the session that states the expected behaviour. At +10 s the window is [9.2, 38.0] and holds the whole repro: click the display-name +field, type "Alice", click save, click save again, plus the utterances that frame +it. Negative values are legitimate (they narrow the window), matching `report`; +only finiteness is required. + +The window is not separately capped: its size is bounded by `timeline.jsonl`, +which already carries the session's 16 MiB total-size limit, and emit writes +nothing to disk. + +### `draft-tests` — emitting the request (host-delegated) + +`EmitRequest(dir string, window float64) (string, error)` builds one +self-contained prompt so that an agent given **only** this text can answer. +Structure, in order, mirroring `analyze.EmitRequest`: + +1. **Rubric header** — `Testimony regression-test drafting rubric: + testimony-testdraft/v1`. The version is a package constant (`RubricVersion`), + pinning the drafting scheme so drafts are comparable across sessions and a + future revision is explicit. +2. **Stance** — every draft is a *proposal*, born `proposed`; a human accepts, + edits, or rejects it. Only the confirmed findings below are eligible. + Reconstruct steps only from the event window supplied with each finding — + never invent a step, a selector, or a route that is not there. Never alter the + quote, the severity, or the session. +3. **Instructions** — one or more drafts per confirmed finding, in finding-id + order; `steps` in time order, each one imperative action a developer can + follow, naming the selector or route where the window names it, ending at the + moment the finding is anchored to; `expected` the behaviour the participant + expected, grounded in their utterances; `observed` what the system actually + did, grounded in the window's events and utterances; `title` one line naming + the defect. +4. **Rubric body** — the field definitions and the hard constraints restated as + the rules ingest enforces (quote copied byte-for-byte from the finding's + `quote`; `severity` and `session` copied unchanged; `finding` naming the + finding the draft came from; `steps` non-empty). +5. **Session context** — manifest `app`, `participant`, and the ordered `tasks`. +6. **Confirmed findings** — per eligible finding, in id order: a prose line + naming its id, `type`, `severity` and clock, then the finding's own JSON line + in a ```jsonl fence (so the `quote` bytes the model must copy are + unambiguous), then its event window as a ```jsonl fence of timeline entries in + time order. +7. **Required output shape + worked example** — + + > Answer with a single JSON document: `{"rubric":"testimony-testdraft/v1","tests":[ … ]}`. + > A bare top-level array of drafts is also accepted. Output JSON only, no prose. + + ```json + {"rubric":"testimony-testdraft/v1","tests":[ + {"id":"T-001","finding":"F-001","session":"sample-session", + "title":"Saving gives no confirmation", + "steps":["Open #general in the settings prototype.", + "Change the display name to Alice.", + "Click the Save button ([data-testid=save-btn])."], + "expected":"The save is confirmed on screen — a toast, or the button briefly disabled.", + "observed":"Nothing visibly changes, so there is no way to tell the save landed.", + "rationale_quote":"I clicked save and nothing happened", + "severity":3,"status":"proposed"} + ]} + ``` + +**Escaping.** The emitter honours exactly the rule the `[Unreleased]` fix +established for `analyze`: a value rendered as prose or a list item **outside** +any code fence goes through `session.SafeInline` (manifest `app`/`participant`/ +`tasks` via the same `(none)`-fallback helper, and the finding id/type in each +per-finding header), so an attacker-authored manifest or findings file cannot +survive as an active link or an image beacon when a saved `request.md` is +previewed; a value rendered **inside** a fence goes through `session.SafeText` +only (each marshalled finding and timeline line), which strips terminal-control +and Trojan-Source bytes that `json.Marshal` passes through. There is one shared +home for the escape set and this adds no second one. + +Nothing in the session directory is mutated by emit. + +### Draft record — the `Draft` type + +One draft per line of `tests.jsonl`. Draft lines carry no `kind` field; the +schema is closed (`DisallowUnknownFields`). + +| Field | Type | Required | Validation at ingest | +|---|---|---|---| +| `id` | string | yes | matches `^T-\d{3}$`; unique within the answer and the file | +| `finding` | string | yes | names a finding in `findings.jsonl` whose **effective status is `confirmed`** and whose `mode` is not `B` | +| `session` | string | yes | equals the manifest's `session` value | +| `title` | string | yes | non-empty after `SafeText`+trim; at most 200 characters | +| `steps` | []string | yes | non-empty; at most 32 entries; every entry non-empty after `SafeText`+trim | +| `expected` | string | yes | non-empty after `SafeText`+trim | +| `observed` | string | yes | non-empty after `SafeText`+trim | +| `rationale_quote` | string | yes | **equals** the source finding's `quote` (compared in `SafeText` form) | +| `severity` | int | yes | **equals** the source finding's `severity` | +| `status` | string | no | **ignored on input and forced to `"proposed"`** on ingest, whatever the JSON says | + +The field is `finding`, not `finding_id`: the verdict record in `findings.jsonl` +already names its referent `finding`, and the decision record below names its +referent `test`. One convention — *the field is named after the kind of record it +points at* — beats a second spelling of the same idea. + +**Why `rationale_quote` must be equal, not merely verbatim-in-the-timeline.** +`analyze -ingest` already proved the finding's quote is a byte-for-byte substring +of a cited evidence utterance, and a human then confirmed *that* finding. Letting +the drafting model re-derive a quote from the same utterance would let it +substitute different words for the evidence the human vouched for. Requiring +equality makes the drafting step structurally incapable of introducing new +evidence: it can only carry forward what is already on the record. A mismatch is +also the cheapest available signal that the model linked the draft to the wrong +finding, which is why the field is required and validated rather than silently +filled in by the CLI. + +**Why `severity` is copied and equality-checked.** This answers the intent's +third open question: yes, the severity carries through, and it is not the model's +to choose. Triage order is a human product — it came out of the finding's own +severity, which `analyze -ingest` bounded to `1..4` — so the draft restates it +and ingest refuses any restatement that disagrees. + +### `draft-tests -ingest FILE` — the validation boundary + +1. Load `manifest.json` (for `session`) and `findings.jsonl`; compute effective + status; build the eligible set as `id → {quote, severity}`. If the eligible + set is empty, refuse loudly (below) before reading a byte of the answer. +2. Read `FILE` (or stdin when `-`) bounded by `session.MaxAnswerBytes`. Accept a + top-level object with a `tests` array, or a bare array. `rubric`, when + present, must be a known version. +3. Decode each draft with **`DisallowUnknownFields`**, through a `rawDraft` whose + `Severity` is a `*int` so an absent `severity` stays distinguishable from a + present one (the `rawFinding.T` precedent): absent is reported as *missing*, + not as a mismatch against a value the answer never gave. +4. Run every rule in the table. Validation is **transactional and exhaustive**: + collect all errors across all drafts, each naming the draft (its id when + well-formed, otherwise its position *in the answer*, via the + `positioned`/`draftLabel` pattern), the field, and the offending value; if any + error exists, write nothing and exit non-zero. +5. Force `status: "proposed"` on every draft, then pre-flight the serialised size + against `session.MaxJSONLLine` per line and `session.MaxJSONLBytes` in total + (`oversizedDrafts`, the sibling of `analyze.oversizedFindings`), so an + oversized answer leaves the previous `tests.jsonl` untouched rather than + writing a file no reader could scan back. +6. Commit through `session.CommitRecords` (below) with a guard that refuses to + overwrite a `tests.jsonl` already holding any `kind:"decision"` line — the + retained human record, protected exactly as `findings.jsonl`'s verdicts are. +7. Print `validated N test drafts → (all proposed)`. + +An answer with an empty `tests` array (a bare `[]`, `{"tests":[]}`, or a +truncated file) is refused rather than written, so it cannot erase a prior +`tests.jsonl`. + +### The human pass — `review -kind tests` + +**Decision record**, appended to `tests.jsonl`, never an in-place rewrite: + +```json +{"kind":"decision","test":"T-001","decision":"accepted","at":"2026-09-12"} +{"kind":"decision","test":"T-002","decision":"rejected","at":"2026-09-12"} +{"kind":"decision","test":"T-003","decision":"edited","at":"2026-09-12","edit":{"title":"Saving a display name gives no confirmation","steps":["Open #general.","Click Save."]}} +``` + +| Field | Type | Validation | +|---|---|---| +| `kind` | string | literal `"decision"` — the only discriminator; draft lines carry no `kind` | +| `test` | string | an existing draft id in the file | +| `decision` | string | one of `accepted \| edited \| rejected` | +| `edit` | object | required iff `decision == "edited"`; a subset of `{title, steps, expected, observed}` with at least one member present, each held to the draft's own rule for that field | +| `at` | string | ISO date `YYYY-MM-DD` | + +**How AC3's "the draft remains linked to its source finding and session" is +guaranteed.** Three mechanisms, each independently testable: + +1. `tests.jsonl` is append-only. A decision is a new line; the draft line is + never rewritten, so `finding`, `session`, `severity`, `rationale_quote` and + `id` are unreachable by any later write. (Asserted byte-for-byte in the + round-trip test.) +2. The `edit` object is a **closed subset** — `title`, `steps`, `expected`, + `observed` — decoded with `DisallowUnknownFields`. An `edit` naming `finding`, + `session`, `severity`, `rationale_quote` or `id` is a hard error, not a + silently dropped key. There is therefore no path through which a human edit + can re-point a draft at a different finding or session; the only way to change + the link is to reject the draft and ingest a new one. +3. `session` on the draft is validated equal to the manifest's `session`, so the + link survives the draft line being copied *out* of the session directory — + which is what the render step exists to do. + +**Effective status.** `EffectiveStatus(drafts, decisions)`: every draft starts +`proposed`; decision records apply in file order and the last one for an id wins; +a decision naming an unknown draft is ignored for display; a decision whose value +is outside the closed enum is ignored rather than applied (the `ParseRecords` +precedent — a draft in an unrenderable status would otherwise vanish from both +the walk and the render). An `edited` draft's rendered fields are the **last** +`edited` decision's `edit` applied over the draft, computed at render time; the +draft itself is untouched. + +**Interactive** (`review -session DIR -kind tests`): load drafts and decisions, +walk the `proposed` ones in id order, print each and prompt +`[a]ccept [e]dit [r]eject [s]kip [q]uit`. Gated on stdin being a character +device, with the same one-line notice and exit 0 otherwise, so CI never blocks. +Printed block: + +``` +(1/3) T-001 — from F-001 (bug, severity 3), [00:22] + Saving gives no confirmation + steps: + 1. Open #general in the settings prototype. + 2. Change the display name to Alice. + 3. Click the Save button ([data-testid=save-btn]). + expected: The save is confirmed on screen — a toast, or the button briefly disabled. + observed: Nothing visibly changes, so there is no way to tell the save landed. + “I clicked save and nothing happened” +[a]ccept [e]dit [r]eject [s]kip [q]uit: +``` + +Every field passes through `session.SafeText` with a placeholder fallback for one +that renders as nothing, matching `review.printFinding`. `e` prompts for each +editable field in turn showing the current value, blank keeping it, and for +`steps` reads lines until a blank one; if nothing changed it prints +` no changes; recorded as accepted.` and records `accepted` — an `edited` +decision with an empty `edit` is not representable, so it is never written. +Recording echoes ` recorded: T-001 accepted (2026-09-12)`. + +**Non-interactive** (`-test T-001 -decision accepted`): validate that the draft +exists and the decision parses; append one record; print +`recorded: T-001 accepted (2026-09-12)`. `-decision edited` requires +`-edit FILE` (or `-` for stdin), a JSON object holding the replacement fields, +read through `session.OpenFileNoFollowRead` and decoded with +`DisallowUnknownFields` and the same field rules as the interactive edit. Every +interactive path in this repo has a non-interactive twin; making `edited` the one +exception would put the only lossy decision out of reach of a script or an agent +host. A decision may be appended even when one already exists (append-only +correction; latest wins). + +**Under-lock target check.** `AppendDecision` re-reads the current drafts under +its exclusive lock and refuses if the targeted id has vanished or now names a +different draft — the `review.verifyTarget` mechanism, via +`drafttests.SameIdentity` (equality in every field but `Status`). `review -kind +tests` snapshots the drafts once and then blocks on the operator, and a +concurrent `draft-tests -ingest` may truncate-and-rewrite in that gap (permitted +until the first decision exists), and draft ids restart at `T-001`, so without +the re-check a decision would silently attach to a different draft. + +### The honest reuse — two shared primitives in `internal/session` + +`review`'s verdict machinery is *not* generalised over two record families, and +`review -kind tests` does not reuse `review`'s findings walk. The walk's +vocabulary genuinely differs (`accepted|edited|rejected`, where `edited` carries +a payload no verdict ever does), and forcing both through a `Subject` interface +would buy less than the ~100 lines of prompt-shaped code it costs. What must +**not** be duplicated is the dangerous part — and there are two such operations, +each currently written once and each about to be written twice: + +```go +// internal/session + +// Append is one record appended to a session JSONL file. +type Append struct { + Path string // the file to append to + Record []byte // the encoded record, without its newline + Label string // names the record in the line-limit error, e.g. "verdict for F-001" + Kind string // names the record kind in the file-limit error, e.g. "verdict" + Verify func(current io.Reader) error // optional re-check of the file's contents, run under the lock +} + +// AppendRecord appends a.Record as its own physical line: it opens Path under +// the no-follow guard (O_APPEND|O_RDWR), takes an exclusive advisory lock, +// pre-flights the record against MaxJSONLLine and the file against +// MaxJSONLBytes, runs a.Verify over the current contents, frames the record +// with a leading newline when the file does not already end in one, writes it, +// truncates back to the pre-write length on a short write, and returns the +// Close error so a record is never reported written when its bytes did not +// reach disk. +func AppendRecord(a Append) error + +// Commit is a whole-file replacement of a session JSONL file. +type Commit struct { + Path string + Records [][]byte // each encoded record, without its newline + Guard func(current io.Reader) error // refuse the replacement by returning an error +} + +// CommitRecords replaces Path's contents with Records under the same no-follow +// guard and exclusive lock: it runs Guard over the current contents, encodes +// the whole set into one buffer before truncating, writes it as a single Write, +// rolls the file back to empty on a short write (an empty JSONL file is +// parseable and re-ingestable, so the failure state does not foreclose its own +// repair), and returns the Close error. +func CommitRecords(c Commit) error +``` + +Callers after the extraction: + +- `review.AppendVerdict` → `session.AppendRecord` with + `Label: "verdict for " + SafeText(v.Finding)`, `Kind: "verdict"`, and + `Verify` wrapping the existing `verifyTarget` logic. `review.writeVerdict` and + its `verdictFile` interface are deleted; their behaviour and their two size + error messages move verbatim (the `Label`/`Kind` parameters and + `filepath.Base(Path)` reproduce today's strings byte-for-byte, so review's + existing message assertions keep passing unchanged). +- `analyze.commitFindings`/`writeFindings` → `session.CommitRecords` with + `Guard: holdsVerdicts` and its existing refusal message. The + `findingsFile` interface is deleted. +- `drafttests.AppendDecision` → `session.AppendRecord`. +- `drafttests.commitDrafts` → `session.CommitRecords` with `Guard: holdsDecisions`. + +The per-record size **labelling** stays with the callers +(`oversizedFindings`/`oversizedDrafts`), because only they can name a record by +its own id or its position in an answer. + +Rejected alternatives, for the record: copying the ~70 lines of lock/frame/ +rollback logic into the new package (two copies of a subtle write path drifting +apart is exactly the hazard the existing comments argue against); a +`review -file FILE` flag (the operator would be naming a path where they mean a +record family, and the file name does not determine the vocabulary); doing the +extraction later (this is the one moment where the cost is one refactor instead +of two implementations plus a unification). + +### `draft-tests -render` — the docs-as-code form + +`Render(dir string) (string, error)` writes one Markdown test-case block per +draft whose effective status is `accepted` or `edited`, in id order, with the +last `edited` decision's fields applied. `proposed` and `rejected` drafts are +omitted: a proposal is not a test, and a rejected draft is retained in +`tests.jsonl` for the record, not for the plan. Output (stdout by default, or +`-out FILE`, which prints `wrote `): + +```markdown +# Regression tests — sample-session + +Drafted from confirmed findings in session `sample-session` (app `testimony demo`, +participant `P1`). 2 of 3 drafts accepted. + +## T-001 — Saving gives no confirmation + +- **Source:** finding `F-001` (bug, severity 3) in session `sample-session`, at [00:22] +- **Decision:** accepted (2026-09-12) + +**Steps** + +1. Open #general in the settings prototype. +2. Change the display name to Alice. +3. Click the Save button ([data-testid=save-btn]). + +**Expected:** The save is confirmed on screen — a toast, or the button briefly disabled. + +**Observed:** Nothing visibly changes, so there is no way to tell the save landed. + +**Rationale (participant, [00:22]):** “I clicked save and nothing happened” +``` + +Every inserted value goes through `session.SafeInline` — the one shared home for +the escape set that `report.md` and the emitted request already use — so an +attacker-authored draft cannot forge Markdown structure, an active link, or an +image beacon in a document the operator pastes into their own repository. The +clock is rendered `[MM:SS]` from the *finding's* `t`, with a leading `-` for a +negative time, matching `report`. Render writes nothing into the session +directory unless `-out` names a path there; the artefact is a hand-off copy, so +it defaults to stdout. + +Why a mode on `draft-tests` rather than a section in `report.md`: `report.md` is +the session record, written at a fixed path, and would have to carry the section +whether or not anyone accepted a draft; the test plan is a different artefact +with a different audience and lives wherever the operator's docs-as-code plan +lives. `report` keeps its single mode and single output path. + +### Loud staging + +Both refusals write nothing and exit 1 (a well-formed invocation whose work +cannot be done), naming the counts so the operator can see *why* they are empty: + +``` +testimony: no confirmed findings to draft tests from (5 findings: 0 confirmed, 2 unverified, 1 duplicate, 1 rejected); confirm one with `testimony review -session sessions/x` first +testimony: no accepted test drafts to render (3 drafts: 0 accepted, 0 edited, 2 proposed, 1 rejected); accept one with `testimony review -session sessions/x -kind tests` first +``` + +The first applies to emit **and** ingest (with no eligible finding there is +nothing a draft could legally reference). The second keeps `-out FILE` from +truncating an existing test plan into an empty document, which is the same +reasoning behind `analyze -ingest`'s empty-answer refusal. + +### Package layout & session constants + +- **`internal/drafttests`** (new) — `Draft` and `Decision` types, `RubricVersion`, + `Load`/`ParseRecords`, `EffectiveStatus`, `SameIdentity`, `Window`, + `EmitRequest`, `Ingest`, `Render`, `Review` (the walk and the single-decision + path), `AppendDecision`, and the unexported `validate`/`oversizedDrafts`/ + `holdsDecisions`. Imports `analyze`, `session`, `timeline`. +- **`internal/review`** — gains the `-kind` dispatch (`Options.Kind`); its + findings path is unchanged except that `AppendVerdict` now calls + `session.AppendRecord`. +- **`internal/analyze`** — `commitFindings` now calls `session.CommitRecords`; + `maxAnswerBytes` moves to `session.MaxAnswerBytes` (16 MiB) beside + `MaxJSONLLine`/`MaxJSONLBytes`, where the shared caps already live. +- **`internal/session`** — gains `TestsFile = "tests.jsonl"`, + `MaxAnswerBytes`, `Append`/`AppendRecord`, and `Commit`/`CommitRecords`. +- **`internal/cli`** — the `draft-tests` case, `review`'s new flags, and the + usage text. + +The rendered Markdown gets **no** session constant: it is a hand-off artefact +whose destination the operator chooses, and stdout is the default. + +### Sample session & the schema-move invariant + +`tests.jsonl` is a new session artefact, so code, sample, tests and the brief's +schema page move in the same change. `examples/sample-session/` gains a +`tests.jsonl` whose three drafts all reference `F-001` — the one confirmed +finding in the bundled `findings.jsonl` — because a finding may legitimately +yield more than one test case, and three drafts let the sample exercise all three +decisions and the render filter: `T-001` **accepted**, `T-002` **edited**, +`T-003` **rejected**. Each draft's `rationale_quote` is F-001's quote verbatim +and each `severity` is `3`, so the bundled file is itself a fixture that passes +ingest. Render on the sample therefore produces two blocks, which CI can grep. + +## Acceptance-criteria mapping + +**AC1** — *Given a finding whose status is `confirmed`, when the drafting step +runs, then a test case draft is produced containing reproduction steps from the +event window, the expected and observed behaviour, and the participant's quote.* + +- *Mechanism:* emit carries exactly the eligible findings and, for each, its + event window (`Window`, `-window` default 10 s); the draft schema makes + `steps` (non-empty), `expected`, `observed` and `rationale_quote` all + **required**, and ingest refuses a draft missing any of them or whose quote is + not the finding's quote byte-for-byte. +- *Tests:* `TestEmitCarriesConfirmedFindingsAndWindows`, + `TestWindowSpansEvidenceWidenedByWindow`, `TestIngestRequiresSteps`, + `TestIngestRequiresExpectedAndObserved`, + `TestIngestRejectsQuoteThatIsNotTheFindingsQuote`, and the round-trip golden. +- **Flagged, as itd-2's AC3 was:** the CLI guarantees that a draft *contains* + steps and that the request it came from carried *only* the event window. It + cannot verify that a given step was in fact derived from the window — `steps` + is free prose, and a validator that tried would either reject honest + paraphrase or accept fabrication. AC1 is therefore met **at the request and + schema level**, with the human accept/edit/reject pass as the check on + content — which is the same second-coder stance the whole layer rests on. The + 10-second default window is the substantive part of the commitment, and the + worked sample above is the evidence it is wide enough to reconstruct a repro. + +**AC2** — *Given a finding whose status is `unverified` or `rejected`, when the +drafting step runs, then no test case is drafted for it.* + +- *Mechanism:* enforced on both sides. Emit omits every non-`confirmed` finding + from the request (so the model never sees it), and ingest independently + refuses any draft whose `finding` is not currently `confirmed` — so a + hand-written or stale answer cannot smuggle one in. Effective status comes + from `analyze.EffectiveStatus`, so a later verdict overriding an earlier one is + honoured. +- *Tests:* `TestEmitOmitsUnverifiedRejectedAndDuplicateFindings`, + `TestIngestRejectsDraftOfUnverifiedFinding`, + `TestIngestRejectsDraftOfRejectedFinding`, + `TestIngestRejectsDraftOfDuplicateFinding`, + `TestEligibilityHonoursLastVerdict` (confirmed-then-rejected excluded, + rejected-then-confirmed included), `TestEmitRefusesWithNoConfirmedFindings`. + +**AC3** — *Given a drafted test case, when a human accepts or rejects it, then +the decision is retained and the draft remains linked to its source finding and +session.* + +- *Mechanism:* decisions are appended records (`kind:"decision"`), never in-place + rewrites; the `edit` object is a closed four-field subset decoded with + `DisallowUnknownFields`, so no decision can reach `finding`, `session`, + `severity`, `rationale_quote` or `id`; `session` is validated equal to the + manifest's, so the link travels with a line copied out of the directory; and + `AppendDecision` re-checks the target under its lock. +- *Tests:* `TestDecisionIsAppendedAndDraftLinesUnchanged` (byte-for-byte), + `TestEditCannotNameFindingOrSessionOrSeverityOrQuoteOrID`, + `TestEffectiveStatusLastDecisionWins`, + `TestIngestRejectsSessionMismatch`, + `TestAppendDecisionRefusesWhenDraftChangedUnderTheLock`, and the interactive + walk tests for `a`/`e`/`r`. + +**Scope bullet 4** — *Emitting the draft in a form the docs-as-code manual test +records can hold.* Met by `draft-tests -render` (one Markdown test-case block per +accepted or edited draft, source finding and session named in each block). Tests: +`TestRenderGoldenFromSampleSession`, `TestRenderOmitsProposedAndRejected`, +`TestRenderAppliesLastEdit`, `TestRenderRefusesWithNoAcceptedDrafts`. + +No acceptance criterion is unmeetable as written. The one reduction is AC1's +"steps *from the event window*", flagged above as a request-level rather than +validator-level guarantee. + +## Decisions on open questions + +**Open question 1 — does the event window alone yield reproduction steps a +developer can follow, or does a reliable repro need the keyframe channel as +well?** The window alone, at a 10-second default half-width, with `edited` in +the decision vocabulary as the recorded repair when it is not enough. The +sample's F-001 window holds the full click-type-click-click sequence plus the +utterance stating the expectation, which is a followable repro. Keyframes are +deferred for the same reason itd-2 deferred them: extraction needs local video +and a multimodal pass, neither of which fits CI or the local-only privacy +boundary, and Mode B (itd-4) is where that channel gets built. The design leaves +the seam open — `Window` already returns whole timeline entries, so a later +revision can attach a keyframe reference to an entry without changing the prompt +contract. Flagged for the maintainer rather than silently chosen. + +**Open question 2 — where do accepted drafts live?** Both, with the boundary +drawn explicitly: the **record** lives alongside the session, as `tests.jsonl`, +append-only and linked to its finding; the **hand-off** is +`draft-tests -render`, whose output the operator places wherever their +docs-as-code test plan lives. Testimony never writes into the application's +repository. This keeps the evidence chain inside the session (which is the +exchange unit the whole tool is built around) and keeps Testimony out of the +business of knowing another repository's layout or test conventions — which the +intent's out-of-scope bullet already rules out for test *code*. + +**Open question 3 — should a drafted test carry the finding's severity +through?** Yes, and it is copied rather than chosen: `severity` is required on +every draft and ingest refuses any value that disagrees with the source +finding's. Triage order is a human product and survives the hand-off unaltered. + +**Further decisions taken.** + +- **One verb per pipeline step, mirroring the existing shape.** `draft-tests` is + the machine step (emit | ingest, plus render), and the human step is + `review -kind tests` — one human-decision verb for the whole pipeline, which + is what the intent's own wording ("the decision retained *as the verdicts + already are*") asks for. A separate `accept-tests` verb would have split the + human surface in two. +- **Two shared write primitives, not a generalised review.** `session.AppendRecord` + and `session.CommitRecords` hold the lock/pre-flight/framing/rollback logic + once; the walks and the vocabularies stay per-kind. Reasoning and rejected + alternatives above. +- **`tests.jsonl`, and the draft field is `finding`.** The file name mirrors + `findings.jsonl` (both hold a machine record plus appended human records) and + the constant mirrors the others (`session.TestsFile`). The reference field is + named after the kind of record it points at, as the verdict's `finding` and the + decision's `test` are. +- **Quote equality, not re-derivation**, so the drafting step cannot introduce + evidence; **`session` equality**, so the link survives the line being copied + out; **`status` forced to `proposed`**, so a draft can never be born accepted — + the same laundering `analyze -ingest` applies to `unverified`. +- **`-window` default 10 s**, not `report`'s 2.5 s, with the sample-session + worked example as the justification. +- **No `type` filter**; `mode: "B"` excluded as a guard. +- **Loud staging** on zero confirmed findings and zero accepted drafts: exit 1, + counts by status, nothing written. +- **`-decision edited` requires `-edit FILE`**, so every interactive decision has + a non-interactive twin. +- **Validator strictness mirrors `analyze` exactly**: `DisallowUnknownFields`, + transactional and exhaustive error collection, positional labels taken from the + answer rather than from a filtered slice, and `SafeText`-form comparison for + every value the model was shown in sanitised form. + +## Test plan + +Hermetic and fixture-based; CI-safe on ubuntu with no model, network, tool, or +TTY. Fixtures live in `internal/drafttests/testdata/`. + +**Emit.** Substring and golden asserts on `EmitRequest`: the rubric version +header; the stance, instruction and rubric bodies; the manifest task list; the +output-shape example; the eligible finding's JSON line and its window entries +present; non-eligible findings (`F-002` unverified, `F-003` rejected, `F-005` +duplicate) absent; entries outside the window absent; `-window` widening and +narrowing the window set; determinism across runs. Escaping: a manifest `app` of +`[x](http://example.test/beacon.png)` renders escaped, and a timeline entry text +carrying `U+202E` renders stripped. + +**The window.** Unit tests for `Window`: span over multiple evidence entries; +`SpeechEnd` used for the upper bound; an event-only evidence set; a negative +window narrowing; the `f.T ± window` fallback when evidence resolves to nothing; +inclusive boundaries. + +**Eligibility.** `TestEligibilityHonoursLastVerdict` — confirmed-then-rejected +excluded, rejected-then-confirmed included, duplicate-of-confirmed excluded, +`mode: "B"` excluded. + +**Ingest validation-failure fixtures** — one per rule, each proving a precise +message: bad id (`T-12`, `X-001`); duplicate id; unknown `finding`; `finding` +that is unverified / rejected / duplicate (three fixtures, AC2's ingest side); +`session` mismatch; empty and whitespace-only `title`; over-long `title`; absent, +empty, and whitespace-only-entry `steps`; over-long `steps`; empty `expected`; +empty `observed`; `rationale_quote` differing by one byte; absent `severity`; +mismatched `severity`; unknown field (`stpes`); a non-object element; empty +`tests` array; unknown `rubric`; an answer over `MaxAnswerBytes`; a draft whose +line exceeds `MaxJSONLLine`; a set exceeding `MaxJSONLBytes`. Plus +status-forcing (`status:"accepted"` in, `proposed` out), and the refusal to +overwrite a `tests.jsonl` holding decision records. Transactionality: an answer +with three bad drafts reports all three errors in one run and writes nothing; +positional labels point at the answer's own numbering when an earlier element +failed to decode. + +**Decisions.** `-test T-001 -decision accepted` appends one record and leaves +every draft line byte-unchanged; `rejected`; `edited` with an `-edit` file and +with `-edit -` on stdin; an `edit` naming `finding`, `session`, `severity`, +`rationale_quote` or `id` rejected; an `edit` with no members rejected; an +unknown draft id rejected; last-decision-wins including accepted-then-rejected; +`EffectiveStatus` unit tests; `AppendDecision` refusing when the draft changed +under the lock. + +**Interactive walk.** Scripted `In` with `IsTTY: true`: `a`, `e` (each field, +including blank-keeps and the all-blank → recorded-as-accepted path), `r`, `s`, +`q`, an unrecognised key re-prompting, end-of-input stopping cleanly, the +no-proposed-drafts notice, and the not-a-character-device notice exiting 0. + +**Render.** Golden `tests.md` from the sample session; only accepted and edited +drafts rendered; the last `edit` applied over the draft; the counts line; clock +rendering including a negative time; `SafeInline` escaping of a draft field +carrying an inline-Markdown trigger; the zero-accepted refusal writing nothing. + +**Shared primitives.** `internal/session` gains the tests moved from +`internal/review` and `internal/analyze`: `AppendRecord`'s newline framing over +an unterminated file, its line- and total-size pre-flights, its partial-write +rollback (via a fake satisfying the file interface), and its `Verify` refusal +path; `CommitRecords`' `Guard` refusal, truncate-then-write, and +rollback-to-empty. Both callers' existing error strings are asserted unchanged in +`internal/review` and `internal/analyze`, so the extraction is provably +behaviour-preserving. + +**CLI.** Exit-2 table: missing `-session`; explicitly-empty `-out`/`-ingest`/ +`-kind`/`-test`/`-decision`/`-edit`; `-out` with `-ingest`; `-render` with +`-ingest`; non-finite `-window`; a positional argument; an unknown `-kind`; a +bad `-test`; a bad `-decision`; `-decision edited` without `-edit`; `-edit` +without `-decision edited`; `-finding`/`-verdict` with `-kind tests` and +`-test`/`-decision`/`-edit` with `-kind findings`. Exit-1: both loud-staging +refusals, and the missing-`tests.jsonl` hint. + +**Round-trip golden.** Sample session: `merge` → `draft-tests` (emit) → ingest a +known-good answer fixture → `review -kind tests` three decisions +(accepted / edited / rejected) → `draft-tests -render`, asserting the golden +Markdown, the append-only property, and that `findings.jsonl` is untouched +throughout. + +**Sample smoke (CI).** `./testimony draft-tests -render -session +examples/sample-session` grepped for `T-001` — it needs no merged timeline, so it +runs before the existing `merge`/`report` smoke; and `./testimony draft-tests +-session examples/sample-session` after `merge`, asserting exit 0. Added to the +`AGENTS.md` command block and `.github/workflows/ci.yml`. + +**Live verification (part of done, not CI).** Run the emitted request against the +maintainer's own host on a real session, ingest the answer, accept one draft and +edit another, render the plan, and read it; fix what it exposes before the PR. + +## Docs plan + +**User-facing (`docs/`, Diátaxis, present tense, British English in prose).** + +- `docs/reference/cli.md` — a new `## testimony draft-tests` section after + `analyze`: the three invocations, the flag table, the exactly-one-mode rule, + what each mode reads, emit behaviour (request structure, the event window and + the 10 s default, the escaping rule, nothing mutated), ingest behaviour (the + schema rules by reference to the session-directory page, transactional + validation, `status` forced to `proposed`, the empty-answer and + decision-holding refusals, the printed line), render behaviour (which statuses + render, the edit application, the zero-accepted refusal), and both + loud-staging messages. `## testimony review` gains the `-kind`, `-test`, + `-decision` and `-edit` rows, the cross-family flag refusals, and a + tests-mode paragraph mirroring its findings paragraphs. +- `docs/reference/session-directory.md` — `tests.jsonl` added to the directory + listing and to the 16 MiB total-size sentence; a new `## tests.jsonl` section + with the Draft record and Decision record tables, an example of each, the + closed-`edit` note, and the effective-status sentence. +- `docs/how-to/draft-regression-tests.md` — new, mirroring + `analyse-a-session.md`'s numbered shape: prerequisite (a session with at least + one confirmed finding), then 1 emit the drafting request, 2 run it with your + assistant of choice, 3 ingest the answer, 4 review the drafts + (`review -kind tests`, interactive and single-decision, including `-edit`), + 5 render the test plan and where to put it. Closes with pointers to the two + reference pages. +- `docs/README.md` — the new how-to added to the How-to guides line. +- `README.md` — "Status and roadmap": `draft-tests` moved into "working today", + and the regression-test bullet out of "Coming next". + +**Durable record (`.abcd/development/`, not user-facing).** + +- `brief/04-surfaces/08-draft-tests.md` — new surface page in the shape of + `06-analyze.md`. +- `brief/04-surfaces/07-review.md` — the `-kind` dispatch and the tests-side + flags. +- `brief/05-internals/01-packages.md` — `internal/drafttests`, and + `internal/session`'s two new write primitives. +- `brief/05-internals/02-schemas.md` — the `tests.jsonl` tables and the + directory listing (the schema-move invariant). +- `decisions/adrs/0001-one-append-primitive-and-one-review-verb.md` — MADR: the + human-decision layer is one verb (`review -kind`) across record families, and + the two dangerous session-file writes live once in `internal/session`. This is + architecture-shaping and is the repository's first ADR. +- `.abcd/work/DECISIONS.md` — one dated line each for: the drafting step is + host-delegated emit/ingest like `analyze`; drafts live in `tests.jsonl` + alongside the session and render out as Markdown; decisions are appended and + the `edit` object cannot reach a draft's link fields; severity is carried + through and equality-checked. +- `.abcd/work/CONTEXT.md` — the pipeline sentence and the "next" pointer. + +**Changelog.** One `### Added` entry under `[Unreleased]` naming +`testimony draft-tests` and `review -kind tests`, the new `tests.jsonl` +artefact, and the two extracted `internal/session` primitives (no behaviour +change for `analyze`/`review`). + +**`AGENTS.md`.** "Current state" moves from seven pipeline commands to eight and +names the drafting layer; the build/test block gains the two new smoke lines. diff --git a/.abcd/work/CONTEXT.md b/.abcd/work/CONTEXT.md index 8ffe89d..fc24859 100644 --- a/.abcd/work/CONTEXT.md +++ b/.abcd/work/CONTEXT.md @@ -12,10 +12,15 @@ Testimony captures usability evidence, on the record. A Go CLI `demo`, `transcribe`, `merge`, and `report` working end-to-end, plus the first-pass analysis layer — `analyze` (emit a host-delegated analysis request, then validate the answer into `findings.jsonl`) and `review` -(record human verdicts, appended non-destructively). The oracle is +(record human verdicts, appended non-destructively) — and the +regression-test drafting layer, `draft-tests` (emit a drafting request +carrying each confirmed finding and its event window, validate the answer +into `tests.jsonl`, render the accepted drafts as a Markdown test plan) +with `review -kind tests` for the accept / edit / reject pass. The oracle is host-delegated: the CLI never calls a model, holds no keys, and adds no -network dependency; every finding is born `unverified` and ingest is the sole -validation boundary. Next is codebase mapping (itd-3), then Mode B / the +network dependency; every finding is born `unverified`, every drafted test is +born `proposed`, and each ingest is the sole validation boundary for its own +answer. Next is codebase mapping (itd-3), then Mode B / the pattern library (itd-4). Command and file contracts in [`../../docs/reference/cli.md`](../../docs/reference/cli.md) and [`../../docs/reference/session-directory.md`](../../docs/reference/session-directory.md). diff --git a/.abcd/work/DECISIONS.md b/.abcd/work/DECISIONS.md index f5116d5..be6bad5 100644 --- a/.abcd/work/DECISIONS.md +++ b/.abcd/work/DECISIONS.md @@ -1617,3 +1617,28 @@ Architecture-shaping decisions graduate to an ADR under footers or session links: disclosure lives solely in the commit's `Assisted-by:` trailer. Both recorded as COMMITTING rules in `.abcd/rules.json` after a harness-default footer reached two PR bodies. +- 2026-09-12 — The regression-test drafting step (itd-9) is host-delegated + emit/ingest, exactly as `analyze` is: `draft-tests` emits one self-contained + request carrying each confirmed finding and its event window (`-window` + defaulting to 10 s, not `report`'s 2.5, so the repro's lead-up and aftermath are + both in scope), and `draft-tests -ingest` is the sole validation boundary, with + every draft forced to `status: "proposed"`. The CLI still never calls a model, + holds no keys, and adds no network dependency. +- 2026-09-12 — Drafted tests live in two places with the boundary drawn + explicitly: the record is `tests.jsonl` alongside the session, append-only and + linked to its finding, and the hand-off is `draft-tests -render`, whose Markdown + the operator places wherever their docs-as-code test plan lives. Testimony never + writes into the application's repository, so the rendered plan defaults to stdout + and gets no session constant. +- 2026-09-12 — A human decision on a draft is an appended `kind:"decision"` + record, never an in-place rewrite, and its `edit` object is a closed + `{title, steps, expected, observed}` subset decoded with + `DisallowUnknownFields`: an `edit` naming `id`, `finding`, `session`, + `severity`, or `rationale_quote` is a hard error, so no edit can re-point a + draft at different evidence. Changing the link means rejecting the draft and + ingesting a new one. +- 2026-09-12 — A draft carries its source finding's `severity` and `quote` + through, and ingest equality-checks both: triage order is a human product and + not the model's to choose, and requiring the quote byte for byte makes the + drafting step structurally incapable of introducing evidence the human never + vouched for. diff --git a/CHANGELOG.md b/CHANGELOG.md index 20edd61..06e72b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,35 @@ break an existing invocation is called out in the entry that records it. ### Added +- `testimony draft-tests` turns a **confirmed** finding into a proposed + regression test case, and `testimony review -kind tests` records the human + accept / edit / reject pass over each draft. The oracle stays host-delegated, + exactly as for `analyze`: `draft-tests -session DIR` emits one self-contained + drafting request — a versioned rubric plus, for each confirmed finding, that + finding's record and its **event window** from `timeline.jsonl` (`-window` + defaults to 10 seconds, wide enough to hold a repro's lead-up and aftermath) — + and `draft-tests -session DIR -ingest FILE` is the validation boundary, forcing + every draft to `status: proposed`, requiring its `rationale_quote` and + `severity` to equal the source finding's, and refusing any draft of a finding + that is not currently confirmed. Drafts and decisions live in a new + `tests.jsonl` session artefact, append-only in both directions: a decision is a + new line, and its `edit` object is a closed `{title, steps, expected, + observed}` subset, so no human edit can re-point a draft at a different finding + or session. `draft-tests -session DIR -render` renders the accepted drafts as + Markdown test-case blocks a docs-as-code test plan can hold, each naming the + finding and session it came from. A session with no confirmed finding, or a plan + with no accepted draft, is staged loudly: the counts by status, nothing written, + exit 1. +- The two dangerous session-file writes now live once in `internal/session`, as + `AppendRecord` (one appended record: the no-follow open, the exclusive lock, the + line- and total-size pre-flights, the newline framing over an unterminated last + line, an optional under-lock target re-check, the partial-write rollback, and + the returned `Close` error) and `CommitRecords` (a guarded whole-file + replacement, buffered before the truncate and rolled back to empty on a short + write). `analyze -ingest` and `testimony review` write through them with no + change in behaviour: the two size refusals and the verdict-overwrite guard are + byte-for-byte the messages they always were, and the drafting layer shares the + primitives rather than carrying a second copy of a subtle write path. - `transcribe` prints an elapsed-time status line every 5 seconds while the ASR engine is still running, instead of staying silent between the offset line and completion — a CPU-only `whisperx`/`whisper-cli` run can take diff --git a/README.md b/README.md index d41d16c..0bb064f 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,8 @@ The demo app contains at least one intentional usability flaw. Find it by talkin - [Tutorials](docs/tutorials/getting-started.md) — your first session, end to end. - [How-to guides](docs/how-to/) — [transcribe a recording](docs/how-to/transcribe-a-recording.md) (engines, languages, offsets), [analyse a session](docs/how-to/analyse-a-session.md) - (findings and verdicts), [instrument your own app](docs/how-to/instrument-your-own-app.md). + (findings and verdicts), [draft regression tests](docs/how-to/draft-regression-tests.md) + (drafts and decisions), [instrument your own app](docs/how-to/instrument-your-own-app.md). - [Reference](docs/reference/) — the [command line](docs/reference/cli.md) and the [session directory](docs/reference/session-directory.md). - [Explanation](docs/explanation/) — [how alignment works](docs/explanation/how-alignment-works.md), @@ -100,6 +101,7 @@ sessions// transcript.jsonl # time-aligned utterances timeline.jsonl # merged, session-relative timeline findings.jsonl # analysis findings + verdicts + tests.jsonl # regression-test drafts + decisions report.md # human-readable aligned record ``` @@ -109,12 +111,16 @@ Exact schemas: [session directory reference](docs/reference/session-directory.md Working today: `record` (managed capture — one command starts the recorders and stamps the session), `demo` (instrumented capture), `transcribe` (local WhisperX -or whisper.cpp), `merge`, `report`, and the first-pass analysis layer — `analyze` +or whisper.cpp), `merge`, `report`, the first-pass analysis layer — `analyze` (emit an analysis request, then validate the answer into findings) and `review` -(record human verdicts). `record` captures the microphone by default; screen -video is opt-in with `-video`. Analysis is host-delegated — `analyze` never calls -a model, holds no keys, and adds no network dependency — and every finding is -*unverified* by default until you confirm or reject it. +(record human verdicts) — and the regression-test drafting layer, `draft-tests` +(turn a confirmed finding into a proposed test case, then render the accepted +ones as a Markdown test plan) with `review -kind tests` for the accept / edit / +reject pass. `record` captures the microphone by default; screen video is opt-in +with `-video`. The model work is host-delegated — the CLI never calls a model, +holds no keys, and adds no network dependency — every finding is *unverified* +until you confirm or reject it, and every drafted test is a *proposal* until you +accept it. Coming next, in user terms: diff --git a/docs/README.md b/docs/README.md index 58c3142..f3021bd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # Testimony documentation - **[Tutorials](tutorials/getting-started.md)** — learn by doing: capture, transcribe, and report on your first session in about five minutes. -- **[How-to guides](how-to/)** — recipes for specific tasks: [transcribe a recording](how-to/transcribe-a-recording.md), [instrument your own app](how-to/instrument-your-own-app.md), [analyse a session](how-to/analyse-a-session.md). +- **[How-to guides](how-to/)** — recipes for specific tasks: [transcribe a recording](how-to/transcribe-a-recording.md), [instrument your own app](how-to/instrument-your-own-app.md), [analyse a session](how-to/analyse-a-session.md), [draft regression tests](how-to/draft-regression-tests.md). - **[Reference](reference/)** — exact descriptions of the [command line](reference/cli.md) and the [session directory](reference/session-directory.md). - **[Explanation](explanation/)** — background and reasoning: [how alignment works](explanation/how-alignment-works.md), [privacy](explanation/privacy.md). diff --git a/docs/how-to/draft-regression-tests.md b/docs/how-to/draft-regression-tests.md new file mode 100644 index 0000000..df9bb37 --- /dev/null +++ b/docs/how-to/draft-regression-tests.md @@ -0,0 +1,171 @@ +# Draft regression tests + +This guide covers the drafting layer: turning a **confirmed** finding into a +proposed regression test case, deciding its fate by hand, and rendering the +accepted ones as a Markdown test plan your docs-as-code records can hold. +Testimony delegates the model work to an assistant of your choice — +`draft-tests` never calls a model, holds no keys, and adds no network +dependency. It emits a request, you run it, and it validates the answer. + +Prerequisite: a session with at least one finding whose current verdict is +`confirmed`, and a merged `timeline.jsonl`. The step sits downstream of +verification on purpose — only evidence a person already vouched for can become +a test — so [analyse the session](analyse-a-session.md) first and confirm what +you believe. A session with no confirmed finding is refused, with the finding +count by status, rather than drafted from. + +The flow is five steps: **emit** the drafting request, **run** it with your +assistant, **ingest** the answer, **review** the drafts, and **render** the plan. + +## 1. Emit the drafting request + +`testimony draft-tests` writes a single, self-contained prompt: a versioned +rubric, the session context, and — for each confirmed finding — that finding's +own record plus its **event window** from the timeline. Send it to stdout to read +it, or to a file to hand off: + +```sh +testimony draft-tests -session sessions/ -out request.md +``` + +The event window is the only material the steps may be reconstructed from, so its +width matters. `-window` sets the half-width in seconds around the finding's +cited evidence, and it defaults to **10** rather than `report`'s 2.5: a repro +needs the lead-up and the aftermath, not only the moment. On the bundled sample +the 10-second window around `F-001` holds the whole sequence — click the +display-name field, type "Alice", click save, click save again — plus the +utterance in which the participant states what they expected. At 2.5 seconds that +utterance falls outside the window, and a draft made from it has nothing to +ground "expected" in. + +```sh +testimony draft-tests -session sessions/ -window 20 # a slower, more deliberate session +``` + +Nothing in the session directory changes. + +## 2. Run it with your assistant of choice + +Give the request to any assistant — a chat model, an agent host, or a colleague. +Ask it to follow the instructions and return **only** the JSON answer. Save that +answer to a file, for example `tests.json`. + +The expected shape is a JSON object with a `tests` array (a bare array is also +accepted): + +```json +{"rubric":"testimony-testdraft/v1","tests":[ + {"id":"T-001","finding":"F-001","session":"sample-session", + "title":"Saving gives no confirmation", + "steps":["Open #general in the settings prototype.", + "Change the display name to Alice.", + "Click the Save button ([data-testid=save-btn])."], + "expected":"The save is confirmed on screen — a toast, or the button briefly disabled.", + "observed":"Nothing visibly changes, so there is no way to tell the save landed.", + "rationale_quote":"I clicked save and nothing happened", + "severity":3,"status":"proposed"} +]} +``` + +One finding may yield more than one draft: a defect that shows up in two +distinct sequences is two test cases. + +## 3. Ingest the answer + +Validate the answer against the draft schema and write `tests.jsonl`: + +```sh +testimony draft-tests -session sessions/ -ingest tests.json +``` + +Ingest is the validation boundary, and it never trusts the model. It rejects, +with a precise message, any draft that names a finding which is not currently +`confirmed`, claims a session other than this one, carries a +`rationale_quote` that is not its finding's quote byte for byte, restates a +different `severity`, lists no step, leaves `title`, `expected`, or `observed` +empty, or carries a stray field. All errors are reported at once and nothing is +written until the whole answer is clean, so you can fix a batch in one pass. +Every draft lands `status: proposed`, whatever the answer claimed. + +The quote and the severity are held to **equality** rather than merely checked +for plausibility. That is what keeps the drafting step incapable of introducing +evidence: it can only carry forward what a person already confirmed, and a +mismatch is the cheapest signal that the draft was linked to the wrong finding. + +You can also pipe the answer straight in with `-ingest -`: + +```sh +your-assistant < request.md | testimony draft-tests -session sessions/ -ingest - +``` + +Once a decision exists in `tests.jsonl`, ingest refuses to overwrite the file: +the human record is retained, so a re-draft needs a fresh session directory or a +removed `tests.jsonl`. + +## 4. Review the drafts + +Each draft is a *proposal* until you judge it. `testimony review -kind tests` +walks the proposed drafts and records your decision: + +```sh +testimony review -session sessions/ -kind tests +``` + +For each draft it shows its id, the source finding with its type, severity and +clock, the title, the steps, the expected and observed behaviour, and the +participant's quote, then prompts `[a]ccept [e]dit [r]eject [s]kip [q]uit`. +Choosing `e` asks for each editable field in turn, showing the current value; a +blank answer keeps it, and steps are read one per line until a blank line. A pass +that changes nothing is recorded as an acceptance. + +Your decision is *appended* to `tests.jsonl` with today's date — the draft line +is never overwritten, so the record of what the machine proposed and what you +decided both survive, and an edit can never re-point a draft at a different +finding or session. + +To record a single decision without the interactive walk (handy in scripts): + +```sh +testimony review -session sessions/ -kind tests -test T-001 -decision accepted +testimony review -session sessions/ -kind tests -test T-003 -decision rejected +testimony review -session sessions/ -kind tests -test T-002 -decision edited -edit edit.json +``` + +`-edit FILE` (or `-edit -` for stdin) holds the replacement fields as a JSON +object — a subset of `title`, `steps`, `expected`, and `observed`, with at least +one member: + +```json +{"title":"Saving a display name gives no confirmation","steps":["Open #general.","Click Save."]} +``` + +Interactive review needs stdin to be a character device (an interactive terminal +is one); when it is a pipe or a redirected regular file (as in CI) it prints a +notice and exits without blocking. A later decision overrides an earlier one, and +both are kept. + +## 5. Render the test plan + +Render the accepted drafts as Markdown test-case blocks: + +```sh +testimony draft-tests -session sessions/ -render -out tests.md +``` + +Each block names its source finding and session, the decision and its date, the +numbered steps, the expected and observed behaviour, and the participant's quote +as the rationale — so a failing test leads back to the evidence that motivated +it. Only drafts whose effective status is `accepted` or `edited` are rendered, +with the latest edit applied; a proposal is not a test, and a rejected draft +stays in `tests.jsonl` for the record rather than for the plan. Rendering with no +accepted draft is refused, so `-out` cannot turn an existing plan into an empty +document. + +Where the rendered plan lives is your choice: Testimony never writes into the +application's repository, so the output defaults to stdout and `-out` puts it +wherever your docs-as-code test records are kept. The **record** stays with the +session, in `tests.jsonl`, linked to its finding. + +For the exact field rules, see the +[session directory reference](../reference/session-directory.md#testsjsonl); for +every flag, the [command-line reference](../reference/cli.md). diff --git a/docs/reference/session-directory.md b/docs/reference/session-directory.md index 69625b0..dfe2252 100644 --- a/docs/reference/session-directory.md +++ b/docs/reference/session-directory.md @@ -13,10 +13,11 @@ sessions// transcript.jsonl # time-aligned utterances (written by transcribe) timeline.jsonl # merged, session-relative timeline (written by merge) findings.jsonl # analysis findings + appended verdicts (written by analyze/review) + tests.jsonl # regression-test drafts + appended decisions (written by draft-tests/review) report.md # human-readable aligned record (written by report) ``` -All `.jsonl` files are JSON Lines: one JSON value per line, blank lines ignored. `timeline.jsonl`, `transcript.jsonl`, `interactions.jsonl`, and `findings.jsonl` each carry a 16 MiB total-size limit: a write that would push one over the cap is refused, and a load of one already over it is refused, so a session that reaches it needs a fresh session directory to continue in. `events.rrweb.jsonl` is archival and carries no such limit. +All `.jsonl` files are JSON Lines: one JSON value per line, blank lines ignored. `timeline.jsonl`, `transcript.jsonl`, `interactions.jsonl`, `findings.jsonl`, and `tests.jsonl` each carry a 16 MiB total-size limit: a write that would push one over the cap is refused, and a load of one already over it is refused, so a session that reaches it needs a fresh session directory to continue in. `events.rrweb.jsonl` is archival and carries no such limit. ## `manifest.json` @@ -152,6 +153,50 @@ Ingest validates every finding against the merged timeline and is the sole valid A finding's effective status starts `unverified`; verdict records apply in file order and the last one for that finding wins. +## `tests.jsonl` + +The regression-test drafting layer's output, written by `testimony draft-tests -ingest` and appended to by `testimony review -kind tests`. Two record kinds share the file, one per line: a **draft** line (no `kind` field) and a **decision** line (`kind: "decision"`). Decisions are appended, never written in place, so a draft's original state and the full decision history are retained. Blank lines are ignored. + +Ingest validates every draft against `findings.jsonl` and is the sole validation boundary — it never trusts the model. Unknown fields are rejected (the shape is closed), and `status` is forced to `"proposed"` on ingest regardless of the answer JSON, so a draft can never be born accepted. A draft may only ever reference a finding whose effective status is `confirmed`. + +**Draft record** + +| Field | Type | Required | Meaning | +|---|---|---|---| +| `id` | string | yes | `T-NNN`, zero-padded (`^T-\d{3}$`); unique within the file | +| `finding` | string | yes | an existing finding id in `findings.jsonl` whose effective status is `confirmed` and whose `mode` is not `B` | +| `session` | string | yes | equal to the manifest's `session`, so the link survives the line being copied out of the session directory | +| `title` | string | yes | one line naming the defect; non-empty, at most 200 characters | +| `steps` | array of strings | yes | the reproduction in time order; non-empty, at most 32 entries, every entry non-empty | +| `expected` | string | yes | the behaviour the participant expected; non-empty | +| `observed` | string | yes | what the system did; non-empty | +| `rationale_quote` | string | yes | **equal** to the source finding's `quote`, byte for byte — the drafting step carries evidence forward and never introduces any | +| `severity` | integer | yes | **equal** to the source finding's `severity`, so triage order survives the hand-off unaltered | +| `status` | string | no | always `"proposed"` on ingest, whatever the answer claims | + +```json +{"id":"T-001","finding":"F-001","session":"sample-session","title":"Saving gives no confirmation","steps":["Open #general in the settings prototype.","Change the display name to Alice.","Click the Save button ([data-testid=save-btn])."],"expected":"The save is confirmed on screen — a toast, or the button briefly disabled.","observed":"Nothing visibly changes, so there is no way to tell the save landed.","rationale_quote":"I clicked save and nothing happened","severity":3,"status":"proposed"} +``` + +**Decision record** + +| Field | Type | Required | Meaning | +|---|---|---|---| +| `kind` | string | yes | literal `"decision"` (the discriminator) | +| `test` | string | yes | an existing draft id in the file | +| `decision` | string | yes | one of `accepted`, `edited`, `rejected` | +| `at` | string | yes | decision date, ISO `YYYY-MM-DD` | +| `edit` | object | when `edited` | the replacement fields: a subset of `{title, steps, expected, observed}` with at least one member, each held to the draft's own rule for that field | + +```json +{"kind":"decision","test":"T-001","decision":"accepted","at":"2026-09-12"} +{"kind":"decision","test":"T-003","decision":"edited","at":"2026-09-12","edit":{"title":"Saving a display name gives no confirmation","steps":["Open #general.","Click Save."]}} +``` + +The `edit` object is a **closed** four-field subset: an `edit` naming `id`, `finding`, `session`, `severity`, or `rationale_quote` is a hard error, not a silently dropped key. A human edit therefore cannot re-point a draft at a different finding or session; the only way to change the link is to reject the draft and ingest a new one. + +A draft's effective status starts `proposed`; decision records apply in file order and the last one for that draft wins. An `edited` draft's rendered fields are the last `edited` decision's `edit` applied over the draft, computed when the plan is rendered — the draft line itself is never rewritten. `testimony draft-tests -render` renders the drafts whose effective status is `accepted` or `edited`. + ## `report.md` Human-readable Markdown rendered from the timeline and findings: diff --git a/examples/sample-session/tests.jsonl b/examples/sample-session/tests.jsonl new file mode 100644 index 0000000..491ee90 --- /dev/null +++ b/examples/sample-session/tests.jsonl @@ -0,0 +1,6 @@ +{"id":"T-001","finding":"F-001","session":"sample-session","title":"Saving gives no confirmation","steps":["Open #general in the settings prototype.","Change the display name to Alice.","Click the Save button ([data-testid=save-btn])."],"expected":"The save is confirmed on screen — a toast, or the button briefly disabled.","observed":"Nothing visibly changes, so there is no way to tell the save landed.","rationale_quote":"I clicked save and nothing happened","severity":3,"status":"proposed"} +{"id":"T-002","finding":"F-001","session":"sample-session","title":"A second save click looks identical to the first","steps":["Open #general in the settings prototype.","Change the display name to Alice.","Click the Save button ([data-testid=save-btn]).","Click the Save button a second time."],"expected":"The second click is either confirmed or visibly a no-op.","observed":"Both clicks look the same, so the only way to check is to click again.","rationale_quote":"I clicked save and nothing happened","severity":3,"status":"proposed"} +{"id":"T-003","finding":"F-001","session":"sample-session","title":"A saved display name survives a tab switch","steps":["Open #general in the settings prototype.","Change the display name to Alice.","Click the Save button ([data-testid=save-btn]).","Switch to Appearance and back to General."],"expected":"The display name still reads Alice.","observed":"The display name still reads Alice, but nothing said so at save time.","rationale_quote":"I clicked save and nothing happened","severity":3,"status":"proposed"} +{"kind":"decision","test":"T-001","decision":"accepted","at":"2026-09-12"} +{"kind":"decision","test":"T-002","decision":"edited","at":"2026-09-12","edit":{"title":"Saving a display name twice gives no confirmation either time","steps":["Open #general in the settings prototype.","Change the display name to Alice.","Click the Save button ([data-testid=save-btn]).","Click the Save button a second time."]}} +{"kind":"decision","test":"T-003","decision":"rejected","at":"2026-09-12"} diff --git a/internal/analyze/analyze_test.go b/internal/analyze/analyze_test.go index 239a176..3a3d2b9 100644 --- a/internal/analyze/analyze_test.go +++ b/internal/analyze/analyze_test.go @@ -14,48 +14,6 @@ import ( "github.com/REPPL/Testimony/internal/session" ) -// failAfterWriter is a findingsFile whose Write fails, recording whether the -// caller truncated back to 0 *after* the failed write — the rollback writeFindings -// must perform so a partial write never bricks findings.jsonl against its own -// recovery. The post-write ordering matters: writeFindings also truncates to 0 -// before writing, so only a truncate that follows the write attempt proves the -// rollback ran. -type failAfterWriter struct { - wrote bool - rolledBack bool -} - -func (w *failAfterWriter) Write(p []byte) (int, error) { - w.wrote = true - return 0, errors.New("no space left on device") -} -func (w *failAfterWriter) Truncate(size int64) error { - if w.wrote && size == 0 { - w.rolledBack = true - } - return nil -} -func (w *failAfterWriter) Seek(offset int64, whence int) (int64, error) { return 0, nil } - -// TestWriteFindingsRollsBackOnWriteError is the corrupt-on-failure regression for -// commitFindings. It runs f.Truncate(0) before writing, so a short write (ENOSPC) -// used to leave a truncated JSON fragment that not only broke every reader but -// blocked the recovery path — the next analyze -ingest's holdsVerdicts errors on -// the fragment before it can rewrite. writeFindings must roll the file back to -// empty (parseable, re-ingestable) on any write error. Pre-fix (neutralise the -// `f.Truncate(0)` in writeFindings' error path to demonstrate) no truncate follows -// the failed write and this fails. -func TestWriteFindingsRollsBackOnWriteError(t *testing.T) { - w := &failAfterWriter{} - err := writeFindings(w, []Finding{{ID: "F-001", T: 1, Type: "bug", Severity: 3, Quote: "x", Evidence: []string{"utt-001"}, Status: "unverified"}}) - if err == nil { - t.Fatal("writeFindings returned nil on a failing write; want the write error") - } - if !w.rolledBack { - t.Fatal("writeFindings did not truncate back to empty after the failed write; a partial line would survive and brick re-ingest") - } -} - // TestIngestRejectsQuoteThatSanitisesToEmpty is the verbatim-bypass regression. A // quote of only stripped characters (a lone U+202E) is raw-non-empty but SafeText // reduces it to "", and strings.Contains(text, "") is always true, so pre-fix the @@ -834,8 +792,8 @@ func TestIngestRejectsOversizedAnswer(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "refusing to ingest") { t.Fatalf("expected an over-size refusal, got %v", err) } - if r.read > maxAnswerBytes+64*1024 { - t.Fatalf("read %d bytes; the cap (%d) was not enforced", r.read, maxAnswerBytes) + if r.read > session.MaxAnswerBytes+64*1024 { + t.Fatalf("read %d bytes; the cap (%d) was not enforced", r.read, session.MaxAnswerBytes) } } @@ -986,7 +944,7 @@ func TestIngestOversizedFindingLeavesPriorFileIntact(t *testing.T) { // TestWriteFindingsRollsBackOnWriteError tests writeFindings), not through // Ingest, for a small and exactly predictable byte count: see // TestIngestRejectsOversizedFindingsTotal for the equivalent reachable -// through the public API at a fraction of maxAnswerBytes. Six findings, each +// through the public API at a fraction of session.MaxAnswerBytes. Six findings, each // ~3.4 MiB (under the 4 MiB line cap) via 64 citations of one long id, sum to // ~20 MiB (over the 16 MiB file cap). func TestOversizedFindingsRejectsOversizedTotal(t *testing.T) { @@ -1019,7 +977,7 @@ func TestOversizedFindingsRejectsOversizedTotal(t *testing.T) { } // TestIngestRejectsOversizedFindingsTotal is the total-size regression -// reached through the public API, at a fraction of maxAnswerBytes rather than +// reached through the public API, at a fraction of session.MaxAnswerBytes rather than // near it: writeFindings and oversizedFindings both encode with Go's default // HTML-escaping JSON encoder (deliberately not session.jsonlEncoder, so the // two agree with each other, but they disagree with the answer an operator's @@ -1028,7 +986,7 @@ func TestOversizedFindingsRejectsOversizedTotal(t *testing.T) { // \uXXXX escape in the line oversizedFindings measures and writeFindings // would persist. Five findings each quoting a ~600,000-byte run of '<' encode // to ~17 MiB once written, comfortably over the 16 MiB MaxJSONLBytes total, -// from a ~3 MiB answer — under a fifth of maxAnswerBytes, so the read-side +// from a ~3 MiB answer — under a fifth of session.MaxAnswerBytes, so the read-side // cap cannot be relied on to keep this path from ever executing. func TestIngestRejectsOversizedFindingsTotal(t *testing.T) { longText := strings.Repeat("<", 600000) @@ -1042,8 +1000,8 @@ func TestIngestRejectsOversizedFindingsTotal(t *testing.T) { i, longText)) } answer := `{"findings":[` + strings.Join(findings, ",") + `]}` - if len(answer) >= maxAnswerBytes { - t.Fatalf("test setup: answer is %d bytes, at or over maxAnswerBytes (%d); the read-side cap would refuse it before this test's own check runs", len(answer), maxAnswerBytes) + if len(answer) >= session.MaxAnswerBytes { + t.Fatalf("test setup: answer is %d bytes, at or over session.MaxAnswerBytes (%d); the read-side cap would refuse it before this test's own check runs", len(answer), session.MaxAnswerBytes) } _, err := Ingest(dir, strings.NewReader(answer)) diff --git a/internal/analyze/emit.go b/internal/analyze/emit.go index 199109a..b852dc3 100644 --- a/internal/analyze/emit.go +++ b/internal/analyze/emit.go @@ -38,7 +38,7 @@ func EmitRequest(dir string) (string, error) { if err != nil { return "", err } - entries, err := loadTimeline(dir) + entries, err := LoadTimeline(dir) if err != nil { return "", err } diff --git a/internal/analyze/ingest.go b/internal/analyze/ingest.go index 18ddf25..12c0679 100644 --- a/internal/analyze/ingest.go +++ b/internal/analyze/ingest.go @@ -7,21 +7,14 @@ import ( "errors" "fmt" "io" - "os" "path/filepath" "sort" - "syscall" "github.com/REPPL/Testimony/internal/session" "github.com/REPPL/Testimony/internal/timeline" ) -// maxAnswerBytes caps the untrusted answer read in Ingest, mirroring the -// bounded reads elsewhere (the demo server's 8 MiB body cap, the 4 MiB JSONL -// line cap). It is generous for a genuine multi-finding answer. -const maxAnswerBytes = 16 << 20 - -// loadTimeline reads the merged timeline, hinting to run merge first when it is +// LoadTimeline reads the merged timeline, hinting to run merge first when it is // missing (matching report), and orders it by time the same way report does: a // hand-edited or exchanged timeline.jsonl reaches this reader directly, and an // out-of-order one was presented to EmitRequest's "read the timeline in order" @@ -47,7 +40,12 @@ const maxAnswerBytes = 16 << 20 // \"\"" would misname the actual problem. Positions are 1-based entry // ordinals, which match file lines only when the file has no blank lines // (ReadJSONL skips those). -func loadTimeline(dir string) ([]timeline.Entry, error) { +// +// It is exported because internal/drafttests reads the same timeline under the +// same refusals: the event window a regression-test draft is reconstructed from +// must not be built over a file whose entry ids are ambiguous or whose src this +// package cannot place. +func LoadTimeline(dir string) ([]timeline.Entry, error) { entries, err := timeline.ReadEntries(filepath.Join(dir, session.TimelineFile)) if err != nil { return nil, fmt.Errorf("read timeline (run `testimony merge` first?): %w", err) @@ -78,7 +76,7 @@ func loadTimeline(dir string) ([]timeline.Entry, error) { // nothing written on any failure). To protect the retained precision record it // refuses to overwrite a findings.jsonl that already holds verdict records. func Ingest(dir string, r io.Reader) ([]Finding, error) { - entries, err := loadTimeline(dir) + entries, err := LoadTimeline(dir) if err != nil { return nil, err } @@ -86,14 +84,15 @@ func Ingest(dir string, r io.Reader) ([]Finding, error) { // The answer is untrusted LLM output (this is the validation boundary) and // -ingest reads it from stdin/a file, so cap the read: a multi-gigabyte - // answer must not OOM the process before validation runs. maxAnswerBytes is - // generous for a real answer; anything larger is rejected, not buffered. - data, err := io.ReadAll(io.LimitReader(r, maxAnswerBytes+1)) + // answer must not OOM the process before validation runs. + // session.MaxAnswerBytes is generous for a real answer; anything larger is + // rejected, not buffered. + data, err := io.ReadAll(io.LimitReader(r, session.MaxAnswerBytes+1)) if err != nil { return nil, err } - if len(data) > maxAnswerBytes { - return nil, fmt.Errorf("answer exceeds %d bytes: refusing to ingest", maxAnswerBytes) + if len(data) > session.MaxAnswerBytes { + return nil, fmt.Errorf("answer exceeds %d bytes: refusing to ingest", session.MaxAnswerBytes) } raws, rubric, err := parseContainer(data) if err != nil { @@ -151,93 +150,43 @@ func Ingest(dir string, r io.Reader) ([]Finding, error) { } // commitFindings runs the verdict guard and the truncating write as one locked -// step. Probing with holdsVerdicts and then calling session.WriteJSONL as two -// separate opens left a TOCTOU window: a concurrent `testimony review` commits a -// verdict (under its own lock, see review.AppendVerdict) between the probe and -// the O_TRUNC open, and the rewrite destroys it — precisely the human-decision -// record the guard exists to protect. Holding one exclusive advisory lock across -// probe, truncate, and write forecloses the interleaving: AppendVerdict blocks -// until the commit completes, so a verdict is either visible to the probe (and -// the re-ingest refused) or appended after the new findings. The findings were -// already held to both session.MaxJSONLLine and session.MaxJSONLBytes by -// oversizedFindings, so writing through the locked descriptor keeps the -// read-side invariants WriteJSONL's two callers get. +// step, through session.CommitRecords: the probe, the truncate, and the write +// share one exclusive advisory lock, so a concurrent `testimony review` +// committing a verdict (under session.AppendRecord's lock, see +// review.AppendVerdict) cannot slip between the probe and the O_TRUNC open and +// have its record — precisely the human-decision record the guard exists to +// protect — destroyed by the rewrite. The findings were already held to both +// session.MaxJSONLLine and session.MaxJSONLBytes by oversizedFindings, which is +// the pre-flight CommitRecords leaves to its callers because only they can name a +// record by its own id or its position in an answer. +// +// Each finding is encoded with json.Marshal, the same encoder (HTML escaping on, +// Go's default) oversizedFindings measures with, so the bytes written are exactly +// the bytes that passed the size check. func commitFindings(dir string, findings []Finding) error { path := filepath.Join(dir, session.FindingsFile) - f, err := session.OpenFileNoFollow(path, os.O_CREATE|os.O_RDWR, 0o644) - if err != nil { - return err - } - if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { - f.Close() - return err - } - held, err := holdsVerdicts(f, path) - if err != nil { - f.Close() - return err - } - if held { - f.Close() - return fmt.Errorf("refusing to overwrite %s: it already holds verdict records (the retained precision record)", session.FindingsFile) - } - if err := writeFindings(f, findings); err != nil { - f.Close() - return err - } - // Close releases the lock with the descriptor. - if err := f.Close(); err != nil { - return fmt.Errorf("write findings: %w", err) - } - return nil -} - -// findingsFile is the subset of *os.File writeFindings needs; a fake satisfies it -// in tests to exercise the truncate-then-write rollback. -type findingsFile interface { - io.Writer - Truncate(size int64) error - Seek(offset int64, whence int) (int64, error) -} - -// writeFindings replaces f's contents with the findings, one JSON object per line, -// rolling the file back to empty if the write only partly lands. -// -// The whole set is encoded into one buffer before f is truncated, so the truncate -// and the write are a single Write of pre-built bytes rather than a streamed series -// of encodes that a mid-way I/O error (ENOSPC) could leave half-flushed. That -// matters because commitFindings ran f.Truncate(0) first: without the rollback a -// short write left findings.jsonl holding a truncated, newline-less JSON fragment, -// which not only breaks every reader but blocks the tool's own recovery — the next -// analyze -ingest calls holdsVerdicts, which json.Unmarshals every line and errors -// on the fragment before it can conclude "no verdicts" and rewrite. On any write -// error the file is therefore truncated back to empty: an empty findings.jsonl is -// parseable (zero findings, zero verdicts) and re-ingestable, so the failure state -// no longer forecloses its own repair. This is the same partial-write rollback -// review.writeVerdict and demo.appendRecords already apply; commitFindings was the -// one writer without it. The set is bounded by oversizedFindings before this runs, -// so buffering it whole holds one bounded answer, not an unbounded stream. -func writeFindings(f findingsFile, findings []Finding) error { - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - for _, v := range findings { - if err := enc.Encode(v); err != nil { + records := make([][]byte, 0, len(findings)) + for _, f := range findings { + b, err := json.Marshal(f) + if err != nil { return fmt.Errorf("write findings: %w", err) } + records = append(records, b) } - if err := f.Truncate(0); err != nil { - return fmt.Errorf("write findings: %w", err) - } - if _, err := f.Seek(0, io.SeekStart); err != nil { - return fmt.Errorf("write findings: %w", err) - } - if _, err := f.Write(buf.Bytes()); err != nil { - // Best-effort roll back to an empty (parseable, re-ingestable) file, then - // surface the original error. - f.Truncate(0) - return fmt.Errorf("write findings: %w", err) - } - return nil + return session.CommitRecords(session.Commit{ + Path: path, + Records: records, + Guard: func(current io.Reader) error { + held, err := holdsVerdicts(current, path) + if err != nil { + return err + } + if held { + return fmt.Errorf("refusing to overwrite %s: it already holds verdict records (the retained precision record)", session.FindingsFile) + } + return nil + }, + }) } // oversizedFindings reports any finding whose findings.jsonl line — its JSON @@ -257,8 +206,8 @@ func writeFindings(f findingsFile, findings []Finding) error { // checks here run before any write and join the transactional error set, so // an oversized answer leaves the previous findings.jsonl untouched rather // than bricking it — the write-side pre-flight WriteJSONL's two callers get, -// which findings.jsonl otherwise lacks because commitFindings writes it -// through its own locked descriptor rather than WriteJSONL. Labels come from +// which findings.jsonl otherwise lacks because it is committed through +// session.CommitRecords rather than WriteJSONL. Labels come from // each finding's answer position for the same reason validate's do; a line // already flagged as over-long is excluded from the total so one oversized // finding cannot also trigger a redundant total-size error. @@ -374,9 +323,9 @@ func decodeFinding(raw json.RawMessage) (Finding, error) { // holdsVerdicts reports whether the findings.jsonl open on f already contains // any verdict record. It reads through the caller's descriptor — opened under -// the no-follow guard and exclusively locked by commitFindings — rather than -// opening the path itself, so the probe and the write it gates observe the same -// locked file. It scans for raw kind:"verdict" lines rather than reusing +// the no-follow guard and exclusively locked by session.CommitRecords — rather +// than opening the path itself, so the probe and the write it gates observe the +// same locked file. It scans for raw kind:"verdict" lines rather than reusing // analyze.Load, whose verdict slice is filtered to the closed enum // (confirmed|rejected|duplicate): a hand-edited or shared file whose only // verdict lines carry a foreign or typo'd value would otherwise slip past the diff --git a/internal/drafttests/drafttests.go b/internal/drafttests/drafttests.go new file mode 100644 index 0000000..d03fa67 --- /dev/null +++ b/internal/drafttests/drafttests.go @@ -0,0 +1,440 @@ +// Package drafttests implements the regression-test drafting layer: it emits a +// self-contained, host-delegated drafting request (a versioned rubric, the +// session context, and each confirmed finding with its event window) and is the +// sole validation boundary for the model's answer, writing validated drafts to +// tests.jsonl. The CLI never calls a model, holds no keys, and adds no network +// dependency, exactly as for internal/analyze. +// +// The step sits downstream of verification: a draft may only ever reference a +// finding a human already confirmed, so a session with none is staged loudly +// rather than drafted from. A draft is born a proposal — ingest forces every +// draft to status:"proposed" regardless of what the answer JSON claims — and the +// human's accept / edit / reject decision is appended as a separate, +// non-destructive record, so the drafted proposal and the decision both survive +// and the draft's link to its source finding and session is unreachable by any +// later write. +package drafttests + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "math" + "path/filepath" + "reflect" + "regexp" + "sort" + "strings" + + "github.com/REPPL/Testimony/internal/analyze" + "github.com/REPPL/Testimony/internal/session" +) + +// RubricVersion pins the drafting scheme so drafts are comparable across +// sessions and future rubric revisions are explicit. +const RubricVersion = "testimony-testdraft/v1" + +// maxTitle and maxSteps bound the two free-text fields that carry no other +// limit. A genuine test case names itself in one line and reproduces in a +// handful of actions; the bounds stop a hostile answer from smuggling a draft +// that is individually well-formed yet serialises past the JSONL line limit its +// readers scan to, and keep the rendered test plan readable. +const ( + maxTitle = 200 + maxSteps = 32 +) + +// maxClockSeconds bounds a value clock will format, mirroring report and review: +// a real session stamp is minutes to hours, and 1e9 seconds (~31 years) stays +// well inside int64 so the float64→int conversion in clock can never go out of +// range on an attacker-authored findings.jsonl time. +const maxClockSeconds = 1e9 + +// Draft is one proposed regression test — one line of tests.jsonl. Draft lines +// carry no "kind" field; the schema is closed (ingest decodes with +// DisallowUnknownFields). +type Draft struct { + ID string `json:"id"` + Finding string `json:"finding"` + Session string `json:"session"` + Title string `json:"title"` + Steps []string `json:"steps"` + Expected string `json:"expected"` + Observed string `json:"observed"` + RationaleQuote string `json:"rationale_quote"` + Severity int `json:"severity"` + Status string `json:"status"` +} + +// Decision is an appended, non-destructive human decision on a draft. It is +// discriminated by kind:"decision"; the last decision for a draft wins. +type Decision struct { + Kind string `json:"kind"` // literal "decision" + Test string `json:"test"` + Decision string `json:"decision"` // accepted | edited | rejected + At string `json:"at"` // YYYY-MM-DD + Edit *Edit `json:"edit,omitempty"` +} + +// Edit is the closed subset of a draft's fields a human decision may replace. +// It deliberately cannot reach id, finding, session, severity, or +// rationale_quote: those are the draft's link to the evidence it came from, and +// the only way to change the link is to reject the draft and ingest a new one. +// Each field is a pointer so an absent member stays distinguishable from one +// present but empty — an empty replacement is refused rather than silently +// treated as "keep". +type Edit struct { + Title *string `json:"title,omitempty"` + Steps *[]string `json:"steps,omitempty"` + Expected *string `json:"expected,omitempty"` + Observed *string `json:"observed,omitempty"` +} + +// Status is a draft's effective status for display. +type Status struct { + Value string // proposed | accepted | edited | rejected + At string // decision date, when a decision exists + Edit *Edit // the winning decision's replacement fields, when Value == "edited" +} + +var ( + draftIDRe = regexp.MustCompile(`^T-\d{3}$`) + decisionSet = map[string]bool{"accepted": true, "edited": true, "rejected": true} + knownRubrics = map[string]bool{RubricVersion: true} +) + +// IsDraftID reports whether s is a well-formed draft id (T-NNN). +func IsDraftID(s string) bool { return draftIDRe.MatchString(s) } + +// ParseDecisionFlag validates a -decision flag value against the closed enum. +func ParseDecisionFlag(s string) (string, error) { + if !decisionSet[s] { + return "", fmt.Errorf("invalid decision %q (want accepted|edited|rejected)", s) + } + return s, nil +} + +// Load reads tests.jsonl from dir, splitting draft lines from appended decision +// lines. A missing file returns an error satisfying fs.ErrNotExist so callers +// can render an absence notice. +func Load(dir string) ([]Draft, []Decision, error) { + path := filepath.Join(dir, session.TestsFile) + // Route through the read-side no-follow guard, not plain os.Open: tests.jsonl + // in an exchanged (attacker-authored) session may be a symlink or a FIFO, and + // a FIFO would block this open in open(2) for ever. + f, err := session.OpenFileNoFollowRead(path) + if err != nil { + return nil, nil, err + } + defer f.Close() + return ParseRecords(f, path) +} + +// ParseRecords splits a tests.jsonl stream into draft and decision records, +// mirroring analyze.ParseRecords rule for rule: blank lines are skipped, a +// decision carrying an out-of-enum value is ignored rather than applied, and the +// file is held to both the per-line and the total-size JSONL caps. name labels +// errors. Load is ParseRecords over the on-disk file opened through the +// no-follow guard; AppendDecision reuses it to re-read the current drafts +// through its own already-locked descriptor, so the re-check and the append +// observe the same file under one lock. +func ParseRecords(r io.Reader, name string) ([]Draft, []Decision, error) { + var drafts []Draft + var decisions []Decision + // Draft ids must be unique across the file, for the reason finding ids must + // be: EffectiveStatus and draftByID both key on the id, so two drafts sharing + // one would collapse onto a single status entry — one decision painting both + // drafts — and the walk would only ever reach the first. Ids are compared in + // their session.SafeText form, the form every surface renders them in, so two + // raw ids distinct only by stripped bytes count as the collision they display + // as. + seenID := map[string]bool{} + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), session.MaxJSONLLine) + line := 0 + var total int64 + for sc.Scan() { + line++ + raw := sc.Bytes() + // A per-line cap alone leaves the file's total size unbounded: a + // hand-edited or exchanged tests.jsonl built from many small, + // individually-legal lines would otherwise drive this loop's per-line + // allocation well past the bytes on disk. + total += int64(len(raw)) + 1 + if total > session.MaxJSONLBytes { + return nil, nil, fmt.Errorf("%s: exceeds %d bytes across %d lines; refusing to read", name, session.MaxJSONLBytes, line) + } + if len(bytes.TrimSpace(raw)) == 0 { + continue + } + var probe struct { + Kind string `json:"kind"` + Severity *int `json:"severity"` + } + if err := json.Unmarshal(raw, &probe); err != nil { + return nil, nil, fmt.Errorf("%s:%d: %w", name, line, err) + } + if probe.Kind == "decision" { + var d Decision + if err := json.Unmarshal(raw, &d); err != nil { + return nil, nil, fmt.Errorf("%s:%d: %w", name, line, err) + } + // The decision enum is closed (accepted|edited|rejected). A decision + // carrying any other value — a typo, an empty string, or a foreign value + // from a shared or hand-edited session — is not representable, so it is + // ignored rather than applied. The draft then keeps "proposed" and still + // appears in the review queue, instead of landing in a status group + // neither the walk nor the render shows and silently vanishing from both. + if !decisionSet[d.Decision] { + continue + } + decisions = append(decisions, d) + continue + } + // A line that is JSON null (or {}) decodes cleanly into a value-typed Draft + // as its zero value, so a hand-edited or exchanged tests.jsonl carrying one + // would silently inject a phantom draft — id "", severity 0 — into the + // review queue and the rendered plan. probe.Severity is a pointer for + // exactly this reason (mirroring rawDraft and analyze's rawFinding): every + // draft this tool ever writes restates its finding's severity, so its + // absence means the line was never a draft at all. + if probe.Severity == nil { + return nil, nil, fmt.Errorf("%s:%d: not a test draft or decision record (missing severity)", name, line) + } + var d Draft + if err := json.Unmarshal(raw, &d); err != nil { + return nil, nil, fmt.Errorf("%s:%d: %w", name, line, err) + } + id := session.SafeText(d.ID) + // An empty (or whitespace-only) id is refused on its own terms rather than + // folded into the duplicate check below: a draft's id is never optional, and + // reporting the second one as `duplicate test draft id ""` would misname + // what is actually wrong with the first. + if strings.TrimSpace(id) == "" { + return nil, nil, fmt.Errorf("%s:%d: test draft has no id; every draft must have a unique id", name, line) + } + if seenID[id] { + return nil, nil, fmt.Errorf("%s:%d: duplicate test draft id %q; each draft must have a unique id", name, line, d.ID) + } + seenID[id] = true + drafts = append(drafts, d) + } + if err := sc.Err(); err != nil { + return nil, nil, fmt.Errorf("%s: %w", name, err) + } + return drafts, decisions, nil +} + +// SameIdentity reports whether a and b are the same draft — equal in every field +// a human decision is recorded against. Status is excluded: it is the one field +// a decision is meant to change, and Ingest launders it to "proposed" on every +// written draft regardless. AppendDecision uses it under the append lock to +// confirm a decision still targets the draft the operator was shown, rather than +// a different draft a concurrent re-ingest slid under the same id. +func SameIdentity(a, b Draft) bool { + a.Status, b.Status = "", "" + return reflect.DeepEqual(a, b) +} + +// EffectiveStatus maps each draft id to its effective status: every draft starts +// "proposed"; decision records are applied in file order and the last one for an +// id wins. A decision naming an unknown draft is ignored for display. This +// single helper is used by the review walk (to pick the work queue) and by the +// render (to pick which drafts belong in the plan, and which edit to apply). +func EffectiveStatus(drafts []Draft, decisions []Decision) map[string]Status { + m := make(map[string]Status, len(drafts)) + for _, d := range drafts { + m[d.ID] = Status{Value: "proposed"} + } + for _, dec := range decisions { + if _, ok := m[dec.Test]; !ok { + continue // a decision referencing an unknown draft is ignored for display + } + st := Status{Value: dec.Decision, At: dec.At} + if dec.Decision == "edited" { + st.Edit = dec.Edit + } + m[dec.Test] = st + } + return m +} + +// Apply returns d with the edit's present members substituted, leaving every +// other field — and d itself — untouched. It is how an "edited" draft is +// rendered: the decision is a separate appended record, so the substitution is +// computed at render time and the draft line on disk is never rewritten. +func (e *Edit) Apply(d Draft) Draft { + if e == nil { + return d + } + if e.Title != nil { + d.Title = *e.Title + } + if e.Steps != nil { + d.Steps = append([]string(nil), (*e.Steps)...) + } + if e.Expected != nil { + d.Expected = *e.Expected + } + if e.Observed != nil { + d.Observed = *e.Observed + } + return d +} + +// empty reports whether the edit names no member at all. An "edited" decision +// with an empty edit is not representable — it records a change that did not +// happen — so both the interactive and the non-interactive paths refuse it. +func (e *Edit) empty() bool { + return e == nil || (e.Title == nil && e.Steps == nil && e.Expected == nil && e.Observed == nil) +} + +// draftByID returns a pointer to the draft with the given id, or nil. Ids are +// compared in their session.SafeText form, matching ParseRecords' load-time +// uniqueness check and review.findByID: a draft's id renders through SafeText +// everywhere it is shown, so an operator matching it via -test only ever has the +// rendered form to type. The returned pointer is into a copy, safe to retain. +func draftByID(drafts []Draft, id string) *Draft { + want := session.SafeText(id) + for i := range drafts { + if session.SafeText(drafts[i].ID) == want { + d := drafts[i] + return &d + } + } + return nil +} + +// eligible returns the findings a draft may reference: those whose effective +// status is "confirmed" and whose mode is not "B", in id order. +// +// Effective status is not recomputed here — analyze.EffectiveStatus is the +// single helper review and report already use — so "the last verdict wins" is +// true for free: a finding confirmed and later rejected is not eligible, and one +// rejected and later confirmed is. A duplicate is never eligible even when its +// target is confirmed: the canonical finding is the one that carries the +// evidence. Mode B (reference capture) is excluded because a design preference +// has nothing to regress against; nothing produces Mode B today, so the +// exclusion is a guard rather than a live filter. type is deliberately not +// filtered: the acceptance criterion names a confirmed finding without +// qualification, the request carries each finding's type so the model can +// calibrate, and a draft with nothing to regress against is what the reject verb +// is for. +func eligible(findings []analyze.Finding, verdicts []analyze.Verdict) []analyze.Finding { + eff := analyze.EffectiveStatus(findings, verdicts) + var out []analyze.Finding + for _, f := range findings { + if eff[f.ID].Value == "confirmed" && f.Mode != "B" { + out = append(out, f) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out +} + +// findingCounts renders the by-status tally the no-confirmed-findings refusal +// names, so an operator staged empty can see why. +func findingCounts(findings []analyze.Finding, verdicts []analyze.Verdict) string { + eff := analyze.EffectiveStatus(findings, verdicts) + n := map[string]int{} + for _, f := range findings { + n[eff[f.ID].Value]++ + } + return fmt.Sprintf("%d findings: %d confirmed, %d unverified, %d duplicate, %d rejected", + len(findings), n["confirmed"], n["unverified"], n["duplicate"], n["rejected"]) +} + +// draftCounts is findingCounts' sibling for the no-accepted-drafts refusal. +func draftCounts(drafts []Draft, decisions []Decision) string { + eff := EffectiveStatus(drafts, decisions) + n := map[string]int{} + for _, d := range drafts { + n[eff[d.ID].Value]++ + } + return fmt.Sprintf("%d drafts: %d accepted, %d edited, %d proposed, %d rejected", + len(drafts), n["accepted"], n["edited"], n["proposed"], n["rejected"]) +} + +// clock renders a session-relative time as [MM:SS]. Negative times are +// legitimate — an external recording whose creation_time predates the manifest +// t0 yields a negative offset, and analyze.indexTimeline deliberately admits +// findings anchored there — so the sign is rendered rather than clamped away. +// This mirrors report.clock and review.clock; see the note in drafttests_test.go +// about the duplication. +func clock(sec float64) string { + // Defend the float64→int conversion below against a non-finite or + // astronomically large sec from a hand-authored findings.jsonl: int(sec+0.5) + // would be an out-of-range conversion the Go spec leaves + // implementation-defined, printing a nonsensical stamp on the surface where + // the operator decides a draft's fate. + if math.IsNaN(sec) || math.Abs(sec) > maxClockSeconds { + return "--:--" + } + neg := sec < 0 + if neg { + sec = -sec + } + s := int(sec + 0.5) + sign := "" + // The sign is taken from the rounded value, not the raw one, so a time a + // fraction of a second before t0 prints as 00:00 rather than "-00:00". + if neg && s > 0 { + sign = "-" + } + return fmt.Sprintf("%s%02d:%02d", sign, s/60, s%60) +} + +// ErrNoConfirmedFindings marks the refusal that stages an empty drafting step +// loudly: a session whose findings are all unverified, rejected, or duplicates +// has nothing a draft could legally reference, so emit and ingest both refuse, +// name the finding count by status, and write nothing. It is a sentinel so the +// CLI can tell a well-formed invocation whose work cannot be done from a genuine +// failure. +var ErrNoConfirmedFindings = errors.New("no confirmed findings to draft tests from") + +// ErrNoAcceptedDrafts is its render-side twin: a plan with no accepted or edited +// draft in it is an empty document, and writing one over an existing test plan +// (with -out) would erase it — the same reasoning behind the empty-answer +// refusal. +var ErrNoAcceptedDrafts = errors.New("no accepted test drafts to render") + +func noConfirmedFindings(dir string, findings []analyze.Finding, verdicts []analyze.Verdict) error { + return fmt.Errorf("%w (%s); confirm one with `testimony review -session %s` first", + ErrNoConfirmedFindings, findingCounts(findings, verdicts), dir) +} + +func noAcceptedDrafts(dir string, drafts []Draft, decisions []Decision) error { + return fmt.Errorf("%w (%s); accept one with `testimony review -session %s -kind tests` first", + ErrNoAcceptedDrafts, draftCounts(drafts, decisions), dir) +} + +// loadFindings reads the session's findings, hinting to run `analyze -ingest` +// first when there is none — every mode of this package needs them, because a +// draft is only ever a proposal about a finding a human confirmed. +func loadFindings(dir string) ([]analyze.Finding, []analyze.Verdict, error) { + findings, verdicts, err := analyze.Load(dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil, fmt.Errorf("no %s (run `testimony analyze -ingest` first)", session.FindingsFile) + } + return nil, nil, err + } + return findings, verdicts, nil +} + +// loadDrafts reads the session's drafts and decisions, hinting to run +// `draft-tests -ingest` first when there is no tests.jsonl yet. +func loadDrafts(dir string) ([]Draft, []Decision, error) { + drafts, decisions, err := Load(dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil, fmt.Errorf("no %s (run `testimony draft-tests -ingest` first)", session.TestsFile) + } + return nil, nil, err + } + return drafts, decisions, nil +} diff --git a/internal/drafttests/drafttests_test.go b/internal/drafttests/drafttests_test.go new file mode 100644 index 0000000..de8d472 --- /dev/null +++ b/internal/drafttests/drafttests_test.go @@ -0,0 +1,398 @@ +package drafttests + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/REPPL/Testimony/internal/analyze" + "github.com/REPPL/Testimony/internal/session" + "github.com/REPPL/Testimony/internal/timeline" +) + +// fixture reads one testdata file. The fixtures are a session mirroring the +// bundled example: F-001 confirmed (the one eligible finding), F-002 unverified, +// F-003 rejected, F-004 confirmed but mode B, F-005 a duplicate of F-001. +func fixture(t *testing.T, name string) []byte { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatalf("read fixture %s: %v", name, err) + } + return b +} + +// writeSession lays a session directory from the testdata fixtures. Any of the +// three artefacts can be replaced by passing its name and contents; empty +// contents omits the artefact, so the missing-file hints can be exercised. +func writeSession(t *testing.T, overrides ...string) string { + t.Helper() + if len(overrides)%2 != 0 { + t.Fatalf("writeSession: overrides must be name/contents pairs") + } + files := map[string][]byte{ + session.ManifestFile: fixture(t, "manifest.json"), + session.FindingsFile: fixture(t, "findings.jsonl"), + session.TimelineFile: fixture(t, "timeline.jsonl"), + } + for i := 0; i < len(overrides); i += 2 { + if overrides[i+1] == "" { + delete(files, overrides[i]) + continue + } + files[overrides[i]] = []byte(overrides[i+1]) + } + dir := t.TempDir() + for name, body := range files { + if body == nil { + continue + } + if err := os.WriteFile(filepath.Join(dir, name), body, 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + return dir +} + +func readEntries(t *testing.T) []timeline.Entry { + t.Helper() + dir := writeSession(t) + entries, err := analyze.LoadTimeline(dir) + if err != nil { + t.Fatalf("LoadTimeline: %v", err) + } + return entries +} + +func loadFixtureFindings(t *testing.T) ([]analyze.Finding, []analyze.Verdict) { + t.Helper() + findings, verdicts, err := analyze.Load(writeSession(t)) + if err != nil { + t.Fatalf("analyze.Load: %v", err) + } + return findings, verdicts +} + +func findingID(t *testing.T, findings []analyze.Finding, id string) analyze.Finding { + t.Helper() + for _, f := range findings { + if f.ID == id { + return f + } + } + t.Fatalf("fixture has no finding %s", id) + return analyze.Finding{} +} + +func entryIDs(entries []timeline.Entry) []string { + ids := make([]string, 0, len(entries)) + for _, e := range entries { + ids = append(ids, e.ID) + } + return ids +} + +func draft(id string) Draft { + return Draft{ + ID: id, Finding: "F-001", Session: "fixture-session", + Title: "Saving gives no confirmation", + Steps: []string{"Open #general.", "Click Save."}, + Expected: "The save is confirmed.", + Observed: "Nothing happens.", + RationaleQuote: "I clicked save and nothing happened", + Severity: 3, Status: "proposed", + } +} + +func ptr(s string) *string { return &s } + +// --- eligibility ----------------------------------------------------------- + +// TestEligibilityHonoursLastVerdict is AC2's load-side half. Effective status +// comes from analyze.EffectiveStatus, so a later verdict overriding an earlier +// one is honoured for free: confirmed-then-rejected is not eligible, and +// rejected-then-confirmed is. A duplicate of a confirmed finding is never +// eligible — the canonical finding carries the evidence — and a mode B +// (reference-capture) finding is excluded as a guard. +func TestEligibilityHonoursLastVerdict(t *testing.T) { + findings, verdicts := loadFixtureFindings(t) + got := map[string]bool{} + for _, f := range eligible(findings, verdicts) { + got[f.ID] = true + } + if !got["F-001"] { + t.Fatalf("F-001 (confirmed, mode A) is not eligible: %v", got) + } + for _, id := range []string{"F-002", "F-003", "F-005"} { + if got[id] { + t.Fatalf("%s is eligible but is not confirmed", id) + } + } + if got["F-004"] { + t.Fatal("F-004 is eligible but is a mode B (reference-capture) finding") + } + + // confirmed-then-rejected drops out; rejected-then-confirmed comes back in. + later := append(verdicts, + analyze.Verdict{Kind: "verdict", Finding: "F-001", Verdict: "rejected", At: "2026-09-13"}, + analyze.Verdict{Kind: "verdict", Finding: "F-003", Verdict: "confirmed", At: "2026-09-13"}, + ) + got = map[string]bool{} + for _, f := range eligible(findings, later) { + got[f.ID] = true + } + if got["F-001"] { + t.Fatal("F-001 stayed eligible after a later rejected verdict") + } + if !got["F-003"] { + t.Fatal("F-003 did not become eligible after a later confirmed verdict") + } +} + +func TestFindingCountsNamesEveryStatus(t *testing.T) { + findings, verdicts := loadFixtureFindings(t) + want := "5 findings: 2 confirmed, 1 unverified, 1 duplicate, 1 rejected" + if got := findingCounts(findings, verdicts); got != want { + t.Fatalf("findingCounts = %q, want %q", got, want) + } +} + +// --- the event window ------------------------------------------------------ + +// TestWindowSpansEvidenceWidenedByWindow is the worked example from the spec: +// F-001 cites utt-004 (22–28 s), ev-003 (19.2 s) and ev-004 (24.1 s), so the +// 10-second window is [9.2, 38.0] and holds the whole repro — click the +// display-name field, type Alice, click save, click save again — plus the +// utterances that frame it. The upper bound comes from the utterance's end +// (SpeechEnd), not its start. +func TestWindowSpansEvidenceWidenedByWindow(t *testing.T) { + entries := readEntries(t) + findings, _ := loadFixtureFindings(t) + f := findingID(t, findings, "F-001") + + got := entryIDs(Window(entries, f, 10)) + want := []string{"ev-001", "ev-002", "utt-003", "ev-003", "utt-004", "ev-004", "utt-005", "ev-005", "utt-006"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("Window(10) = %v, want %v", got, want) + } + // utt-006 sits exactly on the upper bound (38.0): the bounds are inclusive. + if got[len(got)-1] != "utt-006" { + t.Fatalf("the entry on the upper bound was dropped: %v", got) + } + // utt-002 starts at 8, before the lower bound of 9.2. + for _, id := range got { + if id == "utt-002" { + t.Fatalf("an entry before the lower bound was included: %v", got) + } + } +} + +// TestWindowNarrowsAndWidens shows why the default is 10 seconds rather than +// report's 2.5: at 2.5 the window is [16.7, 30.5] and excludes utt-003 — the one +// utterance in the session that states the expected behaviour — so a repro drafted +// from it would have nothing to ground "expected" in. A negative window is +// legitimate and narrows further, to nothing. +func TestWindowNarrowsAndWidens(t *testing.T) { + entries := readEntries(t) + findings, _ := loadFixtureFindings(t) + f := findingID(t, findings, "F-001") + + narrow := entryIDs(Window(entries, f, 2.5)) + want := []string{"ev-003", "utt-004", "ev-004"} + if strings.Join(narrow, ",") != strings.Join(want, ",") { + t.Fatalf("Window(2.5) = %v, want %v", narrow, want) + } + wide := Window(entries, f, 60) + if len(wide) != len(entries) { + t.Fatalf("Window(60) returned %d of %d entries; a wide window holds the whole timeline", len(wide), len(entries)) + } + if got := Window(entries, f, -10); len(got) != 0 { + t.Fatalf("Window(-10) = %v, want an empty (over-narrowed) window", entryIDs(got)) + } +} + +// TestWindowUsesEventOnlyEvidence covers a finding whose evidence resolves only +// to events: the span is still taken from the cited entries, not from f.T. +func TestWindowUsesEventOnlyEvidence(t *testing.T) { + entries := readEntries(t) + f := analyze.Finding{ID: "F-900", T: 0, Evidence: []string{"ev-003"}} + got := entryIDs(Window(entries, f, 1)) + want := []string{"ev-003"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("Window over event-only evidence = %v, want %v", got, want) + } +} + +// TestWindowFallsBackToFindingTime covers a finding whose evidence resolves to no +// entry at all — impossible after analyze -ingest, reachable via a hand-edited +// findings.jsonl. The window falls back to f.T ± window so the finding still +// arrives with context rather than with nothing. +func TestWindowFallsBackToFindingTime(t *testing.T) { + entries := readEntries(t) + f := analyze.Finding{ID: "F-900", T: 22, Evidence: []string{"utt-999"}} + got := entryIDs(Window(entries, f, 2)) + want := []string{"utt-004"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("fallback window = %v, want %v", got, want) + } +} + +// TestWindowOrdersByTime pins the ordering promise: the window is presented to +// the model as the sequence to reconstruct steps from, so an out-of-order +// timeline.jsonl (which reaches these readers directly when a session is +// hand-edited or exchanged) must not hand it a repro in the wrong order. +func TestWindowOrdersByTime(t *testing.T) { + entries := []timeline.Entry{ + {T: 24.1, Src: "event", ID: "ev-004", Payload: map[string]any{"kind": "click"}}, + {T: 19.2, Src: "event", ID: "ev-003", Payload: map[string]any{"kind": "click"}}, + {T: 22, Src: "speech", ID: "utt-004", Payload: map[string]any{"t1": 28.0, "text": "x"}}, + } + f := analyze.Finding{ID: "F-001", T: 22, Evidence: []string{"utt-004"}} + got := entryIDs(Window(entries, f, 10)) + want := []string{"ev-003", "utt-004", "ev-004"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("Window over an out-of-order timeline = %v, want %v", got, want) + } +} + +// TestWindowMatchesEvidenceInSafeTextForm covers the form the answering agent is +// actually shown: EmitRequest routes each marshalled entry through SafeText, so +// an evidence id carrying a stripped byte must resolve here exactly as it does in +// analyze's own validation. +func TestWindowMatchesEvidenceInSafeTextForm(t *testing.T) { + entries := readEntries(t) + f := analyze.Finding{ID: "F-900", T: 0, Evidence: []string{"ev-​003"}} // zero-width space + if got := entryIDs(Window(entries, f, 0)); strings.Join(got, ",") != "ev-003" { + t.Fatalf("Window did not match the evidence id in SafeText form: %v", got) + } +} + +// --- records and effective status ------------------------------------------ + +func TestParseRecordsSplitsDraftsAndDecisions(t *testing.T) { + in := `{"id":"T-001","finding":"F-001","session":"s","title":"t","steps":["a"],"expected":"e","observed":"o","rationale_quote":"q","severity":3,"status":"proposed"} + +{"kind":"decision","test":"T-001","decision":"accepted","at":"2026-09-12"} +` + drafts, decisions, err := ParseRecords(strings.NewReader(in), session.TestsFile) + if err != nil { + t.Fatalf("ParseRecords: %v", err) + } + if len(drafts) != 1 || len(decisions) != 1 { + t.Fatalf("got %d drafts and %d decisions, want 1 and 1", len(drafts), len(decisions)) + } +} + +// TestParseRecordsIgnoresOutOfEnumDecision mirrors analyze.ParseRecords: a +// decision value outside the closed enum is not representable, so it is ignored +// rather than applied — the draft keeps "proposed" and stays in the review queue +// instead of vanishing into a status group nothing renders. +func TestParseRecordsIgnoresOutOfEnumDecision(t *testing.T) { + in := `{"kind":"decision","test":"T-001","decision":"maybe","at":"2026-09-12"}` + "\n" + _, decisions, err := ParseRecords(strings.NewReader(in), session.TestsFile) + if err != nil { + t.Fatalf("ParseRecords: %v", err) + } + if len(decisions) != 0 { + t.Fatalf("an out-of-enum decision was applied: %+v", decisions) + } +} + +func TestParseRecordsRefusesMalformedDraftLines(t *testing.T) { + cases := []struct{ name, in, want string }{ + {"null line", "null\n", "not a test draft or decision record"}, + {"no severity", `{"id":"T-001"}` + "\n", "not a test draft or decision record"}, + {"no id", `{"severity":3}` + "\n", "has no id"}, + {"whitespace id", `{"id":" ","severity":3}` + "\n", "has no id"}, + {"duplicate id", `{"id":"T-001","severity":3}` + "\n" + `{"id":"T-001","severity":3}` + "\n", "duplicate test draft id"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, _, err := ParseRecords(strings.NewReader(tc.in), session.TestsFile) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("got %v, want an error containing %q", err, tc.want) + } + }) + } +} + +// TestParseRecordsRefusesDuplicateIDBySafeText: two ids distinct only by stripped +// bytes render identically on every surface, so they count as the collision they +// display as. +func TestParseRecordsRefusesDuplicateIDBySafeText(t *testing.T) { + in := `{"id":"T-001","severity":3}` + "\n" + `{"id":"T-​001","severity":3}` + "\n" + _, _, err := ParseRecords(strings.NewReader(in), session.TestsFile) + if err == nil || !strings.Contains(err.Error(), "duplicate test draft id") { + t.Fatalf("got %v, want a duplicate-id refusal", err) + } +} + +// TestEffectiveStatusLastDecisionWins is AC3's retention half: decisions apply in +// file order, the last one for a draft wins, and one naming an unknown draft is +// ignored for display. +func TestEffectiveStatusLastDecisionWins(t *testing.T) { + drafts := []Draft{draft("T-001"), draft("T-002")} + decisions := []Decision{ + {Kind: "decision", Test: "T-001", Decision: "accepted", At: "2026-09-12"}, + {Kind: "decision", Test: "T-001", Decision: "rejected", At: "2026-09-13"}, + {Kind: "decision", Test: "T-404", Decision: "accepted", At: "2026-09-13"}, + } + eff := EffectiveStatus(drafts, decisions) + if eff["T-001"].Value != "rejected" || eff["T-001"].At != "2026-09-13" { + t.Fatalf("T-001 status %+v, want rejected on 2026-09-13", eff["T-001"]) + } + if eff["T-002"].Value != "proposed" { + t.Fatalf("T-002 status %+v, want proposed", eff["T-002"]) + } + if _, ok := eff["T-404"]; ok { + t.Fatal("a decision naming an unknown draft created a status entry") + } +} + +func TestEffectiveStatusOnlyCarriesEditForEdited(t *testing.T) { + drafts := []Draft{draft("T-001")} + edit := &Edit{Title: ptr("new title")} + eff := EffectiveStatus(drafts, []Decision{ + {Kind: "decision", Test: "T-001", Decision: "edited", At: "2026-09-12", Edit: edit}, + {Kind: "decision", Test: "T-001", Decision: "accepted", At: "2026-09-13", Edit: edit}, + }) + if eff["T-001"].Edit != nil { + t.Fatalf("an accepted decision carried an edit into the status: %+v", eff["T-001"]) + } +} + +func TestSameIdentityIgnoresStatusOnly(t *testing.T) { + a, b := draft("T-001"), draft("T-001") + b.Status = "accepted" + if !SameIdentity(a, b) { + t.Fatal("SameIdentity treated a status change as a different draft") + } + b.Finding = "F-002" + if SameIdentity(a, b) { + t.Fatal("SameIdentity treated a re-pointed finding as the same draft") + } +} + +// TestClockRendersNegativeSessionRelativeTimes is the clamped-clock regression, +// the sibling of the ones fixed in internal/report and internal/review. A +// recording whose creation_time predates the manifest t0 yields a negative +// offset, and analyze.indexTimeline admits findings anchored there, so the sign +// must be rendered rather than clamped away — and a non-finite or absurd value +// must render a visibly-broken placeholder rather than a precise-looking wrong +// stamp. +func TestClockRendersNegativeSessionRelativeTimes(t *testing.T) { + for _, tc := range []struct { + in float64 + want string + }{ + {22, "00:22"}, + {-90, "-01:30"}, + {-0.2, "00:00"}, + {1e12, "--:--"}, + } { + if got := clock(tc.in); got != tc.want { + t.Fatalf("clock(%g) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/internal/drafttests/emit.go b/internal/drafttests/emit.go new file mode 100644 index 0000000..ed82711 --- /dev/null +++ b/internal/drafttests/emit.go @@ -0,0 +1,199 @@ +package drafttests + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/REPPL/Testimony/internal/analyze" + "github.com/REPPL/Testimony/internal/session" +) + +// outputExample is the worked output-shape example embedded in the request. It +// is illustrative text, not validated data. +const outputExample = `{"rubric":"testimony-testdraft/v1","tests":[ + {"id":"T-001","finding":"F-001","session":"sample-session", + "title":"Saving gives no confirmation", + "steps":["Open #general in the settings prototype.", + "Change the display name to Alice.", + "Click the Save button ([data-testid=save-btn])."], + "expected":"The save is confirmed on screen — a toast, or the button briefly disabled.", + "observed":"Nothing visibly changes, so there is no way to tell the save landed.", + "rationale_quote":"I clicked save and nothing happened", + "severity":3,"status":"proposed"} +]}` + +// EmitRequest builds the single, self-contained regression-test drafting request +// for the session in dir: a versioned rubric, the session context, and, for each +// confirmed finding, that finding's own record plus its event window, so that an +// agent given only this text can answer. window is the event-window half-width in +// seconds. Nothing in the session directory is mutated. +// +// A session with no confirmed finding is staged loudly — the refusal names the +// finding count by status and nothing is emitted — because eligibility is the +// whole point of the step: only evidence a human already vouched for can become +// a test. +func EmitRequest(dir string, window float64) (string, error) { + man, err := session.LoadManifest(dir) + if err != nil { + return "", err + } + findings, verdicts, err := loadFindings(dir) + if err != nil { + return "", err + } + // Eligibility is checked before the timeline is read, so a session with + // nothing to draft from hears why it is empty rather than being sent to run + // merge for a request it could not fill either way. + confirmed := eligible(findings, verdicts) + if len(confirmed) == 0 { + return "", noConfirmedFindings(dir, findings, verdicts) + } + // analyze.LoadTimeline, not a local reader: the event window must not be built + // over a timeline whose entry ids are ambiguous or whose src this pipeline + // cannot place, and the "run merge first" hint is the one an operator without + // a merged timeline needs. + entries, err := analyze.LoadTimeline(dir) + if err != nil { + return "", err + } + + var b strings.Builder + fmt.Fprintf(&b, "Testimony regression-test drafting rubric: %s\n\n", RubricVersion) + + b.WriteString("## Stance\n\n") + b.WriteString("Every test case you draft is a *proposal*, born `proposed`; a human accepts, " + + "edits, or rejects it afterwards. Only the confirmed findings below are eligible — a " + + "human has already vouched for each one. Reconstruct the reproduction steps **only** " + + "from the event window supplied with each finding: never invent a step, a selector, or " + + "a route that is not there. Never alter the quote, the severity, or the session.\n\n") + + b.WriteString("## Instructions\n\n") + b.WriteString("Draft one or more test cases per confirmed finding, in finding-id order. For each draft:\n\n") + b.WriteString("- **`steps`** — the reproduction, in time order. Each entry is one imperative " + + "action a developer can follow, naming the selector or the route where the window names " + + "it, and the sequence ends at the moment the finding is anchored to.\n") + b.WriteString("- **`expected`** — the behaviour the participant expected, grounded in their own utterances in the window.\n") + b.WriteString("- **`observed`** — what the system actually did, grounded in the window's events and utterances.\n") + b.WriteString("- **`title`** — one line naming the defect.\n\n") + + b.WriteString("## Rubric\n\n") + b.WriteString("Field definitions:\n\n") + b.WriteString("- `id` — `T-NNN`, zero-padded and unique within your answer.\n") + b.WriteString("- `finding` — the id of the confirmed finding this draft came from.\n") + b.WriteString("- `session` — the session this draft belongs to, copied unchanged from the session context below.\n") + b.WriteString("- `title` — one line, at most 200 characters.\n") + b.WriteString("- `steps` — a non-empty array of at most 32 non-empty strings, in time order.\n") + b.WriteString("- `expected`, `observed` — non-empty prose.\n") + b.WriteString("- `rationale_quote` — the finding's own `quote`, copied byte for byte.\n") + b.WriteString("- `severity` — the finding's own `severity`, copied unchanged.\n\n") + b.WriteString("Hard constraints (each is enforced when your answer is ingested):\n\n") + b.WriteString("- `rationale_quote` must **equal** the source finding's `quote` — byte for byte, not a re-derivation from the utterance. The drafting step carries evidence forward; it never introduces any.\n") + b.WriteString("- `severity` must equal the source finding's `severity`. Triage order is a human product and is not yours to choose.\n") + b.WriteString("- `session` must equal the session named below.\n") + b.WriteString("- `finding` must name one of the confirmed findings below; a finding that is unverified, rejected, or a duplicate is not eligible.\n") + b.WriteString("- `steps` must list at least one step.\n") + b.WriteString("- `status` is ignored: every draft is written as `proposed`, whatever your answer says.\n\n") + + // The manifest is attacker-authorable — a session directory is an exchange + // unit — and the request is printed to the operator's terminal before it is + // handed to an agent. Every value rendered as prose or a list item outside a + // code fence therefore goes through session.SafeInline, the one shared home + // for the escape set: SafeText's layer stops an ESC-bearing value from driving + // ANSI sequences in the terminal and a newline-bearing one from forging block + // structure (a fake "## " heading, or extra instructions) inside the request + // the agent is asked to obey, and the inline-escape layer stops the constructs + // that need no newline — an unescaped `[x](http://…)` or image form otherwise + // survives as an active link or a tracking beacon the moment a saved + // request.md is previewed. Values rendered inside a fence (each marshalled + // finding and timeline line below) go through SafeText only, which strips the + // terminal-control and Trojan-Source bytes json.Marshal passes through; JSON's + // own structural bytes are ASCII and pass through unchanged. + b.WriteString("## Session\n\n") + fmt.Fprintf(&b, "- Session: %s\n", safeOrNone(man.Session)) + fmt.Fprintf(&b, "- App: %s\n", safeOrNone(man.App)) + fmt.Fprintf(&b, "- Participant: %s\n", safeOrNone(man.Participant)) + // Presence and numbering are decided per task on the rendered form, not raw + // emptiness (analyze.EmitRequest's pattern): a manifest's tasks are + // operator-supplied and unvalidated, so a whitespace-only or + // invisible-only-Unicode entry must not survive SafeText's Cf stripping and + // print as a blank numbered item. + var tasks []string + for _, t := range man.Tasks { + if rendered := session.SafeInline(t); strings.TrimSpace(rendered) != "" { + tasks = append(tasks, rendered) + } + } + if len(tasks) > 0 { + b.WriteString("- Tasks:\n") + for i, t := range tasks { + fmt.Fprintf(&b, " %d. %s\n", i+1, t) + } + } else { + b.WriteString("- Tasks: (none recorded)\n") + } + b.WriteString("\n") + + b.WriteString("## Confirmed findings\n\n") + b.WriteString("Each finding below is confirmed by a human and eligible. Its own record is given " + + "first (copy `quote` and `severity` from it byte for byte), then its event window: the " + + "timeline entries around it, in time order, which are the only source for the steps.\n\n") + for _, f := range confirmed { + fmt.Fprintf(&b, "Finding %s — %s, severity %d, at [%s]:\n\n", + session.SafeInline(f.ID), safeOrDash(f.Type), f.Severity, clock(f.T)) + line, err := json.Marshal(f) + if err != nil { + return "", err + } + b.WriteString("```jsonl\n") + b.WriteString(session.SafeText(string(line))) + b.WriteString("\n```\n\n") + + b.WriteString("Event window:\n\n") + b.WriteString("```jsonl\n") + for _, e := range Window(entries, f, window) { + el, err := json.Marshal(e) + if err != nil { + return "", err + } + b.WriteString(session.SafeText(string(el))) + b.WriteByte('\n') + } + b.WriteString("```\n\n") + } + + b.WriteString("## Answer\n\n") + fmt.Fprintf(&b, "Answer with a single JSON document: `{\"rubric\":\"%s\",\"tests\":[ … ]}`. "+ + "A bare top-level array of drafts is also accepted. Output JSON only, no prose.\n\n", RubricVersion) + b.WriteString("```json\n") + b.WriteString(outputExample) + b.WriteString("\n```\n") + + return b.String(), nil +} + +// safeOrNone applies session.SafeInline and falls back to "(none)" when the +// result renders as nothing (empty or whitespace-only): a manifest field is +// operator-supplied and unvalidated by session.SaveManifest, so a +// whitespace-only or invisible-only value must not survive SafeText's Cf +// stripping and print as a blank field with the "(none)" placeholder skipped. +// The twin of analyze.safeOrNone, for the same fields and the same reason. +func safeOrNone(s string) string { + t := session.SafeInline(s) + if strings.TrimSpace(t) == "" { + return "(none)" + } + return t +} + +// safeOrDash is safeOrNone for a value rendered mid-sentence, where an em dash +// reads better than a parenthesised word: a finding's type is not validated by +// analyze.Load, so a hand-edited or exchanged findings.jsonl can reach this sink +// with one that renders as nothing. +func safeOrDash(s string) string { + t := session.SafeInline(s) + if strings.TrimSpace(t) == "" { + return "—" + } + return t +} diff --git a/internal/drafttests/emit_test.go b/internal/drafttests/emit_test.go new file mode 100644 index 0000000..09616e0 --- /dev/null +++ b/internal/drafttests/emit_test.go @@ -0,0 +1,272 @@ +package drafttests + +import ( + "errors" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/REPPL/Testimony/internal/session" +) + +// TestEmitCarriesConfirmedFindingsAndWindows is AC1's request half: the emitted +// text is self-contained — rubric version, stance, instructions, rubric body, +// session context, the eligible finding's own record, and its event window — so +// an agent given only this text can answer. +func TestEmitCarriesConfirmedFindingsAndWindows(t *testing.T) { + dir := writeSession(t) + got, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + for _, want := range []string{ + "Testimony regression-test drafting rubric: testimony-testdraft/v1", + "## Stance", + "born `proposed`", + "## Instructions", + "## Rubric", + "`rationale_quote` must **equal** the source finding's `quote`", + "## Session", + "- Session: fixture-session", + "- App: testimony demo", + "- Participant: P1", + "1. Change your display name and save it", + "2. Try the appearance settings", + "## Confirmed findings", + "Finding F-001 — bug, severity 3, at [00:22]:", + "Event window:", + "## Answer", + `{"rubric":"testimony-testdraft/v1","tests":[ … ]}`, + outputExample, + } { + if !strings.Contains(got, want) { + t.Fatalf("request is missing %q:\n%s", want, got) + } + } + // The finding's own JSON line, so the quote bytes the model must copy are + // unambiguous. + if !strings.Contains(got, `"quote":"I clicked save and nothing happened"`) { + t.Fatalf("request does not carry F-001's own record:\n%s", got) + } + // Its event window, in time order: the lead-up (the display-name click and + // the typed value) and the aftermath, not merely the anchored moment. + for _, id := range []string{"ev-001", "ev-002", "utt-003", "ev-003", "utt-004", "ev-004", "utt-005", "ev-005", "utt-006"} { + if !strings.Contains(got, `"id":"`+id+`"`) { + t.Fatalf("event window is missing %s:\n%s", id, got) + } + } + // Entries outside the window stay out. + for _, id := range []string{"utt-001", "utt-002", "ev-006", "utt-007"} { + if strings.Contains(got, `"id":"`+id+`"`) { + t.Fatalf("entry %s is outside the window but appears in the request:\n%s", id, got) + } + } +} + +// TestEmitOmitsUnverifiedRejectedAndDuplicateFindings is AC2's emit half: a +// finding that is unverified, rejected, a duplicate, or mode B never reaches the +// model at all. +func TestEmitOmitsUnverifiedRejectedAndDuplicateFindings(t *testing.T) { + dir := writeSession(t) + got, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + for _, id := range []string{"F-002", "F-003", "F-004", "F-005"} { + if strings.Contains(got, id) { + t.Fatalf("non-eligible finding %s appears in the request:\n%s", id, got) + } + } + if strings.Count(got, "Finding F-001") != 1 { + t.Fatalf("expected exactly one per-finding block, got:\n%s", got) + } +} + +// TestEmitWindowFlagWidensAndNarrows pins the -window flag's effect on the +// request itself: at 2.5 seconds utt-003 — the one utterance stating the expected +// behaviour — falls outside the window, which is the worked justification for the +// 10-second default. +func TestEmitWindowFlagWidensAndNarrows(t *testing.T) { + dir := writeSession(t) + narrow, err := EmitRequest(dir, 2.5) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + if strings.Contains(narrow, `"id":"utt-003"`) { + t.Fatalf("utt-003 is outside the 2.5 s window but appears:\n%s", narrow) + } + wide, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + if !strings.Contains(wide, `"id":"utt-003"`) { + t.Fatal("utt-003 is inside the 10 s window but does not appear") + } +} + +func TestEmitRequestIsDeterministic(t *testing.T) { + dir := writeSession(t) + first, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + for i := 0; i < 3; i++ { + again, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + if again != first { + t.Fatal("EmitRequest is not deterministic across runs") + } + } +} + +// TestEmitRefusesWithNoConfirmedFindings is the loud-staging refusal: a session +// whose findings are all unverified, rejected, or duplicates names the counts by +// status, points at `review`, and emits nothing. +func TestEmitRefusesWithNoConfirmedFindings(t *testing.T) { + findings := `{"id":"F-001","t":22,"type":"bug","severity":3,"quote":"q","evidence":["utt-004"],"status":"unverified"} +{"id":"F-002","t":22,"type":"bug","severity":3,"quote":"q","evidence":["utt-004"],"status":"unverified"} +{"id":"F-003","t":22,"type":"bug","severity":3,"quote":"q","evidence":["utt-004"],"status":"unverified"} +{"id":"F-004","t":22,"type":"bug","severity":3,"quote":"q","evidence":["utt-004"],"status":"unverified"} +{"id":"F-005","t":22,"type":"bug","severity":3,"quote":"q","evidence":["utt-004"],"status":"unverified"} +{"kind":"verdict","finding":"F-002","verdict":"rejected","at":"2026-09-12"} +{"kind":"verdict","finding":"F-003","verdict":"rejected","at":"2026-09-12"} +{"kind":"verdict","finding":"F-004","verdict":"duplicate","of":"F-001","at":"2026-09-12"} +` + dir := writeSession(t, session.FindingsFile, findings) + _, err := EmitRequest(dir, 10) + if err == nil { + t.Fatal("EmitRequest on a session with no confirmed finding: want a refusal") + } + if !errors.Is(err, ErrNoConfirmedFindings) { + t.Fatalf("refusal does not wrap ErrNoConfirmedFindings: %v", err) + } + want := "no confirmed findings to draft tests from (5 findings: 0 confirmed, 2 unverified, 1 duplicate, 2 rejected); confirm one with `testimony review -session " + dir + "` first" + if err.Error() != want { + t.Fatalf("refusal message:\n got %q\nwant %q", err.Error(), want) + } +} + +// TestEmitRefusalPrecedesTheTimelineRead: eligibility is the whole point of the +// step, so a session with nothing to draft from hears why it is empty rather than +// being sent to run merge for a request it could not fill either way. +func TestEmitRefusalPrecedesTheTimelineRead(t *testing.T) { + findings := `{"id":"F-001","t":22,"type":"bug","severity":3,"quote":"q","evidence":["utt-004"],"status":"unverified"}` + "\n" + dir := writeSession(t, session.FindingsFile, findings, session.TimelineFile, "") + _, err := EmitRequest(dir, 10) + if !errors.Is(err, ErrNoConfirmedFindings) { + t.Fatalf("want the eligibility refusal ahead of the merge hint, got %v", err) + } +} + +func TestEmitHintsMergeWhenTimelineMissing(t *testing.T) { + dir := writeSession(t, session.TimelineFile, "") + _, err := EmitRequest(dir, 10) + if err == nil || !strings.Contains(err.Error(), "testimony merge") { + t.Fatalf("want a merge hint, got %v", err) + } +} + +func TestEmitHintsIngestWhenFindingsMissing(t *testing.T) { + dir := writeSession(t, session.FindingsFile, "") + _, err := EmitRequest(dir, 10) + if err == nil || !strings.Contains(err.Error(), "testimony analyze -ingest") { + t.Fatalf("want an analyze -ingest hint, got %v", err) + } +} + +// TestEmitMutatesNothing: emit is a read; the session directory is untouched. +func TestEmitMutatesNothing(t *testing.T) { + dir := writeSession(t) + before := dirDigest(t, dir) + if _, err := EmitRequest(dir, 10); err != nil { + t.Fatalf("EmitRequest: %v", err) + } + if after := dirDigest(t, dir); after != before { + t.Fatalf("emit mutated the session directory:\nbefore %s\nafter %s", before, after) + } +} + +func dirDigest(t *testing.T, dir string) string { + t.Helper() + names, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read dir: %v", err) + } + var parts []string + for _, n := range names { + b, err := os.ReadFile(filepath.Join(dir, n.Name())) + if err != nil { + t.Fatalf("read %s: %v", n.Name(), err) + } + parts = append(parts, n.Name()+":"+string(b)) + } + sort.Strings(parts) + return strings.Join(parts, "\n") +} + +// TestEmitEscapesInlineMarkdownInManifestFields is the beacon regression that the +// analysis request already carries: the manifest is attacker-authorable and these +// fields render as list items outside any code fence, so an unescaped +// `[x](http://…)` or image form would survive into a saved request.md as an active +// link or a tracking beacon the moment it is previewed. +func TestEmitEscapesInlineMarkdownInManifestFields(t *testing.T) { + man := `{"session":"fixture-session","app":"[x](http://example.test/beacon.png)","participant":"P1","t0_epoch_ms":1784300400000,"tasks":["![t](http://example.test/t.png)"]}` + dir := writeSession(t, session.ManifestFile, man) + got, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + if strings.Contains(got, "- App: [x](http://example.test/beacon.png)") { + t.Fatalf("manifest app survived as an active link:\n%s", got) + } + if !strings.Contains(got, `\[x\]\(http://example.test/beacon.png\)`) { + t.Fatalf("manifest app is not backslash-escaped:\n%s", got) + } + if !strings.Contains(got, `\!\[t\]\(http://example.test/t.png\)`) { + t.Fatalf("manifest task is not backslash-escaped:\n%s", got) + } +} + +// TestEmitSanitisesTimelineBidi is the Trojan-Source regression for the window +// block. json.Marshal escapes the C0 controls and ESC but passes the Unicode +// Bidi_Control set through as raw bytes, so a right-to-left override in an +// exchanged session's transcript would reorder the displayed window — the exact +// spoofing SafeText strips on every other path. +func TestEmitSanitisesTimelineBidi(t *testing.T) { + tl := "{\"t\":22,\"src\":\"speech\",\"id\":\"utt-004\",\"payload\":{\"speaker\":\"P1\",\"t1\":28,\"text\":\"benign \u202egnihtemos evil\u202c end\"}}\n" + + "{\"t\":19.2,\"src\":\"event\",\"id\":\"ev-003\",\"payload\":{\"kind\":\"click\"}}\n" + + "{\"t\":24.1,\"src\":\"event\",\"id\":\"ev-004\",\"payload\":{\"kind\":\"click\"}}\n" + dir := writeSession(t, session.TimelineFile, tl) + got, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + if strings.ContainsRune(got, 0x202e) || strings.ContainsRune(got, 0x202c) { + t.Fatalf("request carries raw Bidi_Control bytes from the timeline:\n%q", got) + } + if !strings.Contains(got, "benign gnihtemos evil end") { + t.Fatalf("sanitised window lost the utterance text:\n%s", got) + } +} + +// TestEmitPlaceholdersInvisibleOnlyManifestFields: a manifest field that is +// non-empty raw but strips to nothing under SafeText must print the "(none)" +// placeholder rather than a blank field, and a blank task must not be numbered. +func TestEmitPlaceholdersInvisibleOnlyManifestFields(t *testing.T) { + man := "{\"session\":\"fixture-session\",\"app\":\"\u200b\",\"participant\":\" \",\"t0_epoch_ms\":1784300400000,\"tasks\":[\"\u200b\",\"Real task\"]}" + dir := writeSession(t, session.ManifestFile, man) + got, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + if !strings.Contains(got, "- App: (none)") || !strings.Contains(got, "- Participant: (none)") { + t.Fatalf("invisible-only manifest fields did not fall back to the placeholder:\n%s", got) + } + if !strings.Contains(got, " 1. Real task\n") || strings.Contains(got, " 2. ") { + t.Fatalf("a blank task was numbered:\n%s", got) + } +} diff --git a/internal/drafttests/ingest.go b/internal/drafttests/ingest.go new file mode 100644 index 0000000..61daf7a --- /dev/null +++ b/internal/drafttests/ingest.go @@ -0,0 +1,422 @@ +package drafttests + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + "unicode/utf8" + + "github.com/REPPL/Testimony/internal/session" +) + +// source is what a draft must restate about the finding it came from: the quote +// byte for byte and the severity unchanged, both held in the session.SafeText +// form the answering agent was shown. +type source struct { + quote string + severity int + id string // the finding's raw id, for error messages +} + +// Ingest validates the model's answer JSON from r against the draft schema and, +// only if every draft passes, writes tests.jsonl with status forced to +// "proposed". It is the sole validation boundary: a draft naming a finding that +// is not currently confirmed, claiming another session, carrying a quote that is +// not its finding's, restating a different severity, listing no step, or bearing +// a stray field is rejected here, transactionally (all errors reported, nothing +// written on any failure). +// +// It reads manifest.json and findings.jsonl only. Drafts are validated against +// the *findings*, never re-derived from the timeline: the finding is the record a +// human vouched for, and requiring equality against it is what makes this step +// structurally incapable of introducing new evidence. +// +// To protect the retained human record it refuses to overwrite a tests.jsonl +// that already holds decision records. +func Ingest(dir string, r io.Reader) ([]Draft, error) { + man, err := session.LoadManifest(dir) + if err != nil { + return nil, err + } + findings, verdicts, err := loadFindings(dir) + if err != nil { + return nil, err + } + confirmed := eligible(findings, verdicts) + // Refused before a byte of the answer is read: with no eligible finding there + // is nothing a draft could legally reference, so every draft in the answer + // would fail the same rule and the operator would read a wall of errors + // instead of the one fact that explains them. + if len(confirmed) == 0 { + return nil, noConfirmedFindings(dir, findings, verdicts) + } + sources := make(map[string]source, len(confirmed)) + for _, f := range confirmed { + // Keyed and compared in SafeText form, the only form of the finding the + // answering agent is ever shown (EmitRequest routes each marshalled finding + // through SafeText). Indexing the raw bytes while the agent copies the + // sanitised ones would make an honest, verbatim-copied answer impossible to + // validate on a control-character-bearing session. + sources[session.SafeText(f.ID)] = source{ + quote: session.SafeText(f.Quote), + severity: f.Severity, + id: f.ID, + } + } + + // The answer is untrusted model output (this is the validation boundary) and + // -ingest reads it from stdin or a file, so cap the read: a multi-gigabyte + // answer must not OOM the process before validation runs. + data, err := io.ReadAll(io.LimitReader(r, session.MaxAnswerBytes+1)) + if err != nil { + return nil, err + } + if len(data) > session.MaxAnswerBytes { + return nil, fmt.Errorf("answer exceeds %d bytes: refusing to ingest", session.MaxAnswerBytes) + } + raws, rubric, err := parseContainer(data) + if err != nil { + return nil, err + } + if rubric != "" && !knownRubrics[rubric] { + return nil, fmt.Errorf("unknown rubric %q (expected %s)", rubric, RubricVersion) + } + // An empty tests array (a bare `[]`, `{"tests":[]}`, or a truncated answer + // file) is a no-op, not a truncating write: the commit below replaces the + // file whole, so proceeding would erase a prior good tests.jsonl and report + // success. Refuse it, mirroring the decision-overwrite guard. + if len(raws) == 0 { + return nil, fmt.Errorf("answer contains no test drafts; refusing to overwrite %s", session.TestsFile) + } + + // Undecodable elements are dropped before validation, so the surviving slice + // no longer aligns with the answer. Each survivor therefore carries the + // position it held in the answer, and validate labels from that: otherwise a + // failure in the third draft of an answer whose second one was undecodable + // would be reported as "draft #2" — an index into a filtered slice the + // operator never sees, pointing them at the wrong draft to fix. + var ( + decoded []positioned + errs []error + ) + for i, raw := range raws { + d, derr := decodeDraft(raw) + if derr != nil { + errs = append(errs, fmt.Errorf("draft #%d: %v", i+1, derr)) + continue + } + decoded = append(decoded, positioned{draft: d, at: i + 1}) + } + errs = append(errs, validate(decoded, sources, man.Session)...) + + // The model is never trusted: every draft lands proposed, so a draft can never + // be born accepted. Laundering the status here, before the size check below, + // is what makes that check measure the line actually written rather than the + // one the answer proposed. + drafts := make([]Draft, len(decoded)) + for i, p := range decoded { + drafts[i] = p.draft + drafts[i].Status = "proposed" + } + errs = append(errs, oversizedDrafts(drafts, decoded)...) + + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + + if err := commitDrafts(dir, drafts); err != nil { + return nil, err + } + return drafts, nil +} + +// commitDrafts runs the decision guard and the whole-file replacement as one +// locked step, through session.CommitRecords — the same primitive +// analyze.commitFindings uses, so a concurrent `testimony review -kind tests` +// appending a decision cannot slip between the probe and the rewrite and have +// its record destroyed. The drafts were already held to both +// session.MaxJSONLLine and session.MaxJSONLBytes by oversizedDrafts. +func commitDrafts(dir string, drafts []Draft) error { + path := filepath.Join(dir, session.TestsFile) + records := make([][]byte, 0, len(drafts)) + for _, d := range drafts { + b, err := json.Marshal(d) + if err != nil { + return fmt.Errorf("write test drafts: %w", err) + } + records = append(records, b) + } + return session.CommitRecords(session.Commit{ + Path: path, + Records: records, + Guard: func(current io.Reader) error { + held, err := holdsDecisions(current, path) + if err != nil { + return err + } + if held { + return fmt.Errorf("refusing to overwrite %s: it already holds decision records (the retained human record)", session.TestsFile) + } + return nil + }, + }) +} + +// holdsDecisions reports whether the tests.jsonl open on r already contains any +// decision record. It reads through the caller's descriptor — opened under the +// no-follow guard and exclusively locked by session.CommitRecords — rather than +// opening the path itself, so the probe and the write it gates observe the same +// locked file. It scans for raw kind:"decision" lines rather than reusing Load, +// whose decision slice is filtered to the closed enum: a hand-edited or shared +// file whose only decision lines carry a foreign or typo'd value would otherwise +// slip past the guard and have its human-decision records truncated by a +// re-ingest — exactly the record the guard exists to protect. +func holdsDecisions(r io.Reader, path string) (bool, error) { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), session.MaxJSONLLine) + for sc.Scan() { + raw := sc.Bytes() + if len(bytes.TrimSpace(raw)) == 0 { + continue + } + var probe struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(raw, &probe); err != nil { + return false, fmt.Errorf("%s: %w", path, err) + } + if probe.Kind == "decision" { + return true, nil + } + } + if err := sc.Err(); err != nil { + return false, fmt.Errorf("%s: %w", path, err) + } + return false, nil +} + +// oversizedDrafts reports any draft whose tests.jsonl line — its JSON encoding +// plus the newline — would exceed session.MaxJSONLLine, the shared invariant +// every reader scans to, and also refuses an answer whose drafts would together +// exceed session.MaxJSONLBytes once written. maxSteps and maxTitle bound two +// fields, but nothing bounds the length of a step or the number of drafts in an +// answer, so a set of individually valid drafts can still serialise to a +// tests.jsonl that ParseRecords refuses to read back. Both checks run before any +// write and join the transactional error set, so an oversized answer leaves the +// previous tests.jsonl untouched rather than bricking it. Labels come from each +// draft's answer position for the same reason validate's do; a line already +// flagged as over-long is excluded from the total so one oversized draft cannot +// also trigger a redundant total-size error. +func oversizedDrafts(drafts []Draft, decoded []positioned) []error { + var errs []error + var total int64 + var counted int + for i, d := range drafts { + label := draftLabel(d, decoded[i].at) + line, err := json.Marshal(d) + if err != nil { + errs = append(errs, fmt.Errorf("%s: cannot encode as JSON: %w", label, err)) + continue + } + lineLen := int64(len(line) + 1) + if lineLen > session.MaxJSONLLine { + errs = append(errs, fmt.Errorf("%s: encodes to %d bytes, exceeding the %d-byte %s line limit", label, lineLen, session.MaxJSONLLine, session.TestsFile)) + continue + } + total += lineLen + counted++ + } + if total > session.MaxJSONLBytes { + errs = append(errs, fmt.Errorf("test drafts encode to %d bytes across %d drafts, exceeding the %d-byte %s file limit ParseRecords enforces; refusing to write a file review and render could not read back", total, counted, session.MaxJSONLBytes, session.TestsFile)) + } + return errs +} + +// parseContainer accepts either a top-level object with a "tests" array (the +// preferred container, optionally carrying a "rubric") or a bare array of +// drafts. It returns the raw draft elements and the rubric string (empty for the +// bare-array form). +func parseContainer(data []byte) ([]json.RawMessage, string, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 { + return nil, "", fmt.Errorf("empty answer: expected a JSON object or array of test drafts") + } + switch trimmed[0] { + case '[': + var arr []json.RawMessage + if err := json.Unmarshal(trimmed, &arr); err != nil { + return nil, "", fmt.Errorf("parse test drafts array: %w", err) + } + return arr, "", nil + case '{': + var doc struct { + Rubric string `json:"rubric"` + Tests []json.RawMessage `json:"tests"` + } + if err := json.Unmarshal(trimmed, &doc); err != nil { + return nil, "", fmt.Errorf("parse answer: %w", err) + } + if doc.Tests == nil { + return nil, "", fmt.Errorf("answer object has no \"tests\" array") + } + return doc.Tests, doc.Rubric, nil + default: + return nil, "", fmt.Errorf("expected a JSON object or array of test drafts") + } +} + +// rawDraft is how one element of the untrusted answer is decoded before it is +// trusted. Its Severity is a pointer so that an absent "severity" stays +// distinguishable from a present one (the rawFinding.T precedent): without it an +// answer omitting the field decodes to 0 and is reported as a *mismatch* against +// a value the answer never gave, sending the operator to correct a number +// instead of to supply one. Everything else mirrors Draft, which is the shape +// DisallowUnknownFields is closed against. +type rawDraft struct { + ID string `json:"id"` + Finding string `json:"finding"` + Session string `json:"session"` + Title string `json:"title"` + Steps []string `json:"steps"` + Expected string `json:"expected"` + Observed string `json:"observed"` + RationaleQuote string `json:"rationale_quote"` + Severity *int `json:"severity"` + Status string `json:"status"` +} + +// decodeDraft strictly decodes one draft element. DisallowUnknownFields closes +// the shape: a hallucinated or mistyped field is a hard error rather than +// silently dropped. A missing "severity" is rejected here rather than in +// validate, because by the time a draft reaches validate its unset severity is +// indistinguishable from a stated 0 (see rawDraft). +func decodeDraft(raw json.RawMessage) (Draft, error) { + var rd rawDraft + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&rd); err != nil { + return Draft{}, err + } + if rd.Severity == nil { + return Draft{}, fmt.Errorf("missing severity; a draft must restate its finding's severity") + } + return Draft{ + ID: rd.ID, + Finding: rd.Finding, + Session: rd.Session, + Title: rd.Title, + Steps: rd.Steps, + Expected: rd.Expected, + Observed: rd.Observed, + RationaleQuote: rd.RationaleQuote, + Severity: *rd.Severity, + Status: rd.Status, + }, nil +} + +// positioned pairs a decoded draft with at: its 1-based position in the answer +// the operator actually wrote. The two differ whenever an earlier element failed +// to decode, because Ingest drops those before validation; the pairing is what +// lets an error say "draft #3" and mean the third draft in the answer, which is +// the only index the operator can count to. +type positioned struct { + draft Draft + at int +} + +// draftLabel names a draft in an error message: its own id when that id is +// well-formed, and otherwise its position in the answer, which is the only +// handle the operator has on a draft whose id is unusable. +func draftLabel(d Draft, at int) string { + if IsDraftID(d.ID) { + return d.ID + } + return fmt.Sprintf("draft #%d", at) +} + +// validate runs every schema rule against the decoded drafts and returns all +// errors (transactional and exhaustive), each naming the draft, the field, and +// the offending value. Positional labels come from each draft's recorded answer +// position, never from this loop's counter. +// +// Every value the answering agent saw in sanitised form is compared in +// session.SafeText form, and every presence test is decided on the rendered +// form: a value that is non-empty raw but strips to nothing renders as a blank +// in the review prompt and the test plan, so it must be refused here rather than +// admitted as content. +func validate(drafts []positioned, sources map[string]source, wantSession string) []error { + var errs []error + seen := map[string]int{} + + for _, p := range drafts { + d := p.draft + label := draftLabel(d, p.at) + if !IsDraftID(d.ID) { + errs = append(errs, fmt.Errorf("%s: id %q must match ^T-\\d{3}$", label, d.ID)) + } else if prev, dup := seen[d.ID]; dup { + errs = append(errs, fmt.Errorf("%s: duplicate id (first seen at draft #%d)", d.ID, prev)) + } else { + seen[d.ID] = p.at + } + + // finding: must name a currently-confirmed, non-Mode-B finding. Enforced + // here as well as in the request (emit omits every non-eligible finding), + // so a hand-written or stale answer cannot smuggle a draft of an + // unverified, rejected, or duplicate finding past this boundary. + src, ok := sources[session.SafeText(d.Finding)] + if !ok { + errs = append(errs, fmt.Errorf("%s: finding %q is not a confirmed finding in %s", label, d.Finding, session.FindingsFile)) + } + + if session.SafeText(d.Session) != session.SafeText(wantSession) { + errs = append(errs, fmt.Errorf("%s: session %q is not this session (%q in %s)", label, d.Session, wantSession, session.ManifestFile)) + } + + title := strings.TrimSpace(session.SafeText(d.Title)) + if title == "" { + errs = append(errs, fmt.Errorf("%s: title must be non-empty", label)) + } else if n := utf8.RuneCountInString(title); n > maxTitle { + errs = append(errs, fmt.Errorf("%s: title is %d characters, exceeding the limit of %d", label, n, maxTitle)) + } + + if len(d.Steps) == 0 { + errs = append(errs, fmt.Errorf("%s: steps must be non-empty", label)) + } + if len(d.Steps) > maxSteps { + errs = append(errs, fmt.Errorf("%s: steps lists %d entries, exceeding the limit of %d", label, len(d.Steps), maxSteps)) + } + for i, s := range d.Steps { + if strings.TrimSpace(session.SafeText(s)) == "" { + errs = append(errs, fmt.Errorf("%s: step %d must be non-empty", label, i+1)) + } + } + + if strings.TrimSpace(session.SafeText(d.Expected)) == "" { + errs = append(errs, fmt.Errorf("%s: expected must be non-empty", label)) + } + if strings.TrimSpace(session.SafeText(d.Observed)) == "" { + errs = append(errs, fmt.Errorf("%s: observed must be non-empty", label)) + } + + // rationale_quote and severity must be the source finding's own, not a + // re-derivation: equality is what makes this step structurally incapable of + // introducing evidence the human never vouched for, and a mismatch is the + // cheapest available signal that the draft was linked to the wrong finding. + // Both are skipped when the finding itself is unknown — there is nothing to + // compare against, and a second error would only restate the first. + if ok { + if session.SafeText(d.RationaleQuote) != src.quote { + errs = append(errs, fmt.Errorf("%s: rationale_quote %q is not finding %s's quote; it must be copied byte for byte", label, d.RationaleQuote, session.SafeText(src.id))) + } + if d.Severity != src.severity { + errs = append(errs, fmt.Errorf("%s: severity %d does not match finding %s's severity %d", label, d.Severity, session.SafeText(src.id), src.severity)) + } + } + } + return errs +} diff --git a/internal/drafttests/ingest_test.go b/internal/drafttests/ingest_test.go new file mode 100644 index 0000000..a657d19 --- /dev/null +++ b/internal/drafttests/ingest_test.go @@ -0,0 +1,469 @@ +package drafttests + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/REPPL/Testimony/internal/session" +) + +// goodDraft is a schema-clean draft of F-001, the fixture's one eligible +// finding. The validation-failure table below mutates one rule at a time from it. +const goodDraft = `{"id":"T-001","finding":"F-001","session":"fixture-session",` + + `"title":"Saving gives no confirmation","steps":["Open #general.","Click Save."],` + + `"expected":"The save is confirmed on screen.","observed":"Nothing visibly changes.",` + + `"rationale_quote":"I clicked save and nothing happened","severity":3}` + +func answer(drafts ...string) string { + return `{"rubric":"testimony-testdraft/v1","tests":[` + strings.Join(drafts, ",") + `]}` +} + +// with returns goodDraft with one field replaced (or, with a "-" prefix on the +// field name, removed), so each failure case differs from the clean draft by +// exactly the rule under test. +func with(t *testing.T, field string, value any) string { + t.Helper() + var m map[string]any + if err := json.Unmarshal([]byte(goodDraft), &m); err != nil { + t.Fatalf("unmarshal goodDraft: %v", err) + } + if strings.HasPrefix(field, "-") { + delete(m, strings.TrimPrefix(field, "-")) + } else { + m[field] = value + } + b, err := json.Marshal(m) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return string(b) +} + +func testsPath(dir string) string { return filepath.Join(dir, session.TestsFile) } + +// TestIngestGood is the happy path: one clean draft validates, lands in +// tests.jsonl, and is forced to "proposed" even though the answer claimed +// "accepted" — a draft can never be born accepted, the same laundering +// analyze -ingest applies to "unverified". +func TestIngestGood(t *testing.T) { + dir := writeSession(t) + drafts, err := Ingest(dir, strings.NewReader(string(fixture(t, "answer.json")))) + if err != nil { + t.Fatalf("Ingest: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("got %d drafts, want 1", len(drafts)) + } + if drafts[0].Status != "proposed" { + t.Fatalf("draft status %q, want proposed", drafts[0].Status) + } + onDisk, decisions, err := Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(onDisk) != 1 || len(decisions) != 0 { + t.Fatalf("tests.jsonl holds %d drafts and %d decisions, want 1 and 0", len(onDisk), len(decisions)) + } + if onDisk[0].Status != "proposed" { + t.Fatalf("written status %q, want proposed", onDisk[0].Status) + } + if onDisk[0].Finding != "F-001" || onDisk[0].Session != "fixture-session" || onDisk[0].Severity != 3 { + t.Fatalf("written draft lost its link fields: %+v", onDisk[0]) + } +} + +func TestIngestBareArrayAccepted(t *testing.T) { + dir := writeSession(t) + if _, err := Ingest(dir, strings.NewReader("["+goodDraft+"]")); err != nil { + t.Fatalf("Ingest of a bare array: %v", err) + } +} + +// TestIngestValidationFailures is one case per schema rule, each proving a +// precise message. Validation is the sole boundary: a hand-written or stale +// answer cannot smuggle a draft past any of these. +func TestIngestValidationFailures(t *testing.T) { + longTitle := strings.Repeat("x", maxTitle+1) + tooManySteps := make([]string, maxSteps+1) + for i := range tooManySteps { + tooManySteps[i] = "Click Save." + } + cases := []struct { + name, draft, want string + }{ + {"short id", with(t, "id", "T-12"), `id "T-12" must match`}, + {"foreign id prefix", with(t, "id", "X-001"), `id "X-001" must match`}, + {"unknown finding", with(t, "finding", "F-404"), `finding "F-404" is not a confirmed finding`}, + {"unverified finding", with(t, "finding", "F-002"), `finding "F-002" is not a confirmed finding`}, + {"rejected finding", with(t, "finding", "F-003"), `finding "F-003" is not a confirmed finding`}, + {"mode B finding", with(t, "finding", "F-004"), `finding "F-004" is not a confirmed finding`}, + {"duplicate finding", with(t, "finding", "F-005"), `finding "F-005" is not a confirmed finding`}, + {"session mismatch", with(t, "session", "other-session"), `session "other-session" is not this session`}, + {"empty title", with(t, "title", ""), "title must be non-empty"}, + {"whitespace title", with(t, "title", " "), "title must be non-empty"}, + {"over-long title", with(t, "title", longTitle), fmt.Sprintf("title is %d characters, exceeding the limit of %d", maxTitle+1, maxTitle)}, + {"absent steps", with(t, "-steps", nil), "steps must be non-empty"}, + {"empty steps", with(t, "steps", []string{}), "steps must be non-empty"}, + {"whitespace-only step", with(t, "steps", []string{"Open #general.", " "}), "step 2 must be non-empty"}, + {"over-long steps", with(t, "steps", tooManySteps), fmt.Sprintf("steps lists %d entries, exceeding the limit of %d", maxSteps+1, maxSteps)}, + {"empty expected", with(t, "expected", ""), "expected must be non-empty"}, + {"empty observed", with(t, "observed", ""), "observed must be non-empty"}, + {"quote off by one byte", with(t, "rationale_quote", "I clicked save and nothing happened."), "is not finding F-001's quote"}, + {"absent severity", with(t, "-severity", nil), "missing severity"}, + {"mismatched severity", with(t, "severity", 4), "severity 4 does not match finding F-001's severity 3"}, + {"unknown field", strings.Replace(goodDraft, `"steps"`, `"stpes"`, 1), "stpes"}, + {"non-object element", `42`, "cannot unmarshal"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := writeSession(t) + _, err := Ingest(dir, strings.NewReader(answer(tc.draft))) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("got %v, want an error containing %q", err, tc.want) + } + if _, statErr := os.Stat(testsPath(dir)); !os.IsNotExist(statErr) { + t.Fatalf("a refused ingest wrote %s", session.TestsFile) + } + }) + } +} + +func TestIngestRejectsDuplicateID(t *testing.T) { + dir := writeSession(t) + _, err := Ingest(dir, strings.NewReader(answer(goodDraft, goodDraft))) + if err == nil || !strings.Contains(err.Error(), "duplicate id (first seen at draft #1)") { + t.Fatalf("got %v, want a duplicate-id refusal naming the first position", err) + } +} + +// TestIngestRejectsQuoteMatchedInSafeTextForm: the comparison is made in the +// SafeText form, the only form of the finding the answering agent was shown, so +// an honest byte-for-byte copy of the sanitised request validates while a +// different quote does not. +func TestIngestRejectsQuoteMatchedInSafeTextForm(t *testing.T) { + dir := writeSession(t) + // A zero-width space inside the quote strips away under SafeText, so this copy + // is equal in the form both sides are compared in. + d := with(t, "rationale_quote", "I clicked save\u200b and nothing happened") + if _, err := Ingest(dir, strings.NewReader(answer(d))); err != nil { + t.Fatalf("a quote equal in SafeText form was refused: %v", err) + } +} + +// TestIngestRefusesEmptyAnswer: an empty tests array (a bare [], {"tests":[]}, or +// a truncated file) must not erase a prior good tests.jsonl and report success. +func TestIngestRefusesEmptyAnswer(t *testing.T) { + for _, in := range []string{`[]`, `{"tests":[]}`, `{"rubric":"testimony-testdraft/v1","tests":[]}`} { + dir := writeSession(t) + prior := `{"id":"T-009","finding":"F-001","session":"fixture-session","title":"kept","steps":["a"],"expected":"e","observed":"o","rationale_quote":"I clicked save and nothing happened","severity":3,"status":"proposed"}` + "\n" + if err := os.WriteFile(testsPath(dir), []byte(prior), 0o644); err != nil { + t.Fatalf("seed: %v", err) + } + _, err := Ingest(dir, strings.NewReader(in)) + if err == nil || !strings.Contains(err.Error(), "no test drafts") { + t.Fatalf("%s: got %v, want an empty-answer refusal", in, err) + } + b, rerr := os.ReadFile(testsPath(dir)) + if rerr != nil { + t.Fatalf("read: %v", rerr) + } + if string(b) != prior { + t.Fatalf("%s: the prior tests.jsonl was modified: %q", in, b) + } + } +} + +func TestIngestUnknownRubric(t *testing.T) { + dir := writeSession(t) + in := `{"rubric":"testimony-testdraft/v9","tests":[` + goodDraft + `]}` + _, err := Ingest(dir, strings.NewReader(in)) + if err == nil || !strings.Contains(err.Error(), "unknown rubric") { + t.Fatalf("got %v, want an unknown-rubric refusal", err) + } +} + +func TestIngestRejectsOversizedAnswer(t *testing.T) { + dir := writeSession(t) + big := strings.Repeat("x", session.MaxAnswerBytes+1) + _, err := Ingest(dir, strings.NewReader(big)) + if err == nil || !strings.Contains(err.Error(), "refusing to ingest") { + t.Fatalf("got %v, want an over-size refusal", err) + } +} + +// TestIngestRefusesWithNoConfirmedFindings is the ingest half of the loud +// staging: with no eligible finding there is nothing a draft could legally +// reference, so the refusal comes before a byte of the answer is read rather than +// as a wall of per-draft errors. +func TestIngestRefusesWithNoConfirmedFindings(t *testing.T) { + findings := `{"id":"F-001","t":22,"type":"bug","severity":3,"quote":"q","evidence":["utt-004"],"status":"unverified"}` + "\n" + dir := writeSession(t, session.FindingsFile, findings) + _, err := Ingest(dir, strings.NewReader(answer(goodDraft))) + if !errors.Is(err, ErrNoConfirmedFindings) { + t.Fatalf("got %v, want ErrNoConfirmedFindings", err) + } + want := "no confirmed findings to draft tests from (1 findings: 0 confirmed, 1 unverified, 0 duplicate, 0 rejected); confirm one with `testimony review -session " + dir + "` first" + if err.Error() != want { + t.Fatalf("refusal message:\n got %q\nwant %q", err.Error(), want) + } +} + +func TestIngestHintsIngestWhenFindingsMissing(t *testing.T) { + dir := writeSession(t, session.FindingsFile, "") + _, err := Ingest(dir, strings.NewReader(answer(goodDraft))) + if err == nil || !strings.Contains(err.Error(), "testimony analyze -ingest") { + t.Fatalf("got %v, want an analyze -ingest hint", err) + } +} + +// TestIngestReadsNoTimeline: ingest validates drafts against the *findings*, +// never re-derives them from the timeline, so a session with no timeline.jsonl +// ingests cleanly. +func TestIngestReadsNoTimeline(t *testing.T) { + dir := writeSession(t, session.TimelineFile, "") + if _, err := Ingest(dir, strings.NewReader(answer(goodDraft))); err != nil { + t.Fatalf("Ingest without a timeline: %v", err) + } +} + +// TestIngestIsTransactional: three bad drafts report three errors in one run, and +// nothing is written. +func TestIngestIsTransactional(t *testing.T) { + dir := writeSession(t) + a := answer( + with(t, "id", "T-12"), + strings.Replace(with(t, "id", "T-002"), `"expected":"The save is confirmed on screen."`, `"expected":""`, 1), + strings.Replace(with(t, "id", "T-003"), `"severity":3`, `"severity":2`, 1), + ) + _, err := Ingest(dir, strings.NewReader(a)) + if err == nil { + t.Fatal("Ingest accepted an answer with three bad drafts") + } + for _, want := range []string{"must match", "expected must be non-empty", "does not match finding"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the joined error is missing %q:\n%v", want, err) + } + } + if _, statErr := os.Stat(testsPath(dir)); !os.IsNotExist(statErr) { + t.Fatalf("a refused ingest wrote %s", session.TestsFile) + } +} + +// TestIngestLabelsUndecodableNeighbourByAnswerPosition: positions are counted in +// the answer the operator actually wrote, not in the filtered slice validation +// sees, so the error names the draft they can count to. +func TestIngestLabelsUndecodableNeighbourByAnswerPosition(t *testing.T) { + dir := writeSession(t) + a := answer( + with(t, "id", "T-001"), + `{"id":"T-002","stpes":["x"]}`, // undecodable: unknown field + with(t, "id", ""), // third in the answer, unusable id + ) + _, err := Ingest(dir, strings.NewReader(a)) + if err == nil { + t.Fatal("Ingest accepted an answer with an undecodable element") + } + if !strings.Contains(err.Error(), "draft #2:") { + t.Fatalf("the undecodable element is not labelled by its answer position:\n%v", err) + } + if !strings.Contains(err.Error(), "draft #3:") { + t.Fatalf("the third draft is not labelled #3:\n%v", err) + } +} + +// TestIngestRefusesOverwriteWithDecisions protects the retained human record: a +// tests.jsonl already holding a decision is never truncated by a re-ingest. +func TestIngestRefusesOverwriteWithDecisions(t *testing.T) { + dir := writeSession(t) + if _, err := Ingest(dir, strings.NewReader(answer(goodDraft))); err != nil { + t.Fatalf("first Ingest: %v", err) + } + drafts, _, err := Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + rec := Decision{Kind: "decision", Test: "T-001", Decision: "accepted", At: "2026-09-12"} + if err := AppendDecision(dir, rec, &drafts[0]); err != nil { + t.Fatalf("AppendDecision: %v", err) + } + before, err := os.ReadFile(testsPath(dir)) + if err != nil { + t.Fatalf("read: %v", err) + } + + _, err = Ingest(dir, strings.NewReader(answer(goodDraft))) + if err == nil || !strings.Contains(err.Error(), "decision records") { + t.Fatalf("got %v, want a decision-guard refusal", err) + } + after, err := os.ReadFile(testsPath(dir)) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(after) != string(before) { + t.Fatalf("the refused re-ingest modified tests.jsonl:\nbefore %q\nafter %q", before, after) + } +} + +// TestIngestRefusesOverwriteWithForeignDecision: the guard counts any +// kind:"decision" line, including one whose value is outside the closed enum (a +// hand-edited or shared file), so a foreign-valued human decision is never +// silently truncated. +func TestIngestRefusesOverwriteWithForeignDecision(t *testing.T) { + dir := writeSession(t) + seed := `{"kind":"decision","test":"T-001","decision":"maybe","at":"2026-09-12"}` + "\n" + if err := os.WriteFile(testsPath(dir), []byte(seed), 0o644); err != nil { + t.Fatalf("seed: %v", err) + } + _, err := Ingest(dir, strings.NewReader(answer(goodDraft))) + if err == nil || !strings.Contains(err.Error(), "decision records") { + t.Fatalf("got %v, want a decision-guard refusal", err) + } +} + +// TestIngestRejectsOversizedDraftLine holds a single draft to MaxJSONLLine, the +// shared invariant ParseRecords scans to: a line above it makes tests.jsonl +// durably unreadable. +func TestIngestRejectsOversizedDraftLine(t *testing.T) { + dir := writeSession(t) + huge := with(t, "steps", []string{strings.Repeat("x", session.MaxJSONLLine)}) + _, err := Ingest(dir, strings.NewReader(answer(huge))) + if err == nil || !strings.Contains(err.Error(), "line limit") { + t.Fatalf("got %v, want an over-line refusal", err) + } + if _, statErr := os.Stat(testsPath(dir)); !os.IsNotExist(statErr) { + t.Fatalf("a refused ingest wrote %s", session.TestsFile) + } +} + +// TestOversizedDraftsRejectsOversizedTotal is the total-size check, tested +// directly for a small and exactly predictable byte count (as +// session.TestWriteRecordsRollsBackOnWriteError tests writeRecords): five drafts +// each ~3.5 MiB stay under the 4 MiB line cap and sum past the 16 MiB file cap. +// The refusal names the file, not one draft. +func TestOversizedDraftsRejectsOversizedTotal(t *testing.T) { + long := strings.Repeat("x", 3_500_000) + var drafts []Draft + var decoded []positioned + for i := 1; i <= 5; i++ { + d := draft(fmt.Sprintf("T-%03d", i)) + d.Observed = long + drafts = append(drafts, d) + decoded = append(decoded, positioned{draft: d, at: i}) + } + errs := oversizedDrafts(drafts, decoded) + joined := errors.Join(errs...) + if joined == nil || !strings.Contains(joined.Error(), "file limit") { + t.Fatalf("expected a total-size refusal naming the file limit, got %v", joined) + } + for _, err := range errs { + if strings.Contains(err.Error(), "T-001:") { + t.Fatalf("the total-size refusal was attributed to one draft rather than the file: %v", err) + } + } +} + +// TestIngestRejectsOversizedDraftsTotal is the same regression reached through +// the public API at a fraction of session.MaxAnswerBytes: commitDrafts and +// oversizedDrafts both encode with Go's default HTML-escaping encoder, so a run +// of '<' inflates roughly sixfold between the answer and the line that would be +// written — the read-side answer cap therefore cannot be relied on to keep this +// path from executing. +func TestIngestRejectsOversizedDraftsTotal(t *testing.T) { + dir := writeSession(t) + long := strings.Repeat("<", 600_000) + var drafts []string + for i := 1; i <= 5; i++ { + drafts = append(drafts, with(t, "id", fmt.Sprintf("T-%03d", i))) + } + for i := range drafts { + drafts[i] = strings.Replace(drafts[i], `"Nothing visibly changes."`, fmt.Sprintf("%q", long), 1) + } + a := answer(drafts...) + if len(a) >= session.MaxAnswerBytes { + t.Fatalf("test setup: answer is %d bytes, at or over session.MaxAnswerBytes (%d)", len(a), session.MaxAnswerBytes) + } + _, err := Ingest(dir, strings.NewReader(a)) + if err == nil || !strings.Contains(err.Error(), "file limit") { + t.Fatalf("expected a total-size refusal naming the file limit, got %v", err) + } + if _, statErr := os.Stat(testsPath(dir)); !os.IsNotExist(statErr) { + t.Fatalf("a refused ingest wrote %s", session.TestsFile) + } +} + +// TestIngestRefusesSymlink is the arbitrary-file-truncation regression: a +// tests.jsonl planted as a symlink in an exchanged session must not be followed. +func TestIngestRefusesSymlink(t *testing.T) { + dir := writeSession(t) + outside := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(outside, []byte("original\n"), 0o600); err != nil { + t.Fatalf("seed victim: %v", err) + } + if err := os.Symlink(outside, testsPath(dir)); err != nil { + t.Fatalf("symlink: %v", err) + } + if _, err := Ingest(dir, strings.NewReader(answer(goodDraft))); err == nil { + t.Fatal("Ingest followed a symlink; want refusal") + } + if b, _ := os.ReadFile(outside); string(b) != "original\n" { + t.Fatalf("victim file rewritten through symlink: %q", b) + } +} + +// TestSampleTestsJSONLPassesIngest holds the bundled sample to the schema its own +// validator enforces: the three draft lines in examples/sample-session/tests.jsonl +// are re-ingested against that session's manifest and findings, and the written +// lines must come back byte-for-byte identical. The sample is documentation, and +// documentation that would not survive the tool's own boundary is wrong. +func TestSampleTestsJSONLPassesIngest(t *testing.T) { + const sample = "../../examples/sample-session" + drafts, decisions, err := Load(sample) + if err != nil { + t.Fatalf("Load the bundled sample: %v", err) + } + if len(drafts) != 3 || len(decisions) != 3 { + t.Fatalf("the sample holds %d drafts and %d decisions, want 3 and 3", len(drafts), len(decisions)) + } + + // A scratch session carrying only the sample's manifest and findings, so the + // re-ingest is not blocked by the sample's own decision records. + dir := t.TempDir() + for _, name := range []string{session.ManifestFile, session.FindingsFile} { + b, rerr := os.ReadFile(filepath.Join(sample, name)) + if rerr != nil { + t.Fatalf("read %s: %v", name, rerr) + } + if werr := os.WriteFile(filepath.Join(dir, name), b, 0o644); werr != nil { + t.Fatalf("write %s: %v", name, werr) + } + } + var lines []string + for _, d := range drafts { + b, merr := json.Marshal(d) + if merr != nil { + t.Fatalf("marshal: %v", merr) + } + lines = append(lines, string(b)) + } + if _, err := Ingest(dir, strings.NewReader(answer(lines...))); err != nil { + t.Fatalf("the bundled sample's drafts do not pass Ingest: %v", err) + } + + written, err := os.ReadFile(testsPath(dir)) + if err != nil { + t.Fatalf("read: %v", err) + } + sampleBytes, err := os.ReadFile(filepath.Join(sample, session.TestsFile)) + if err != nil { + t.Fatalf("read sample: %v", err) + } + wantPrefix := strings.Join(strings.Split(strings.TrimRight(string(sampleBytes), "\n"), "\n")[:3], "\n") + "\n" + if string(written) != wantPrefix { + t.Fatalf("re-ingesting the sample's drafts does not reproduce them byte for byte:\n got %q\nwant %q", written, wantPrefix) + } +} diff --git a/internal/drafttests/render.go b/internal/drafttests/render.go new file mode 100644 index 0000000..14090b3 --- /dev/null +++ b/internal/drafttests/render.go @@ -0,0 +1,135 @@ +package drafttests + +import ( + "fmt" + "sort" + "strings" + + "github.com/REPPL/Testimony/internal/session" +) + +// Render returns the accepted test plan as Markdown: one test-case block per +// draft whose effective status is "accepted" or "edited", in id order, with the +// winning edit applied over the draft. +// +// "proposed" and "rejected" drafts are omitted — a proposal is not a test, and a +// rejected draft is retained in tests.jsonl for the record, not for the plan. A +// plan with no block in it is refused rather than returned, so `-out FILE` cannot +// truncate an existing test plan into an empty document. +// +// The result is a hand-off artefact: it is never written into the session +// directory unless the operator names a path there, because where a +// docs-as-code test plan lives is their repository's business, not this tool's. +func Render(dir string) (string, error) { + man, err := session.LoadManifest(dir) + if err != nil { + return "", err + } + findings, _, err := loadFindings(dir) + if err != nil { + return "", err + } + drafts, decisions, err := loadDrafts(dir) + if err != nil { + return "", err + } + eff := EffectiveStatus(drafts, decisions) + + // One entry per rendered block: the draft with its winning edit applied, and + // the decision that admitted it. The edit is applied to a copy here; the draft + // line on disk is untouched, which is what keeps its link fields unreachable. + type block struct { + draft Draft + status Status + } + var plan []block + for _, d := range drafts { + st := eff[d.ID] + if st.Value != "accepted" && st.Value != "edited" { + continue + } + plan = append(plan, block{draft: st.Edit.Apply(d), status: st}) + } + if len(plan) == 0 { + return "", noAcceptedDrafts(dir, drafts, decisions) + } + sort.SliceStable(plan, func(i, j int) bool { return plan[i].draft.ID < plan[j].draft.ID }) + + // Every inserted value is untrusted (a draft in an exchanged session is + // attacker-authorable) and this document is one the operator pastes into their + // own repository, so each goes through the shared escape set: mdInline + // (session.SafeInline) wherever the value renders as prose, and mdCode inside + // a code span, where a backslash escape does not apply and a stray backtick + // would close the span early and let the tail render as active markup. + var b strings.Builder + fmt.Fprintf(&b, "# Regression tests — %s\n\n", mdOrNone(man.Session)) + fmt.Fprintf(&b, "Drafted from confirmed findings in session %s (app %s, participant %s). %d of %d drafts accepted.\n\n", + codeOrNone(man.Session), codeOrNone(man.App), codeOrNone(man.Participant), len(plan), len(drafts)) + + for k, blk := range plan { + d, st := blk.draft, blk.status + f := findingByID(findings, d.Finding) + typ, at := "—", "--:--" + if f != nil { + typ, at = mdOrDash(f.Type), clock(f.T) + } + fmt.Fprintf(&b, "## %s — %s\n\n", mdInline(d.ID), mdOrPlaceholder(d.Title, "no title")) + fmt.Fprintf(&b, "- **Source:** finding %s (%s, severity %d) in session %s, at [%s]\n", + mdCode(d.Finding), typ, d.Severity, codeOrNone(d.Session), at) + fmt.Fprintf(&b, "- **Decision:** %s (%s)\n\n", mdOrDash(st.Value), mdOrDash(st.At)) + + b.WriteString("**Steps**\n\n") + for n, s := range d.Steps { + fmt.Fprintf(&b, "%d. %s\n", n+1, mdOrPlaceholder(s, "—")) + } + b.WriteString("\n") + fmt.Fprintf(&b, "**Expected:** %s\n\n", mdOrPlaceholder(d.Expected, "no expected behaviour")) + fmt.Fprintf(&b, "**Observed:** %s\n\n", mdOrPlaceholder(d.Observed, "no observed behaviour")) + fmt.Fprintf(&b, "**Rationale (participant, [%s]):** “%s”\n", at, mdOrPlaceholder(d.RationaleQuote, "no quote")) + if k < len(plan)-1 { + b.WriteString("\n") + } + } + return b.String(), nil +} + +// mdInline neutralises the inline Markdown an attacker-authored draft could +// otherwise smuggle into the rendered plan. The escape set lives in +// session.SafeInline, shared with report.md and the emitted requests, so the +// Markdown artefacts built from untrusted text cannot drift. +func mdInline(s string) string { return session.SafeInline(s) } + +// mdCode renders untrusted text inside a Markdown code span, where backslash +// escapes do not apply. session.SafeText leaves the backtick that would close the +// span early and let the tail render as active markup, so backticks are stripped +// from the span content (a real session name, app, or finding id never carries +// one); everything else is literal inside the span and needs no escaping. This is +// report.mdCode's rule, applied for the same reason. +func mdCode(s string) string { + return "`" + strings.ReplaceAll(session.SafeText(s), "`", "") + "`" +} + +// codeOrNone is mdCode with a "(none)" fallback for a value that renders as +// nothing: a manifest field is operator-supplied and unvalidated, so a +// whitespace-only or invisible-only one must not print as an empty code span. +func codeOrNone(s string) string { + if strings.TrimSpace(strings.ReplaceAll(session.SafeText(s), "`", "")) == "" { + return "`(none)`" + } + return mdCode(s) +} + +func mdOrNone(s string) string { return mdOrPlaceholder(s, "(none)") } + +func mdOrDash(s string) string { return mdOrPlaceholder(s, "—") } + +// mdOrPlaceholder decides presence on the rendered form, not the raw one: a +// value that is non-empty raw but renders to nothing or to whitespace only must +// fall through to the placeholder rather than leave a blank where the plan +// promises content. +func mdOrPlaceholder(s, placeholder string) string { + if strings.TrimSpace(session.SafeText(s)) == "" { + return placeholder + } + return mdInline(s) +} diff --git a/internal/drafttests/render_test.go b/internal/drafttests/render_test.go new file mode 100644 index 0000000..325988d --- /dev/null +++ b/internal/drafttests/render_test.go @@ -0,0 +1,292 @@ +package drafttests + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/REPPL/Testimony/internal/session" + "github.com/REPPL/Testimony/internal/timeline" +) + +const samplePath = "../../examples/sample-session" + +// TestRenderGoldenFromSampleSession holds the bundled sample's rendered test plan +// to a golden file. The sample is the reference instance of the schema, so the +// artefact an operator copies into their own docs-as-code plan is pinned here +// byte-for-byte. +func TestRenderGoldenFromSampleSession(t *testing.T) { + got, err := Render(samplePath) + if err != nil { + t.Fatalf("Render: %v", err) + } + want := string(fixture(t, "tests.md")) + if got != want { + t.Fatalf("rendered plan does not match testdata/tests.md:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestRenderOmitsProposedAndRejected: a proposal is not a test, and a rejected +// draft is retained in tests.jsonl for the record, not for the plan. +func TestRenderOmitsProposedAndRejected(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, Test: "T-001", Decision: "accepted", Out: &out, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + if err := Review(ReviewOptions{Dir: dir, Test: "T-003", Decision: "rejected", Out: &out, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + md, err := Render(dir) + if err != nil { + t.Fatalf("Render: %v", err) + } + if !strings.Contains(md, "## T-001") { + t.Fatalf("the accepted draft is missing:\n%s", md) + } + for _, id := range []string{"## T-002", "## T-003"} { + if strings.Contains(md, id) { + t.Fatalf("%s is proposed or rejected but was rendered:\n%s", id, md) + } + } + if !strings.Contains(md, "1 of 3 drafts accepted.") { + t.Fatalf("the counts line is wrong:\n%s", md) + } +} + +// TestRenderAppliesLastEdit: the rendered fields are the last edited decision's +// edit applied over the draft, computed at render time — the draft line on disk +// stays untouched, which is what keeps its link fields unreachable. +func TestRenderAppliesLastEdit(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + for _, payload := range []string{ + `{"title":"First edit","expected":"First expectation."}`, + `{"title":"Second edit","steps":["Only step."]}`, + } { + err := Review(ReviewOptions{ + Dir: dir, Test: "T-001", Decision: "edited", + EditIn: strings.NewReader(payload), Out: &out, Today: "2026-09-12", + }) + if err != nil { + t.Fatalf("Review: %v", err) + } + } + md, err := Render(dir) + if err != nil { + t.Fatalf("Render: %v", err) + } + if !strings.Contains(md, "## T-001 — Second edit") { + t.Fatalf("the last edit's title was not applied:\n%s", md) + } + if strings.Contains(md, "First edit") || strings.Contains(md, "First expectation.") { + t.Fatalf("an earlier edit leaked into the plan:\n%s", md) + } + if !strings.Contains(md, "1. Only step.\n") { + t.Fatalf("the last edit's steps were not applied:\n%s", md) + } + // A field the last edit does not name keeps the draft's own value. + if !strings.Contains(md, "**Expected:** The save is confirmed on screen.") { + t.Fatalf("a field absent from the last edit did not fall back to the draft:\n%s", md) + } + // The draft line is byte-unchanged. + drafts, _, err := Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + for _, d := range drafts { + if d.ID == "T-001" && d.Title != "Saving gives no confirmation" { + t.Fatalf("the render's edit was written back to the draft: %+v", d) + } + } +} + +// TestRenderRefusesWithNoAcceptedDrafts keeps `-out FILE` from truncating an +// existing test plan into an empty document. +func TestRenderRefusesWithNoAcceptedDrafts(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, Test: "T-003", Decision: "rejected", Out: &out, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + _, err := Render(dir) + if !errors.Is(err, ErrNoAcceptedDrafts) { + t.Fatalf("got %v, want ErrNoAcceptedDrafts", err) + } + want := "no accepted test drafts to render (3 drafts: 0 accepted, 0 edited, 2 proposed, 1 rejected); accept one with `testimony review -session " + dir + " -kind tests` first" + if err.Error() != want { + t.Fatalf("refusal message:\n got %q\nwant %q", err.Error(), want) + } +} + +func TestRenderHintsIngestWhenTestsMissing(t *testing.T) { + dir := writeSession(t) + _, err := Render(dir) + if err == nil || !strings.Contains(err.Error(), "testimony draft-tests -ingest") { + t.Fatalf("got %v, want a draft-tests -ingest hint", err) + } +} + +// TestRenderRendersNegativeClock is the clamped-clock sibling: a recording whose +// creation_time predates the manifest t0 yields negative session-relative times, +// and a finding anchored there must be stamped with its real (signed) moment in +// the plan, not misreported as 00:00. +func TestRenderRendersNegativeClock(t *testing.T) { + findings := `{"id":"F-001","t":-90,"type":"bug","severity":3,"mode":"A","quote":"I clicked save and nothing happened","evidence":["utt-004"],"status":"unverified"} +{"kind":"verdict","finding":"F-001","verdict":"confirmed","at":"2026-09-12"} +` + dir := writeSession(t, session.FindingsFile, findings) + if _, err := Ingest(dir, strings.NewReader(answer(goodDraft))); err != nil { + t.Fatalf("Ingest: %v", err) + } + if err := Review(ReviewOptions{Dir: dir, Test: "T-001", Decision: "accepted", Out: io.Discard, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + md, err := Render(dir) + if err != nil { + t.Fatalf("Render: %v", err) + } + if !strings.Contains(md, "at [-01:30]") { + t.Fatalf("a pre-t0 finding was not stamped with its signed clock:\n%s", md) + } +} + +// TestRenderEscapesInlineMarkdown is the beacon regression for the hand-off +// artefact: the plan is a document the operator pastes into their own repository, +// so an attacker-authored draft field must not survive as an active link or an +// image beacon, and a backtick must not close a code span early and let the tail +// render as markup. +func TestRenderEscapesInlineMarkdown(t *testing.T) { + dir := writeSession(t) + d := with(t, "title", "![x](http://example.test/beacon.png)") + if _, err := Ingest(dir, strings.NewReader(answer(d))); err != nil { + t.Fatalf("Ingest: %v", err) + } + if err := Review(ReviewOptions{Dir: dir, Test: "T-001", Decision: "accepted", Out: io.Discard, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + md, err := Render(dir) + if err != nil { + t.Fatalf("Render: %v", err) + } + if strings.Contains(md, "![x](http://example.test/beacon.png)") { + t.Fatalf("a draft title survived as a live image beacon:\n%s", md) + } + if !strings.Contains(md, `\!\[x\]\(http://example.test/beacon.png\)`) { + t.Fatalf("the draft title is not backslash-escaped:\n%s", md) + } +} + +func TestRenderStripsBackticksFromCodeSpans(t *testing.T) { + man := "{\"session\":\"fixture-session\",\"app\":\"ev`il`\",\"participant\":\"P1\",\"t0_epoch_ms\":1784300400000}" + dir := writeSession(t, session.ManifestFile, man) + if _, err := Ingest(dir, strings.NewReader(answer(goodDraft))); err != nil { + t.Fatalf("Ingest: %v", err) + } + if err := Review(ReviewOptions{Dir: dir, Test: "T-001", Decision: "accepted", Out: io.Discard, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + md, err := Render(dir) + if err != nil { + t.Fatalf("Render: %v", err) + } + if !strings.Contains(md, "(app `evil`, ") { + t.Fatalf("a backtick in a code span was not stripped:\n%s", md) + } +} + +// TestRoundTripGolden is the whole pipeline over a copy of the bundled sample: +// merge → draft-tests (emit) → ingest a known-good answer → three decisions +// (accepted / edited / rejected) → render. It asserts the golden Markdown, the +// append-only property of tests.jsonl, and that findings.jsonl is untouched +// throughout — the drafting layer reads the verified record and never writes to +// it. +func TestRoundTripGolden(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{session.ManifestFile, session.FindingsFile, session.TranscriptFile, session.InteractionsFile} { + b, err := os.ReadFile(filepath.Join(samplePath, name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + if err := os.WriteFile(filepath.Join(dir, name), b, 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + findingsBefore, err := os.ReadFile(filepath.Join(dir, session.FindingsFile)) + if err != nil { + t.Fatalf("read findings: %v", err) + } + + if _, _, err := timeline.Merge(dir); err != nil { + t.Fatalf("merge: %v", err) + } + + // Emit: the request carries the one confirmed finding and its event window. + req, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + if !strings.Contains(req, "Finding F-001 — bug, severity 3, at [00:22]:") { + t.Fatalf("the emitted request does not carry F-001:\n%s", req) + } + + // Ingest: the answer is the bundled sample's own three drafts. + sampleDrafts, sampleDecisions, err := Load(samplePath) + if err != nil { + t.Fatalf("Load the bundled sample: %v", err) + } + var lines []string + for _, d := range sampleDrafts { + b, merr := json.Marshal(d) + if merr != nil { + t.Fatalf("marshal: %v", merr) + } + lines = append(lines, string(b)) + } + if _, err := Ingest(dir, strings.NewReader(answer(lines...))); err != nil { + t.Fatalf("Ingest: %v", err) + } + draftsAfterIngest := draftLines(t, dir) + + // Three decisions, replayed from the sample's own decision records. + for _, dec := range sampleDecisions { + opts := ReviewOptions{Dir: dir, Test: dec.Test, Decision: dec.Decision, Out: io.Discard, Today: dec.At} + if dec.Decision == "edited" { + b, merr := json.Marshal(dec.Edit) + if merr != nil { + t.Fatalf("marshal edit: %v", merr) + } + opts.EditIn = bytes.NewReader(b) + } + if err := Review(opts); err != nil { + t.Fatalf("Review %s %s: %v", dec.Test, dec.Decision, err) + } + } + + // Append-only: every draft line is byte-unchanged by the three decisions. + if got, want := strings.Join(draftLines(t, dir), "\n"), strings.Join(draftsAfterIngest, "\n"); got != want { + t.Fatalf("draft lines changed across the decisions:\n got %q\nwant %q", got, want) + } + + md, err := Render(dir) + if err != nil { + t.Fatalf("Render: %v", err) + } + if want := string(fixture(t, "tests.md")); md != want { + t.Fatalf("the round-trip plan does not match testdata/tests.md:\n--- got ---\n%s\n--- want ---\n%s", md, want) + } + + // findings.jsonl is untouched: the drafting layer reads the verified record. + findingsAfter, err := os.ReadFile(filepath.Join(dir, session.FindingsFile)) + if err != nil { + t.Fatalf("read findings: %v", err) + } + if !bytes.Equal(findingsBefore, findingsAfter) { + t.Fatal("the drafting pipeline modified findings.jsonl") + } +} diff --git a/internal/drafttests/review.go b/internal/drafttests/review.go new file mode 100644 index 0000000..609324e --- /dev/null +++ b/internal/drafttests/review.go @@ -0,0 +1,486 @@ +package drafttests + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/REPPL/Testimony/internal/analyze" + "github.com/REPPL/Testimony/internal/session" +) + +// ReviewOptions configures a `review -kind tests` run. It mirrors review.Options +// field for field where the two agree; the vocabulary differs because a +// decision's "edited" carries a payload no verdict ever does. +type ReviewOptions struct { + Dir string // session directory + Test string // non-interactive: the draft to decide (T-NNN) + Decision string // non-interactive: accepted | edited | rejected + EditIn io.Reader // with Decision "edited": the replacement fields as a JSON object + In io.Reader // interactive input + Out io.Writer // status and prompts + IsTTY bool // whether In is an interactive terminal + Today string // ISO date stamped onto decisions (YYYY-MM-DD) +} + +// Review records human decisions on the session's test drafts. With +// -test/-decision it records one decision non-interactively; otherwise it walks +// the proposed drafts interactively (skipping cleanly when stdin is not a +// terminal, so CI never blocks). +func Review(opts ReviewOptions) error { + // Load reads dir/tests.jsonl, so a session directory that does not exist at + // all satisfies fs.ErrNotExist exactly like one that exists but has simply + // never been through `draft-tests -ingest`. Checking the directory itself + // first names the actual problem and reserves the ingest hint for the case it + // actually describes — the same order review.Run uses for findings. + if fi, err := os.Stat(opts.Dir); err != nil || !fi.IsDir() { + if err == nil { + err = fmt.Errorf("%s is not a directory", opts.Dir) + } + return fmt.Errorf("session directory: %w", err) + } + drafts, decisions, err := loadDrafts(opts.Dir) + if err != nil { + return err + } + + if opts.Test != "" || opts.Decision != "" { + return singleDecision(opts, drafts) + } + + if !opts.IsTTY { + fmt.Fprintln(opts.Out, "review: stdin is not a terminal; skipping the interactive walk "+ + "(use -test T-NNN -decision accepted|edited|rejected for a single decision).") + return nil + } + return walk(opts, drafts, decisions) +} + +// findingsFor reads the session's findings so the walk and the render can name +// each draft's source type and clock. They are decoration, not substance — the +// draft carries the steps, the expectation, and the quote — so an unreadable or +// absent findings.jsonl degrades to placeholders rather than blocking a human +// decision that is already overdue. +func findingsFor(dir string) []analyze.Finding { + findings, _, err := analyze.Load(dir) + if err != nil { + return nil + } + return findings +} + +func findingByID(findings []analyze.Finding, id string) *analyze.Finding { + want := session.SafeText(id) + for i := range findings { + if session.SafeText(findings[i].ID) == want { + f := findings[i] + return &f + } + } + return nil +} + +// singleDecision records one decision non-interactively. +func singleDecision(opts ReviewOptions, drafts []Draft) error { + if opts.Test == "" { + return fmt.Errorf("-test is required with -decision") + } + if opts.Decision == "" { + return fmt.Errorf("-decision is required with -test") + } + decision, err := ParseDecisionFlag(opts.Decision) + if err != nil { + return err + } + // The CLI refuses this at the usage status; checked here too so the refusal is + // a property of the API rather than of one caller's invariants — an edit + // carried alongside "accepted" or "rejected" would otherwise be silently + // discarded. + if decision != "edited" && opts.EditIn != nil { + return fmt.Errorf("-edit applies only to -decision edited") + } + target := draftByID(drafts, opts.Test) + if target == nil { + return fmt.Errorf("test draft %s not found", session.SafeText(opts.Test)) + } + var edit *Edit + if decision == "edited" { + // Every interactive path in this repo has a non-interactive twin; making + // "edited" the one exception would put the only lossy decision out of reach + // of a script or an agent host. + if opts.EditIn == nil { + return fmt.Errorf("-edit is required with -decision edited") + } + edit, err = ParseEdit(opts.EditIn) + if err != nil { + return err + } + } + // The decision's Test must carry the draft's actual (raw) id, not the + // operator's clean flag value: EffectiveStatus keys on the raw id, so a + // decision recorded under the rendered form would silently fail to attach to a + // draft whose raw id draftByID only matched via SafeText. + rec := Decision{Kind: "decision", Test: target.ID, Decision: decision, At: opts.Today, Edit: edit} + if err := AppendDecision(opts.Dir, rec, target); err != nil { + return err + } + fmt.Fprintln(opts.Out, describe(rec)) + return nil +} + +// ParseEdit decodes a replacement-fields object — the `-edit FILE` payload — with +// unknown fields disallowed, so an edit naming `finding`, `session`, `severity`, +// `rationale_quote`, or `id` is a hard error rather than a silently dropped key. +// That is the mechanism by which no human edit can ever re-point a draft at a +// different finding or session: the only way to change the link is to reject the +// draft and ingest a new one. Each present member is held to the draft's own rule +// for that field, and an edit naming no member at all is refused — an "edited" +// decision that changes nothing is not representable. +func ParseEdit(r io.Reader) (*Edit, error) { + data, err := io.ReadAll(io.LimitReader(r, session.MaxAnswerBytes+1)) + if err != nil { + return nil, err + } + if len(data) > session.MaxAnswerBytes { + return nil, fmt.Errorf("edit exceeds %d bytes: refusing to read", session.MaxAnswerBytes) + } + var e Edit + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&e); err != nil { + return nil, fmt.Errorf("parse edit: %w", err) + } + if err := checkEdit(&e); err != nil { + return nil, err + } + return &e, nil +} + +// checkEdit holds each present member of an edit to the draft rule for that +// field, so a replacement can never be weaker than what it replaces. +func checkEdit(e *Edit) error { + if e.empty() { + return fmt.Errorf("edit names no field; give at least one of title, steps, expected, observed") + } + if e.Title != nil { + t := strings.TrimSpace(session.SafeText(*e.Title)) + if t == "" { + return fmt.Errorf("edit: title must be non-empty") + } + if n := len([]rune(t)); n > maxTitle { + return fmt.Errorf("edit: title is %d characters, exceeding the limit of %d", n, maxTitle) + } + } + if e.Steps != nil { + steps := *e.Steps + if len(steps) == 0 { + return fmt.Errorf("edit: steps must be non-empty") + } + if len(steps) > maxSteps { + return fmt.Errorf("edit: steps lists %d entries, exceeding the limit of %d", len(steps), maxSteps) + } + for i, s := range steps { + if strings.TrimSpace(session.SafeText(s)) == "" { + return fmt.Errorf("edit: step %d must be non-empty", i+1) + } + } + } + if e.Expected != nil && strings.TrimSpace(session.SafeText(*e.Expected)) == "" { + return fmt.Errorf("edit: expected must be non-empty") + } + if e.Observed != nil && strings.TrimSpace(session.SafeText(*e.Observed)) == "" { + return fmt.Errorf("edit: observed must be non-empty") + } + return nil +} + +// errPersist marks an error that arose while writing a decision to disk, as +// distinct from the validation errors the walk raises for an unrecognised +// keystroke or a rejected edit. The walk must be able to tell them apart: a +// validation error is a genuine retry situation and is printed as a hint, +// whereas a failed append means the human's decision never reached tests.jsonl. +// Conflating the two would let the walk print a retry hint and exit 0 while +// silently losing the decision, so anything wrapping this sentinel aborts the +// walk and propagates to the CLI's non-zero exit. +var errPersist = errors.New("recording the decision failed") + +// walk interactively decides each proposed draft in id order. +func walk(opts ReviewOptions, drafts []Draft, decisions []Decision) error { + eff := EffectiveStatus(drafts, decisions) + var queue []Draft + for _, d := range drafts { + if eff[d.ID].Value == "proposed" { + queue = append(queue, d) + } + } + sort.Slice(queue, func(i, j int) bool { return queue[i].ID < queue[j].ID }) + if len(queue) == 0 { + fmt.Fprintln(opts.Out, "No proposed test drafts to review.") + return nil + } + findings := findingsFor(opts.Dir) + + r := bufio.NewReader(opts.In) + for i, d := range queue { + fmt.Fprintf(opts.Out, "\n(%d/%d) ", i+1, len(queue)) + printDraft(opts.Out, d, findingByID(findings, d.Finding)) + for { + fmt.Fprint(opts.Out, "[a]ccept [e]dit [r]eject [s]kip [q]uit: ") + choice, err := readLine(r) + if err != nil { + fmt.Fprintln(opts.Out, "\n(end of input) stopping.") + return nil + } + done, quit, verr := applyChoice(opts, d, choice, r) + if verr != nil { + // Only an invalid choice or an invalid edit is worth re-prompting + // for; a persistence failure is not something the operator can + // retype their way out of, and swallowing it here would end the run + // successfully with the decision lost. + if errors.Is(verr, errPersist) { + return verr + } + fmt.Fprintf(opts.Out, " %v\n", verr) + continue + } + if quit { + return nil + } + if done { + break + } + } + } + return nil +} + +// applyChoice handles one keystroke. done means advance to the next draft; quit +// means stop the walk. +func applyChoice(opts ReviewOptions, d Draft, choice string, r *bufio.Reader) (done, quit bool, err error) { + trimmed := strings.TrimSpace(choice) + switch strings.ToLower(trimmed) { + case "a": + return true, false, record(opts, d, Decision{Kind: "decision", Test: d.ID, Decision: "accepted", At: opts.Today}) + case "r": + return true, false, record(opts, d, Decision{Kind: "decision", Test: d.ID, Decision: "rejected", At: opts.Today}) + case "e": + edit, eerr := promptEdit(opts.Out, r, d) + if eerr != nil { + return false, false, eerr + } + // An "edited" decision with an empty edit is not representable, so a pass + // through the prompts that changed nothing is recorded as the acceptance it + // actually was, rather than as a change that never happened. + if edit.empty() { + fmt.Fprintln(opts.Out, " no changes; recorded as accepted.") + return true, false, record(opts, d, Decision{Kind: "decision", Test: d.ID, Decision: "accepted", At: opts.Today}) + } + return true, false, record(opts, d, Decision{Kind: "decision", Test: d.ID, Decision: "edited", At: opts.Today, Edit: edit}) + case "s", "": + fmt.Fprintln(opts.Out, " skipped.") + return true, false, nil + case "q": + return false, true, nil + default: + return false, false, fmt.Errorf("unrecognised choice %q", trimmed) + } +} + +// promptEdit asks for each editable field in turn, showing the current value; a +// blank answer keeps it. steps are read a line at a time until a blank line, and +// a blank first line keeps the current steps. The returned edit names only the +// fields the operator actually replaced. +func promptEdit(w io.Writer, r *bufio.Reader, d Draft) (*Edit, error) { + e := &Edit{} + + fmt.Fprintf(w, " title [%s]: ", orPlaceholder(d.Title, "no title")) + title, err := readLine(r) + if err != nil { + return nil, fmt.Errorf("no input given") + } + if t := strings.TrimSpace(title); t != "" { + e.Title = &t + } + + fmt.Fprintln(w, " steps (one per line, blank line ends; a blank first line keeps them):") + var steps []string + for { + fmt.Fprint(w, " ") + line, lerr := readLine(r) + if lerr != nil { + return nil, fmt.Errorf("no input given") + } + s := strings.TrimSpace(line) + if s == "" { + break + } + // Read to the terminating blank line whatever the count, but stop growing + // one past the cap: the overflow is then reported by checkEdit, naming the + // limit, rather than silently truncated at 32 with the rest of the + // operator's typing left to be consumed as the next prompt's answer. + if len(steps) <= maxSteps { + steps = append(steps, s) + } + } + if len(steps) > 0 { + e.Steps = &steps + } + + fmt.Fprintf(w, " expected [%s]: ", orPlaceholder(d.Expected, "no expected behaviour")) + expected, err := readLine(r) + if err != nil { + return nil, fmt.Errorf("no input given") + } + if t := strings.TrimSpace(expected); t != "" { + e.Expected = &t + } + + fmt.Fprintf(w, " observed [%s]: ", orPlaceholder(d.Observed, "no observed behaviour")) + observed, err := readLine(r) + if err != nil { + return nil, fmt.Errorf("no input given") + } + if t := strings.TrimSpace(observed); t != "" { + e.Observed = &t + } + + if e.empty() { + return e, nil + } + if err := checkEdit(e); err != nil { + return nil, err + } + return e, nil +} + +func record(opts ReviewOptions, judged Draft, rec Decision) error { + if err := AppendDecision(opts.Dir, rec, &judged); err != nil { + // Wrapped so walk can distinguish a lost decision from a mistyped + // keystroke; see errPersist. + return fmt.Errorf("%w: %v", errPersist, err) + } + fmt.Fprintf(opts.Out, " %s\n", describe(rec)) + return nil +} + +// AppendDecision appends one decision record to tests.jsonl without touching any +// existing line (append-only; the latest decision wins for display). The draft +// line is never rewritten, which is what keeps a draft's id, finding, session, +// severity, and rationale_quote unreachable by any later write. +// +// The dangerous part of the write lives once in session.AppendRecord, shared with +// the verdicts findings.jsonl holds; this function supplies the vocabulary and +// the target re-check. +// +// expect, when non-nil, is the draft the operator was shown when they made this +// decision. session.AppendRecord runs the Verify closure over the current drafts +// under its lock and refuses if the targeted id is gone or now names a different +// draft: `review -kind tests` snapshots the drafts once and then blocks on the +// operator, a concurrent `draft-tests -ingest` may truncate-and-rewrite in that +// gap (permitted until the first decision exists), and draft ids restart at +// T-001 — so without the re-check a decision would silently attach to a different +// draft. +func AppendDecision(dir string, d Decision, expect *Draft) error { + b, err := json.Marshal(d) + if err != nil { + return err + } + a := session.Append{ + Path: filepath.Join(dir, session.TestsFile), + Record: b, + // The draft id is attacker-authorable in an exchanged session and the label + // reaches the operator's terminal through cli.fail, so it is sanitised here + // rather than inside the shared primitive, which never sees an id as such. + Label: "decision for " + session.SafeText(d.Test), + Kind: "decision", + } + if expect != nil { + judged := *expect + a.Verify = func(current io.Reader) error { return verifyTarget(current, d, judged) } + } + return session.AppendRecord(a) +} + +// verifyTarget re-reads the drafts currently in the locked tests.jsonl +// (session.AppendRecord hands it a reader over the file's contents, taken under +// the append lock) and confirms the decision d still applies to the draft expect +// — the one the operator was shown. +func verifyTarget(current io.Reader, d Decision, expect Draft) error { + drafts, _, err := ParseRecords(current, session.TestsFile) + if err != nil { + return err + } + cur := draftByID(drafts, d.Test) + if cur == nil { + return fmt.Errorf("test draft %s is no longer in %s; it changed since review started — re-run `testimony review -kind tests`", + session.SafeText(d.Test), session.TestsFile) + } + if !SameIdentity(*cur, expect) { + return fmt.Errorf("test draft %s changed since review started (a re-ingest rewrote %s); re-run `testimony review -kind tests` before recording a decision", + session.SafeText(d.Test), session.TestsFile) + } + return nil +} + +// printDraft writes a draft to the operator's terminal. Every +// attacker-influenceable field (a draft in a downloaded session is untrusted) is +// passed through session.SafeText first, so embedded ESC/ANSI or control bytes +// cannot manipulate the terminal, and presence is decided on the rendered form so +// a value that strips to nothing falls through to a placeholder rather than +// printing as a blank. f is the source finding when it is still readable: it +// supplies the type and the clock, and its absence degrades to placeholders +// rather than blocking the decision. +func printDraft(w io.Writer, d Draft, f *analyze.Finding) { + typ, at := "—", "--:--" + if f != nil { + typ, at = orPlaceholder(f.Type, "—"), clock(f.T) + } + fmt.Fprintf(w, "%s — from %s (%s, severity %d), [%s]\n", + session.SafeText(d.ID), session.SafeText(d.Finding), typ, d.Severity, at) + fmt.Fprintf(w, " %s\n", orPlaceholder(d.Title, "no title")) + fmt.Fprintln(w, " steps:") + if len(d.Steps) == 0 { + fmt.Fprintln(w, " (none)") + } + for i, s := range d.Steps { + fmt.Fprintf(w, " %d. %s\n", i+1, orPlaceholder(s, "—")) + } + fmt.Fprintf(w, " expected: %s\n", orPlaceholder(d.Expected, "no expected behaviour")) + fmt.Fprintf(w, " observed: %s\n", orPlaceholder(d.Observed, "no observed behaviour")) + fmt.Fprintf(w, " “%s”\n", orPlaceholder(d.RationaleQuote, "no quote")) +} + +// describe echoes a recorded decision to the operator's terminal. Its fields +// derive from an attacker-authorable draft id in a downloaded session, so each is +// passed through SafeText — matching printDraft and review.describe. +func describe(d Decision) string { + return fmt.Sprintf("recorded: %s %s (%s)", + session.SafeText(d.Test), session.SafeText(d.Decision), session.SafeText(d.At)) +} + +// orPlaceholder renders untrusted text for the terminal, falling back to a +// placeholder when it renders as nothing (empty, whitespace-only, or +// invisible-only Unicode) — the review.printFinding pattern, applied to every +// field of a draft because ParseRecords validates none of them. +func orPlaceholder(s, placeholder string) string { + t := session.SafeText(s) + if strings.TrimSpace(t) == "" { + return placeholder + } + return t +} + +func readLine(r *bufio.Reader) (string, error) { + line, err := r.ReadString('\n') + if err != nil && line == "" { + return "", err + } + return line, nil +} diff --git a/internal/drafttests/review_test.go b/internal/drafttests/review_test.go new file mode 100644 index 0000000..9cb724b --- /dev/null +++ b/internal/drafttests/review_test.go @@ -0,0 +1,584 @@ +package drafttests + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +// ingestThree lays a session whose tests.jsonl holds three proposed drafts of +// F-001 — a finding may legitimately yield more than one test case — which is +// what lets the walk and the render exercise all three decisions. +func ingestThree(t *testing.T) string { + t.Helper() + dir := writeSession(t) + var drafts []string + for i := 1; i <= 3; i++ { + drafts = append(drafts, with(t, "id", fmt.Sprintf("T-%03d", i))) + } + if _, err := Ingest(dir, strings.NewReader(answer(drafts...))); err != nil { + t.Fatalf("Ingest: %v", err) + } + return dir +} + +// draftLines returns only the draft (non-decision) lines, to assert the +// append-only property. +func draftLines(t *testing.T, dir string) []string { + t.Helper() + b, err := os.ReadFile(testsPath(dir)) + if err != nil { + t.Fatalf("read: %v", err) + } + var out []string + for _, l := range strings.Split(strings.TrimRight(string(b), "\n"), "\n") { + if !strings.Contains(l, `"kind":"decision"`) { + out = append(out, l) + } + } + return out +} + +func effective(t *testing.T, dir string) map[string]Status { + t.Helper() + drafts, decisions, err := Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + return EffectiveStatus(drafts, decisions) +} + +// TestDecisionIsAppendedAndDraftLinesUnchanged is AC3's first mechanism, asserted +// byte-for-byte: a decision is a new line, the draft line is never rewritten, so +// finding, session, severity, rationale_quote, and id are unreachable by any +// later write. +func TestDecisionIsAppendedAndDraftLinesUnchanged(t *testing.T) { + dir := ingestThree(t) + before := draftLines(t, dir) + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, Test: "T-001", Decision: "accepted", Out: &out, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + if got := out.String(); got != "recorded: T-001 accepted (2026-09-12)\n" { + t.Fatalf("echo = %q", got) + } + if effective(t, dir)["T-001"].Value != "accepted" { + t.Fatal("T-001 is not accepted") + } + after := draftLines(t, dir) + if strings.Join(before, "\n") != strings.Join(after, "\n") { + t.Fatalf("draft lines changed after a decision:\nbefore %q\nafter %q", before, after) + } +} + +func TestSingleDecisionRejected(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, Test: "T-003", Decision: "rejected", Out: &out, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + if effective(t, dir)["T-003"].Value != "rejected" { + t.Fatal("T-003 is not rejected") + } +} + +// TestSingleDecisionEdited covers the non-interactive twin of the interactive +// edit: every interactive path in this repo has one, and making "edited" the +// exception would put the only lossy decision out of reach of a script or an +// agent host. +func TestSingleDecisionEdited(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + err := Review(ReviewOptions{ + Dir: dir, Test: "T-002", Decision: "edited", + EditIn: bytes.NewReader(fixture(t, "edit.json")), + Out: &out, Today: "2026-09-12", + }) + if err != nil { + t.Fatalf("Review: %v", err) + } + st := effective(t, dir)["T-002"] + if st.Value != "edited" { + t.Fatalf("T-002 status %+v, want edited", st) + } + if st.Edit == nil || st.Edit.Title == nil || *st.Edit.Title != "Saving a display name gives no confirmation" { + t.Fatalf("the edit did not survive the round trip: %+v", st.Edit) + } + // The draft line itself is untouched: the edit is applied at render time. + drafts, _, err := Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + for _, d := range drafts { + if d.ID == "T-002" && d.Title != "Saving gives no confirmation" { + t.Fatalf("the edit rewrote the draft line: %+v", d) + } + } +} + +// TestEditCannotNameFindingOrSessionOrSeverityOrQuoteOrID is AC3's second +// mechanism: the edit object is a closed four-field subset decoded with +// DisallowUnknownFields, so there is no path through which a human edit can +// re-point a draft at a different finding or session. Each is a hard error, not a +// silently dropped key. +func TestEditCannotNameFindingOrSessionOrSeverityOrQuoteOrID(t *testing.T) { + for _, field := range []string{"finding", "session", "severity", "rationale_quote", "id", "status"} { + t.Run(field, func(t *testing.T) { + dir := ingestThree(t) + payload := fmt.Sprintf(`{"title":"ok","%s":"x"}`, field) + var out bytes.Buffer + err := Review(ReviewOptions{ + Dir: dir, Test: "T-002", Decision: "edited", + EditIn: strings.NewReader(payload), Out: &out, Today: "2026-09-12", + }) + if err == nil || !strings.Contains(err.Error(), field) { + t.Fatalf("an edit naming %s: got %v, want a hard error naming the field", field, err) + } + if effective(t, dir)["T-002"].Value != "proposed" { + t.Fatal("a refused edit still recorded a decision") + } + }) + } +} + +func TestEditRejectsEmptyAndWeakMembers(t *testing.T) { + cases := []struct{ name, payload, want string }{ + {"no members", `{}`, "edit names no field"}, + {"empty title", `{"title":" "}`, "title must be non-empty"}, + {"empty steps", `{"steps":[]}`, "steps must be non-empty"}, + {"blank step", `{"steps":["ok"," "]}`, "step 2 must be non-empty"}, + {"empty expected", `{"expected":""}`, "expected must be non-empty"}, + {"empty observed", `{"observed":""}`, "observed must be non-empty"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := ParseEdit(strings.NewReader(tc.payload)); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("got %v, want an error containing %q", err, tc.want) + } + }) + } +} + +func TestSingleDecisionErrors(t *testing.T) { + dir := ingestThree(t) + cases := []struct{ name, test, decision, want string }{ + {"unknown draft", "T-404", "accepted", "test draft T-404 not found"}, + {"bad decision", "T-001", "maybe", "invalid decision"}, + {"decision without test", "", "accepted", "-test is required"}, + {"test without decision", "T-001", "", "-decision is required"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + err := Review(ReviewOptions{Dir: dir, Test: tc.test, Decision: tc.decision, Out: &out, Today: "2026-09-12"}) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("got %v, want an error containing %q", err, tc.want) + } + }) + } +} + +func TestSingleDecisionEditedRequiresEdit(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + err := Review(ReviewOptions{Dir: dir, Test: "T-001", Decision: "edited", Out: &out, Today: "2026-09-12"}) + if err == nil || !strings.Contains(err.Error(), "-edit is required") { + t.Fatalf("got %v, want an -edit requirement", err) + } +} + +// TestLastDecisionWinsOnDisk: a decision may be appended even when one already +// exists (append-only correction), and the later one wins. +func TestLastDecisionWinsOnDisk(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + for _, d := range []string{"accepted", "rejected"} { + if err := Review(ReviewOptions{Dir: dir, Test: "T-001", Decision: d, Out: &out, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review %s: %v", d, err) + } + } + if got := effective(t, dir)["T-001"].Value; got != "rejected" { + t.Fatalf("effective status %q, want rejected (the later decision)", got) + } + drafts, decisions, err := Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(drafts) != 3 || len(decisions) != 2 { + t.Fatalf("got %d drafts and %d decisions, want 3 and 2 (both retained)", len(drafts), len(decisions)) + } +} + +// TestAppendDecisionRefusesWhenDraftChangedUnderTheLock is the +// decision-misattribution regression. `review -kind tests` snapshots the drafts +// once and then blocks on the operator; a concurrent `draft-tests -ingest` may +// truncate-and-rewrite in that gap (permitted until the first decision exists), +// and because draft ids restart at T-001 the decision would otherwise attach to a +// different draft. +func TestAppendDecisionRefusesWhenDraftChangedUnderTheLock(t *testing.T) { + dir := ingestThree(t) + drafts, _, err := Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + shown := drafts[0] + + // A concurrent re-ingest replaces T-001 with a different draft under the same id. + replacement := strings.Replace(with(t, "id", "T-001"), `"Saving gives no confirmation"`, `"Something else entirely"`, 1) + if _, err := Ingest(dir, strings.NewReader(answer(replacement))); err != nil { + t.Fatalf("re-ingest: %v", err) + } + + rec := Decision{Kind: "decision", Test: "T-001", Decision: "accepted", At: "2026-09-12"} + err = AppendDecision(dir, rec, &shown) + if err == nil || !strings.Contains(err.Error(), "changed since review started") { + t.Fatalf("got %v, want a changed-under-the-lock refusal", err) + } + if _, decisions, lerr := Load(dir); lerr != nil || len(decisions) != 0 { + t.Fatalf("the refused decision was written anyway (%d decisions, err %v)", len(decisions), lerr) + } +} + +func TestAppendDecisionRefusesWhenDraftVanished(t *testing.T) { + dir := ingestThree(t) + drafts, _, err := Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + shown := drafts[0] + // A re-ingest that keeps only T-002 — T-001 is gone. + if _, err := Ingest(dir, strings.NewReader(answer(with(t, "id", "T-002")))); err != nil { + t.Fatalf("re-ingest: %v", err) + } + err = AppendDecision(dir, Decision{Kind: "decision", Test: "T-001", Decision: "accepted", At: "2026-09-12"}, &shown) + if err == nil || !strings.Contains(err.Error(), "no longer in") { + t.Fatalf("got %v, want a vanished-draft refusal", err) + } +} + +// TestAppendDecisionSanitisesDraftID: a draft id in an exchanged tests.jsonl is +// attacker-controlled and these errors reach the operator's terminal through +// cli.fail, so every id-bearing error path routes it through session.SafeText. +func TestAppendDecisionSanitisesDraftID(t *testing.T) { + dir := ingestThree(t) + evil := "\x1b]0;pwned\x07T-404" + shown := draft(evil) + err := AppendDecision(dir, Decision{Kind: "decision", Test: evil, Decision: "accepted", At: "2026-09-12"}, &shown) + if err == nil { + t.Fatal("want a refusal for an absent draft") + } + if strings.ContainsRune(err.Error(), '\x1b') || strings.ContainsRune(err.Error(), '\x07') { + t.Fatalf("error carries raw terminal-control bytes from the draft id: %q", err.Error()) + } + if !strings.Contains(err.Error(), "T-404") { + t.Fatalf("sanitised error dropped the id's printable tail: %q", err.Error()) + } +} + +func TestAppendDecisionRefusesSymlink(t *testing.T) { + dir := writeSession(t) + outside := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(outside, []byte("original\n"), 0o600); err != nil { + t.Fatalf("seed victim: %v", err) + } + if err := os.Symlink(outside, testsPath(dir)); err != nil { + t.Fatalf("symlink: %v", err) + } + err := AppendDecision(dir, Decision{Kind: "decision", Test: "T-001", Decision: "accepted", At: "2026-09-12"}, nil) + if err == nil { + t.Fatal("AppendDecision followed a symlink; want refusal") + } + if b, _ := os.ReadFile(outside); string(b) != "original\n" { + t.Fatalf("victim file appended through symlink: %q", b) + } +} + +// --- the interactive walk -------------------------------------------------- + +func TestInteractiveGatedWhenNotTTY(t *testing.T) { + dir := ingestThree(t) + before := draftLines(t, dir) + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader(""), Out: &out, IsTTY: false, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + if !strings.Contains(out.String(), "not a terminal") { + t.Fatalf("expected a TTY-gating notice, got %q", out.String()) + } + if _, decisions, _ := Load(dir); len(decisions) != 0 { + t.Fatal("a gated walk recorded a decision") + } + if strings.Join(before, "\n") != strings.Join(draftLines(t, dir), "\n") { + t.Fatal("a gated walk mutated the file") + } +} + +// TestInteractiveWalk covers a, e and r in one pass over three drafts, with the +// printed block asserted in full: the block is the surface the operator actually +// decides from. +func TestInteractiveWalk(t *testing.T) { + dir := ingestThree(t) + script := strings.Join([]string{ + "a", + "e", "Saving a display name gives no confirmation", "Open #general.", "Click Save.", "", + "New expectation.", "New observation.", + "r", + "", + }, "\n") + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader(script), Out: &out, IsTTY: true, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + eff := effective(t, dir) + if eff["T-001"].Value != "accepted" || eff["T-002"].Value != "edited" || eff["T-003"].Value != "rejected" { + t.Fatalf("statuses: %+v", eff) + } + got := out.String() + for _, want := range []string{ + "(1/3) T-001 — from F-001 (bug, severity 3), [00:22]\n", + " Saving gives no confirmation\n", + " steps:\n 1. Open #general.\n 2. Click Save.\n", + " expected: The save is confirmed on screen.\n", + " observed: Nothing visibly changes.\n", + " “I clicked save and nothing happened”\n", + "[a]ccept [e]dit [r]eject [s]kip [q]uit: ", + " recorded: T-001 accepted (2026-09-12)\n", + " recorded: T-002 edited (2026-09-12)\n", + " recorded: T-003 rejected (2026-09-12)\n", + } { + if !strings.Contains(got, want) { + t.Fatalf("the walk's output is missing %q:\n%s", want, got) + } + } + st := eff["T-002"] + if st.Edit == nil || st.Edit.Steps == nil || len(*st.Edit.Steps) != 2 || (*st.Edit.Steps)[1] != "Click Save." { + t.Fatalf("the interactive edit did not capture the steps: %+v", st.Edit) + } + if st.Edit.Expected == nil || *st.Edit.Expected != "New expectation." { + t.Fatalf("the interactive edit did not capture expected: %+v", st.Edit) + } + if strings.Contains(got, "(4/3)") { + t.Fatal("the walk continued past its queue") + } +} + +// TestInteractiveEditBlankKeepsEachField: a blank answer keeps the current value, +// so an operator correcting one field does not have to retype the rest. +func TestInteractiveEditBlankKeepsEachField(t *testing.T) { + dir := ingestThree(t) + // title kept, steps kept (blank first line), expected replaced, observed kept. + script := strings.Join([]string{"e", "", "", "Only the expectation changes.", "", "q", ""}, "\n") + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader(script), Out: &out, IsTTY: true, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + st := effective(t, dir)["T-001"] + if st.Value != "edited" || st.Edit == nil { + t.Fatalf("T-001 status %+v, want edited", st) + } + if st.Edit.Title != nil || st.Edit.Steps != nil || st.Edit.Observed != nil { + t.Fatalf("a blank answer replaced a field: %+v", st.Edit) + } + if st.Edit.Expected == nil || *st.Edit.Expected != "Only the expectation changes." { + t.Fatalf("the replaced field did not survive: %+v", st.Edit) + } + // The prompt shows the current value, so the operator can see what blank keeps. + if !strings.Contains(out.String(), " title [Saving gives no confirmation]: ") { + t.Fatalf("the edit prompt does not show the current title:\n%s", out.String()) + } +} + +// TestInteractiveEditWithNoChangesRecordsAccepted: an "edited" decision with an +// empty edit records a change that did not happen, and is not representable — so +// a pass through the prompts that changed nothing is recorded as the acceptance +// it actually was. +func TestInteractiveEditWithNoChangesRecordsAccepted(t *testing.T) { + dir := ingestThree(t) + script := strings.Join([]string{"e", "", "", "", "", "q", ""}, "\n") + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader(script), Out: &out, IsTTY: true, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + if !strings.Contains(out.String(), " no changes; recorded as accepted.\n") { + t.Fatalf("expected the no-changes notice:\n%s", out.String()) + } + st := effective(t, dir)["T-001"] + if st.Value != "accepted" || st.Edit != nil { + t.Fatalf("T-001 status %+v, want accepted with no edit", st) + } +} + +func TestInteractiveSkipAndQuit(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + // skip the first, quit on the second. + if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader("s\nq\n"), Out: &out, IsTTY: true, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + if !strings.Contains(out.String(), " skipped.\n") { + t.Fatalf("expected a skip notice:\n%s", out.String()) + } + if _, decisions, _ := Load(dir); len(decisions) != 0 { + t.Fatal("skip or quit recorded a decision") + } + if !strings.Contains(out.String(), "(2/3)") || strings.Contains(out.String(), "(3/3)") { + t.Fatalf("quit did not stop the walk at the second draft:\n%s", out.String()) + } +} + +func TestInteractiveUnrecognisedChoiceReprompts(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader("x\nq\n"), Out: &out, IsTTY: true, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + if !strings.Contains(out.String(), `unrecognised choice "x"`) { + t.Fatalf("expected an unrecognised-choice hint:\n%s", out.String()) + } + if strings.Count(out.String(), "[a]ccept") < 2 { + t.Fatalf("the walk did not re-prompt after an unrecognised choice:\n%s", out.String()) + } +} + +func TestInteractiveEndOfInputStopsCleanly(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader(""), Out: &out, IsTTY: true, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + if !strings.Contains(out.String(), "(end of input) stopping.") { + t.Fatalf("expected an end-of-input notice:\n%s", out.String()) + } +} + +func TestInteractiveNoProposedDrafts(t *testing.T) { + dir := ingestThree(t) + for _, id := range []string{"T-001", "T-002", "T-003"} { + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, Test: id, Decision: "rejected", Out: &out, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review %s: %v", id, err) + } + } + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader("a\n"), Out: &out, IsTTY: true, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + if !strings.Contains(out.String(), "No proposed test drafts to review.") { + t.Fatalf("expected the empty-queue notice:\n%s", out.String()) + } +} + +func TestReviewHintsIngestWhenTestsMissing(t *testing.T) { + dir := writeSession(t) + err := Review(ReviewOptions{Dir: dir, In: strings.NewReader(""), Out: io.Discard, IsTTY: true, Today: "2026-09-12"}) + if err == nil || !strings.Contains(err.Error(), "testimony draft-tests -ingest") { + t.Fatalf("got %v, want a draft-tests -ingest hint", err) + } +} + +// TestReviewRefusesNonexistentSessionDir: a wrong -session path names the actual +// problem rather than sending the operator to run an ingest that would fail +// identically one command later. +func TestReviewRefusesNonexistentSessionDir(t *testing.T) { + dir := filepath.Join(t.TempDir(), "does-not-exist") + err := Review(ReviewOptions{Dir: dir, Test: "T-001", Decision: "accepted", Out: io.Discard, Today: "2026-09-12"}) + if err == nil { + t.Fatal("want an error for a nonexistent session directory") + } + if strings.Contains(err.Error(), "draft-tests -ingest") { + t.Fatalf("a missing session directory must not be reported as an un-ingested one: %v", err) + } + if !strings.Contains(err.Error(), dir) { + t.Fatalf("error must name the session directory: %v", err) + } +} + +// TestPrintDraftSanitisesAndPlaceholders: a draft in a downloaded session is +// untrusted and ParseRecords validates none of its text, so every field is routed +// through session.SafeText and a field that renders as nothing falls through to a +// placeholder rather than printing as a blank. +func TestPrintDraftSanitisesAndPlaceholders(t *testing.T) { + var out bytes.Buffer + d := Draft{ + ID: "T-001", Finding: "F-001", Severity: 3, + Title: "\x1b]0;pwned\x07hijacked", + Steps: []string{"\u200b"}, + Expected: " ", + Observed: "", + Status: "proposed", + } + printDraft(&out, d, nil) + got := out.String() + if strings.ContainsRune(got, '\x1b') || strings.ContainsRune(got, '\x07') { + t.Fatalf("printDraft emitted raw terminal-control bytes: %q", got) + } + for _, want := range []string{" 1. —\n", " expected: no expected behaviour\n", " observed: no observed behaviour\n", " “no quote”\n", "(—, severity 3), [--:--]"} { + if !strings.Contains(got, want) { + t.Fatalf("printDraft is missing %q:\n%s", want, got) + } + } +} + +// TestEditRoundTripsThroughJSON pins the decision record's shape on disk, since +// it is the schema the session-directory reference documents. +func TestEditRoundTripsThroughJSON(t *testing.T) { + steps := []string{"Open #general.", "Click Save."} + rec := Decision{Kind: "decision", Test: "T-003", Decision: "edited", At: "2026-09-12", + Edit: &Edit{Title: ptr("Saving a display name gives no confirmation"), Steps: &steps}} + b, err := json.Marshal(rec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + want := `{"kind":"decision","test":"T-003","decision":"edited","at":"2026-09-12","edit":{"title":"Saving a display name gives no confirmation","steps":["Open #general.","Click Save."]}}` + if string(b) != want { + t.Fatalf("decision record:\n got %s\nwant %s", b, want) + } +} + +// TestInteractiveEditRefusesTooManySteps: the steps prompt reads to the +// terminating blank line whatever the count, so an over-long list is reported +// against the limit rather than silently truncated at 32 with the rest of the +// operator's typing consumed as the next prompt's answer. +func TestInteractiveEditRefusesTooManySteps(t *testing.T) { + dir := ingestThree(t) + lines := []string{"e", ""} // edit, then a blank title (keep it) + for i := 0; i <= maxSteps; i++ { + lines = append(lines, fmt.Sprintf("Step %d.", i+1)) + } + lines = append(lines, "", "", "") // end the steps, then blank expected and observed + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader(strings.Join(lines, "\n") + "\n"), Out: &out, IsTTY: true, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + want := fmt.Sprintf("steps lists %d entries, exceeding the limit of %d", maxSteps+1, maxSteps) + if !strings.Contains(out.String(), want) { + t.Fatalf("expected the over-long steps refusal (%q):\n%s", want, out.String()) + } + if _, decisions, _ := Load(dir); len(decisions) != 0 { + t.Fatal("a refused edit recorded a decision") + } +} + +// TestSingleDecisionRefusesEditOutsideEdited: an edit carried alongside +// "accepted" or "rejected" would be silently discarded, so it is refused. The CLI +// refuses it at the usage status too; the check lives here as well so the refusal +// is a property of the API rather than of one caller's invariants. +func TestSingleDecisionRefusesEditOutsideEdited(t *testing.T) { + dir := ingestThree(t) + var out bytes.Buffer + err := Review(ReviewOptions{ + Dir: dir, Test: "T-001", Decision: "accepted", + EditIn: strings.NewReader(`{"title":"sneaky"}`), Out: &out, Today: "2026-09-12", + }) + if err == nil || !strings.Contains(err.Error(), "-edit applies only to -decision edited") { + t.Fatalf("got %v, want a refusal naming the flag pairing", err) + } + if _, decisions, _ := Load(dir); len(decisions) != 0 { + t.Fatal("a refused decision was recorded") + } +} diff --git a/internal/drafttests/testdata/answer.json b/internal/drafttests/testdata/answer.json new file mode 100644 index 0000000..f38c82d --- /dev/null +++ b/internal/drafttests/testdata/answer.json @@ -0,0 +1,11 @@ +{"rubric":"testimony-testdraft/v1","tests":[ + {"id":"T-001","finding":"F-001","session":"fixture-session", + "title":"Saving gives no confirmation", + "steps":["Open #general in the settings prototype.", + "Change the display name to Alice.", + "Click the Save button ([data-testid=save-btn])."], + "expected":"The save is confirmed on screen — a toast, or the button briefly disabled.", + "observed":"Nothing visibly changes, so there is no way to tell the save landed.", + "rationale_quote":"I clicked save and nothing happened", + "severity":3,"status":"accepted"} +]} diff --git a/internal/drafttests/testdata/edit.json b/internal/drafttests/testdata/edit.json new file mode 100644 index 0000000..89a17b6 --- /dev/null +++ b/internal/drafttests/testdata/edit.json @@ -0,0 +1 @@ +{"title":"Saving a display name gives no confirmation","steps":["Open #general.","Click Save."]} diff --git a/internal/drafttests/testdata/findings.jsonl b/internal/drafttests/testdata/findings.jsonl new file mode 100644 index 0000000..89b4790 --- /dev/null +++ b/internal/drafttests/testdata/findings.jsonl @@ -0,0 +1,9 @@ +{"id":"F-001","t":22,"type":"bug","severity":3,"mode":"A","quote":"I clicked save and nothing happened","evidence":["utt-004","ev-003","ev-004"],"ui":{"selector":"[data-testid=save-btn]","route":"#general"},"status":"unverified"} +{"id":"F-002","t":38,"type":"preference","severity":2,"mode":"A","quote":"I like this dark mode toggle","evidence":["utt-006","ev-006"],"status":"unverified"} +{"id":"F-003","t":48,"type":"friction","severity":2,"mode":"A","quote":"The label just says dark mode","evidence":["utt-007"],"status":"unverified"} +{"id":"F-004","t":16,"type":"preference","severity":1,"mode":"B","quote":"Now I expect this save button to confirm somehow","evidence":["utt-003"],"status":"unverified"} +{"id":"F-005","t":31,"type":"idea","severity":3,"mode":"A","quote":"The tab switch is instant","evidence":["utt-005"],"status":"unverified"} +{"kind":"verdict","finding":"F-001","verdict":"confirmed","at":"2026-09-12"} +{"kind":"verdict","finding":"F-003","verdict":"rejected","at":"2026-09-12"} +{"kind":"verdict","finding":"F-004","verdict":"confirmed","at":"2026-09-12"} +{"kind":"verdict","finding":"F-005","verdict":"duplicate","of":"F-001","at":"2026-09-12"} diff --git a/internal/drafttests/testdata/manifest.json b/internal/drafttests/testdata/manifest.json new file mode 100644 index 0000000..5829e1c --- /dev/null +++ b/internal/drafttests/testdata/manifest.json @@ -0,0 +1,11 @@ +{ + "session": "fixture-session", + "app": "testimony demo", + "commit": "walking-skeleton", + "participant": "P1", + "t0_epoch_ms": 1784300400000, + "tasks": [ + "Change your display name and save it", + "Try the appearance settings" + ] +} diff --git a/internal/drafttests/testdata/tests.md b/internal/drafttests/testdata/tests.md new file mode 100644 index 0000000..e86e3d7 --- /dev/null +++ b/internal/drafttests/testdata/tests.md @@ -0,0 +1,38 @@ +# Regression tests — sample-session + +Drafted from confirmed findings in session `sample-session` (app `testimony demo`, participant `P1`). 2 of 3 drafts accepted. + +## T-001 — Saving gives no confirmation + +- **Source:** finding `F-001` (bug, severity 3) in session `sample-session`, at [00:22] +- **Decision:** accepted (2026-09-12) + +**Steps** + +1. Open #general in the settings prototype. +2. Change the display name to Alice. +3. Click the Save button \(\[data-testid=save-btn\]\). + +**Expected:** The save is confirmed on screen — a toast, or the button briefly disabled. + +**Observed:** Nothing visibly changes, so there is no way to tell the save landed. + +**Rationale (participant, [00:22]):** “I clicked save and nothing happened” + +## T-002 — Saving a display name twice gives no confirmation either time + +- **Source:** finding `F-001` (bug, severity 3) in session `sample-session`, at [00:22] +- **Decision:** edited (2026-09-12) + +**Steps** + +1. Open #general in the settings prototype. +2. Change the display name to Alice. +3. Click the Save button \(\[data-testid=save-btn\]\). +4. Click the Save button a second time. + +**Expected:** The second click is either confirmed or visibly a no-op. + +**Observed:** Both clicks look the same, so the only way to check is to click again. + +**Rationale (participant, [00:22]):** “I clicked save and nothing happened” diff --git a/internal/drafttests/testdata/timeline.jsonl b/internal/drafttests/testdata/timeline.jsonl new file mode 100644 index 0000000..32d1bf9 --- /dev/null +++ b/internal/drafttests/testdata/timeline.jsonl @@ -0,0 +1,13 @@ +{"t":2,"src":"speech","id":"utt-001","payload":{"speaker":"P1","t1":5.5,"text":"Session start. I'm looking at the settings prototype for the first time."}} +{"t":8,"src":"speech","id":"utt-002","payload":{"speaker":"P1","t1":12.5,"text":"Okay, three tabs, General is open. I'll change my display name first."}} +{"t":9.5,"src":"event","id":"ev-001","payload":{"kind":"click","route":"#general","selector":"[data-testid=display-name]","text":"Display name"}} +{"t":14.8,"src":"event","id":"ev-002","payload":{"kind":"input","route":"#general","selector":"[data-testid=display-name]","value":"Alice"}} +{"t":16,"src":"speech","id":"utt-003","payload":{"speaker":"P1","t1":21,"text":"Typing feels fine. Now I expect this save button to confirm somehow."}} +{"t":19.2,"src":"event","id":"ev-003","payload":{"kind":"click","route":"#general","selector":"[data-testid=save-btn]","text":"Save"}} +{"t":22,"src":"speech","id":"utt-004","payload":{"speaker":"P1","t1":28,"text":"Hm. I clicked save and nothing happened. No message, no spinner."}} +{"t":24.1,"src":"event","id":"ev-004","payload":{"kind":"click","route":"#general","selector":"[data-testid=save-btn]","text":"Save"}} +{"t":31,"src":"speech","id":"utt-005","payload":{"speaker":"P1","t1":36,"text":"I'll switch to Appearance. The tab switch is instant, that's nice."}} +{"t":32,"src":"event","id":"ev-005","payload":{"kind":"click","route":"#general","selector":"[data-testid=tab-appearance]","text":"Appearance"}} +{"t":38,"src":"speech","id":"utt-006","payload":{"speaker":"P1","t1":45,"text":"Oh, I like this dark mode toggle. Immediate feedback."}} +{"t":40.5,"src":"event","id":"ev-006","payload":{"kind":"click","route":"#appearance","selector":"[data-testid=theme-toggle]","text":"Dark mode"}} +{"t":48,"src":"speech","id":"utt-007","payload":{"speaker":"P1","t1":54,"text":"Turning it back. The label just says dark mode whether it's on or off."}} diff --git a/internal/drafttests/window.go b/internal/drafttests/window.go new file mode 100644 index 0000000..0823564 --- /dev/null +++ b/internal/drafttests/window.go @@ -0,0 +1,83 @@ +package drafttests + +import ( + "sort" + + "github.com/REPPL/Testimony/internal/analyze" + "github.com/REPPL/Testimony/internal/session" + "github.com/REPPL/Testimony/internal/timeline" +) + +// Window returns the timeline entries around f, in time order: every entry whose +// time falls in [lo, hi], where lo and hi span f's cited evidence entries +// widened by window on both sides. +// +// The reproduction steps are the expensive part of a regression test, and this +// window is the only thing the drafting model is allowed to reconstruct them +// from — so it must hold the lead-up and the aftermath, not merely the moment. +// lo is the earliest cited entry's start minus window; hi is the latest cited +// entry's end (an utterance's t1, via timeline.SpeechEnd) plus window. Evidence +// ids are matched in their session.SafeText form, the form the request shows and +// the form analyze already validates against, so an id carrying a stripped byte +// resolves here exactly as it does there. +// +// Speech and event entries are both included: the utterances around the moment +// are what carry the *expected* behaviour, and the events are what carry the +// steps. Bounds are inclusive. +// +// A finding whose evidence resolves to no entry at all — impossible after +// analyze -ingest, reachable via a hand-edited findings.jsonl — falls back to +// [f.T - window, f.T + window], so the finding still arrives with context rather +// than with nothing. +// +// A negative window is legitimate: it narrows the span (and can empty it), the +// same latitude report's join window allows. The result is not separately +// capped, because its size is bounded by timeline.jsonl, which already carries +// the session's total-size limit. +func Window(entries []timeline.Entry, f analyze.Finding, window float64) []timeline.Entry { + cited := make(map[string]bool, len(f.Evidence)) + for _, id := range f.Evidence { + if s := session.SafeText(id); s != "" { + cited[s] = true + } + } + + var lo, hi float64 + found := false + for _, e := range entries { + if !cited[session.SafeText(e.ID)] { + continue + } + end := timeline.SpeechEnd(e) + if !found { + lo, hi, found = e.T, end, true + continue + } + if e.T < lo { + lo = e.T + } + if end > hi { + hi = end + } + } + if !found { + lo, hi = f.T, f.T + } + lo -= window + hi += window + + var out []timeline.Entry + for _, e := range entries { + if e.T >= lo && e.T <= hi { + out = append(out, e) + } + } + // Ordered here rather than assumed of the caller: the window is presented to + // the model as the sequence to reconstruct steps from, so an out-of-order + // timeline.jsonl — which reaches these readers directly when a session is + // hand-edited or exchanged — must not hand it a repro in the wrong order. + // SliceStable with the same less function timeline.Merge uses leaves a + // merge-produced timeline untouched. + sort.SliceStable(out, func(i, j int) bool { return out[i].T < out[j].T }) + return out +} diff --git a/internal/review/review.go b/internal/review/review.go index b698d24..7358e61 100644 --- a/internal/review/review.go +++ b/internal/review/review.go @@ -1,10 +1,15 @@ -// Package review records human verdicts on candidate findings. A verdict is -// appended to findings.jsonl as a separate, non-destructive record (never an -// in-place rewrite of the finding), so the finding's birth state and the full -// verdict history survive as the precision measure the method stands on -// (architecture note §2; itd-2 press release). Interactive review is gated on -// stdin being a character device so a redirected or piped run (CI) never -// blocks; a single verdict can also be recorded non-interactively. +// Package review is the pipeline's one human-decision surface, across both +// record families. Its own half records verdicts on candidate findings: a +// verdict is appended to findings.jsonl as a separate, non-destructive record +// (never an in-place rewrite of the finding), so the finding's birth state and +// the full verdict history survive as the precision measure the method stands on +// (architecture note §2; itd-2 press release). Options.Kind dispatches the tests +// half to internal/drafttests, which records an accept / edit / reject decision +// on each drafted regression test the same appended way — one verb for the whole +// pipeline, with the vocabularies kept per-kind because "edited" carries a +// payload no verdict ever does (ADR 0001). Interactive review is gated on stdin +// being a character device so a redirected or piped run (CI) never blocks; a +// single decision can also be recorded non-interactively on either side. package review import ( @@ -19,9 +24,9 @@ import ( "path/filepath" "sort" "strings" - "syscall" "github.com/REPPL/Testimony/internal/analyze" + "github.com/REPPL/Testimony/internal/drafttests" "github.com/REPPL/Testimony/internal/session" ) @@ -31,21 +36,77 @@ import ( // range on an attacker-authored findings.jsonl time. const maxClockSeconds = 1e9 -// Options configures a review run. +// The record families review can walk. An empty Kind means KindFindings, so a +// caller that predates the tests side keeps the findings behaviour unchanged. +const ( + KindFindings = "findings" + KindTests = "tests" +) + +// ParseKindFlag validates a -kind flag value against the closed set, so the CLI +// can refuse an unknown record family as a wrong invocation rather than let it +// reach a dispatch that has no case for it. An empty value is the documented +// default. +func ParseKindFlag(s string) (string, error) { + switch s { + case "", KindFindings: + return KindFindings, nil + case KindTests: + return KindTests, nil + } + return "", fmt.Errorf("invalid kind %q (want findings|tests)", s) +} + +// Options configures a review run. The Finding/Verdict pair belongs to +// KindFindings and the Test/Decision/EditIn set to KindTests; the CLI refuses a +// flag from the other family at the usage status, and Run refuses it too, so the +// pairing is a property of this API rather than of one caller's invariants. type Options struct { - Dir string // session directory - Finding string // non-interactive: the finding to judge (F-NNN) - Verdict string // non-interactive: confirmed | rejected | duplicate-of-F-NNN - In io.Reader // interactive input - Out io.Writer // status and prompts - IsTTY bool // whether In is an interactive terminal - Today string // ISO date stamped onto verdicts (YYYY-MM-DD) + Dir string // session directory + Kind string // record family: "findings" (the default) or "tests" + Finding string // non-interactive: the finding to judge (F-NNN) + Verdict string // non-interactive: confirmed | rejected | duplicate-of-F-NNN + Test string // non-interactive, -kind tests: the draft to decide (T-NNN) + Decision string // non-interactive, -kind tests: accepted | edited | rejected + EditIn io.Reader // -kind tests, with Decision "edited": the replacement fields as a JSON object + In io.Reader // interactive input + Out io.Writer // status and prompts + IsTTY bool // whether In is an interactive terminal + Today string // ISO date stamped onto verdicts and decisions (YYYY-MM-DD) } -// Run records verdicts for the session. With -finding/-verdict it records one -// verdict non-interactively; otherwise it walks the unverified findings +// Run records human decisions for the session. With -kind tests it delegates to +// the drafting layer's walk; otherwise, with -finding/-verdict it records one +// verdict non-interactively, and with neither it walks the unverified findings // interactively (skipping cleanly when stdin is not a terminal). func Run(opts Options) error { + kind, err := ParseKindFlag(opts.Kind) + if err != nil { + return err + } + // A flag belonging to the other record family is a wrong invocation, not a + // silently ignored one: a caller who typed -verdict against -kind tests meant + // something this walk cannot do, and recording nothing while exiting 0 would + // let a script believe the decision landed. + if kind == KindTests { + if opts.Finding != "" || opts.Verdict != "" { + return fmt.Errorf("-finding and -verdict apply to -kind findings, not -kind tests") + } + return drafttests.Review(drafttests.ReviewOptions{ + Dir: opts.Dir, + Test: opts.Test, + Decision: opts.Decision, + EditIn: opts.EditIn, + In: opts.In, + Out: opts.Out, + IsTTY: opts.IsTTY, + Today: opts.Today, + }) + } + if opts.Test != "" || opts.Decision != "" || opts.EditIn != nil { + return fmt.Errorf("-test, -decision and -edit apply to -kind tests, not -kind findings") + } + // analyze.Load reads dir/findings.jsonl, so a session directory that does // not exist at all satisfies fs.ErrNotExist exactly like one that exists // but has simply never been through `analyze -ingest` — and review, @@ -291,114 +352,60 @@ func ParseVerdictFlag(s string) (verdict, of string, err error) { // AppendVerdict appends one verdict record to findings.jsonl without touching // any existing line (append-only; latest verdict wins for display). // +// The dangerous part of the write — the no-follow open, the exclusive lock, the +// two size pre-flights, the newline framing over an unterminated last line, the +// partial-write rollback, and returning the Close error — lives once in +// session.AppendRecord, shared with the decision records tests.jsonl holds. This +// function supplies only the vocabulary: the verdict's encoding, the labels the +// two size errors name it by, and the target re-check below. +// // expect, when non-nil, is the finding the analyst was shown when they made this -// decision. AppendVerdict re-reads the current findings under its lock and -// refuses if the targeted id is gone or now names a different finding — see -// verifyTarget. Callers pass nil only when there is no snapshot to bind against -// (there are none in production; the review paths always pass the judged -// finding). +// decision. session.AppendRecord runs the Verify closure over the current +// findings under its lock, and it refuses if the targeted id is gone or now names +// a different finding — see verifyTarget. Callers pass nil only when there is no +// snapshot to bind against (there are none in production; the review paths always +// pass the judged finding). func AppendVerdict(dir string, v analyze.Verdict, expect *analyze.Finding) error { b, err := json.Marshal(v) if err != nil { return err } - // Hold the verdict to MaxJSONLLine, the shared read-side invariant every other - // JSONL writer respects (session.WriteJSONL, analyze.oversizedFindings, - // demo.tooLongForJSONL). A verdict carries its finding id verbatim, and in an - // exchanged or hand-edited findings.jsonl that id can be just under the 4 MiB - // scanner cap — small enough that the finding line loads, large enough that the - // verdict's own framing tips the line over it. Appending it would durably brick - // the verdict history this package exists to protect: every later analyze.Load, - // review, report, and holdsVerdicts would fail with "token too long". - if len(b)+1 > session.MaxJSONLLine { - return fmt.Errorf("verdict for %s encodes to %d bytes, over the %d-byte %s line limit", - session.SafeText(v.Finding), len(b)+1, session.MaxJSONLLine, session.FindingsFile) - } - path := filepath.Join(dir, session.FindingsFile) - // O_RDWR rather than O_WRONLY because the record cannot be framed correctly - // without first reading the byte already at the end of the file. - f, err := session.OpenFileNoFollow(path, os.O_APPEND|os.O_RDWR, 0o644) - if err != nil { - return err - } - // Take an exclusive advisory lock across the probe → write → rollback sequence. - // Two `testimony review` processes appending to one session's findings.jsonl - // would otherwise race: A measures the end, B appends a full verdict past it, A's - // write fails part-way (ENOSPC — the case the rollback exists for), and A's - // Truncate then cuts the file back below B's committed record, deleting it. - // writeVerdict re-measuring the end only shrinks that window; the lock closes it, - // so the length A rolls back to is the true end before A's own bytes. The lock - // releases with the descriptor on Close. - if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { - f.Close() - return err + a := session.Append{ + Path: filepath.Join(dir, session.FindingsFile), + Record: b, + // The finding id is attacker-authorable in an exchanged session and the + // label reaches the operator's terminal through cli.fail, so it is + // sanitised here rather than inside the shared primitive, which never sees + // an id as such. + Label: "verdict for " + session.SafeText(v.Finding), + Kind: "verdict", } - // Hold the file to MaxJSONLBytes, the total-size invariant ParseRecords - // enforces and every other JSONL writer (session.WriteJSONL, - // analyze.oversizedFindings) already pre-flights before appending. A - // findings.jsonl built by analyze -ingest can legally sit right at the cap; - // without this check, the next verdict recorded against it would land findings - // past MaxJSONLBytes, and every later analyze.Load, review, and report would - // refuse it — including the verdict just appended, and any recorded before it. - // holdsVerdicts (analyze.Ingest's re-ingest guard) is the one reader that does - // NOT refuse an over-total file: it bounds only a per-line scan, not the total, - // so it still finds the verdict and blocks re-ingest as a repair path. - // Measured under the lock, after the file is open, so a - // concurrent append cannot land between this check and the write below. writeVerdict - // is called with append(b, '\n') below — len(b)+1 bytes — but writeVerdict itself - // prepends a second leading newline when the file is non-empty and its last byte - // is not already '\n' (an exchanged or hand-edited findings.jsonl can end - // unterminated), so the worst case it actually writes is len(b)+2, not len(b)+1; - // budgeting only +1 here let a file at exactly the cap minus (len(b)+1) pass this - // check and still land one byte over MaxJSONLBytes. - info, err := f.Stat() - if err != nil { - f.Close() - return err - } - if info.Size()+int64(len(b))+2 > session.MaxJSONLBytes { - f.Close() - return fmt.Errorf("%s is %d bytes; appending this verdict would push it past the %d-byte JSONL file limit; refusing to write a session no command could read back", - session.FindingsFile, info.Size(), session.MaxJSONLBytes) - } - // Under the lock, confirm the verdict still targets the finding the analyst - // judged. review.Run snapshots findings once (analyze.Load) and then blocks on - // the operator for the whole interactive walk; a concurrent `analyze -ingest` - // can truncate-and-rewrite findings.jsonl in that gap — permitted until the - // first verdict exists — and because finding ids restart at F-001 the verdict - // would otherwise attach to a different finding under the same id, silently - // misattributing the human decision this file exists to hold. The re-check runs - // under the same exclusive lock analyze.commitFindings takes, so the re-ingest - // is either already visible here (mismatch → refuse, no verdict written) or - // serialised after this append and then blocked by its own verdict-guard. if expect != nil { - if err := verifyTarget(f, v, *expect); err != nil { - f.Close() - return err - } - } - if err := writeVerdict(f, append(b, '\n')); err != nil { - f.Close() - return err - } - // Return the Close error so a verdict is never reported recorded when its - // bytes did not reach disk (write-back deferred to close on NFS/full device). - return f.Close() + // Confirm, under the append lock, that the verdict still targets the finding + // the analyst judged. review.Run snapshots findings once (analyze.Load) and + // then blocks on the operator for the whole interactive walk; a concurrent + // `analyze -ingest` can truncate-and-rewrite findings.jsonl in that gap — + // permitted until the first verdict exists — and because finding ids restart + // at F-001 the verdict would otherwise attach to a different finding under + // the same id, silently misattributing the human decision this file exists to + // hold. The re-check runs under the same exclusive lock the commit side + // takes, so the re-ingest is either already visible here (mismatch → refuse, + // no verdict written) or serialised after this append and then blocked by its + // own verdict-guard. + judged := *expect + a.Verify = func(current io.Reader) error { return verifyTarget(current, v, judged) } + } + return session.AppendRecord(a) } -// verifyTarget re-reads the findings currently in f (the locked findings.jsonl -// descriptor) and confirms the verdict v still applies to the finding expect — -// the one the analyst was shown. It refuses if the id has vanished or now names -// a different finding, and for a duplicate verdict if the "of" target has -// vanished. f is read via a SectionReader over ReadAt, which leaves the file -// offset untouched, and the descriptor is O_APPEND, so the subsequent write -// still lands at the true end of file. -func verifyTarget(f *os.File, v analyze.Verdict, expect analyze.Finding) error { - info, err := f.Stat() - if err != nil { - return err - } - findings, _, err := analyze.ParseRecords(io.NewSectionReader(f, 0, info.Size()), session.FindingsFile) +// verifyTarget re-reads the findings currently in the locked findings.jsonl +// (session.AppendRecord hands it a reader over the file's contents, taken under +// the append lock) and confirms the verdict v still applies to the finding +// expect — the one the analyst was shown. It refuses if the id has vanished or +// now names a different finding, and for a duplicate verdict if the "of" target +// has vanished. +func verifyTarget(current io.Reader, v analyze.Verdict, expect analyze.Finding) error { + findings, _, err := analyze.ParseRecords(current, session.FindingsFile) if err != nil { return err } @@ -418,67 +425,6 @@ func verifyTarget(f *os.File, v analyze.Verdict, expect analyze.Finding) error { return nil } -// verdictFile is the subset of *os.File that writeVerdict needs; a fake -// satisfies it in tests to exercise the partial-write rollback. -type verdictFile interface { - io.Writer - io.ReaderAt - Seek(offset int64, whence int) (int64, error) - Truncate(size int64) error -} - -// writeVerdict frames rec so it lands as its own physical line and writes it, -// rolling the file back if the write only partly lands. -// -// A findings.jsonl need not end in a newline: it may have been hand edited, -// produced by another tool, or left short by a crash part-way through an -// earlier write. Appending blindly would fuse the verdict onto that -// unterminated final line, producing one physical line holding two JSON -// objects — which makes not just those two records but the entire file -// unparseable to every reader. So probe the last byte and open a fresh line -// when it is not already one. (O_APPEND still puts the write at the end, -// whatever the seek offset; the seek is only to learn the size.) -// -// The write itself needs the same rollback the capture path has in -// demo.appendRecords: os.File.Write gives no atomicity guarantee, so a full -// disk fills the remaining space, returns a short count, and leaves a -// truncated, newline-less fragment (e.g. `{"kind":"verdict","find`) behind. -// That fragment would fuse with the next successful write into one malformed -// physical line and make the whole findings.jsonl — the human verdict record -// this package exists to protect — unparseable to every reader. So on any -// write error the file is truncated back to the length it had immediately -// before the write. The size is re-measured at that point rather than reusing -// the offset the newline probe learned: the descriptor is O_APPEND, so the -// write lands at the true end of file, which a concurrent appender (a second -// `testimony review`, or an analyze ingest) may have moved on since the probe. -// Truncating to the stale offset would delete that other writer's record -// instead of only our own partial bytes. -func writeVerdict(f verdictFile, rec []byte) error { - end, err := f.Seek(0, io.SeekEnd) - if err != nil { - return err - } - if end > 0 { - var last [1]byte - if _, err := f.ReadAt(last[:], end-1); err != nil { - return err - } - if last[0] != '\n' { - rec = append([]byte{'\n'}, rec...) - } - } - before, err := f.Seek(0, io.SeekEnd) - if err != nil { - return err - } - if _, err := f.Write(rec); err != nil { - // Best-effort roll back any partial bytes; surface the original error. - f.Truncate(before) - return err - } - return nil -} - // printFinding writes a finding to the analyst's terminal. Every // attacker-influenceable field (a finding in a downloaded session is untrusted) // is passed through session.SafeText first, so embedded ESC/ANSI or control diff --git a/internal/review/review_test.go b/internal/review/review_test.go index f80ce1a..9c68b8e 100644 --- a/internal/review/review_test.go +++ b/internal/review/review_test.go @@ -3,7 +3,6 @@ package review import ( "bytes" "encoding/json" - "errors" "io" "math" "os" @@ -638,93 +637,6 @@ func TestReviewErrorPathsSanitiseFindingIDs(t *testing.T) { }) } -// shortWriteFile is a verdictFile whose Write persists a prefix and then errors, -// standing in for a full disk (write(2) fills the remaining space, returns a -// short count, and the next write returns ENOSPC — os.File.Write persists the -// truncated prefix before returning the error). Seek always reports the current -// length, so it also stands in for the O_APPEND descriptor writeVerdict holds. -type shortWriteFile struct { - buf []byte - fail bool // when true, Write keeps only a prefix then returns an error -} - -func (f *shortWriteFile) Seek(offset int64, whence int) (int64, error) { - return int64(len(f.buf)), nil -} - -func (f *shortWriteFile) ReadAt(p []byte, off int64) (int, error) { - if off < 0 || off >= int64(len(f.buf)) { - return 0, io.EOF - } - return copy(p, f.buf[off:]), nil -} - -func (f *shortWriteFile) Truncate(size int64) error { - f.buf = f.buf[:size] - return nil -} - -func (f *shortWriteFile) Write(p []byte) (int, error) { - if f.fail { - half := len(p) / 2 - f.buf = append(f.buf, p[:half]...) - return half, errors.New("no space left on device") - } - f.buf = append(f.buf, p...) - return len(p), nil -} - -// TestWriteVerdictRollsBackPartialWrite is the ENOSPC regression on the verdict -// path: a short write that persists a newline-less prefix must be truncated -// away, so findings.jsonl never retains a partial line that would fuse with the -// next verdict into one malformed physical record and make the whole file — the -// human verdict record the package exists to protect — unparseable to every -// reader. Pre-fix AppendVerdict wrote directly with no rollback, so the prefix -// survived, exactly as demo.appendLines did before its own fix. -func TestWriteVerdictRollsBackPartialWrite(t *testing.T) { - f := &shortWriteFile{} - first, err := json.Marshal(analyze.Verdict{Kind: "verdict", Finding: "F-001", Verdict: "confirmed", At: "2026-07-17"}) - if err != nil { - t.Fatalf("marshal: %v", err) - } - if err := writeVerdict(f, append(first, '\n')); err != nil { - t.Fatalf("first verdict: %v", err) - } - good := string(f.buf) - - f.fail = true - second, err := json.Marshal(analyze.Verdict{Kind: "verdict", Finding: "F-002", Verdict: "rejected", At: "2026-07-17"}) - if err != nil { - t.Fatalf("marshal: %v", err) - } - if err := writeVerdict(f, append(second, '\n')); err == nil { - t.Fatalf("expected a write error on a full disk") - } - if string(f.buf) != good { - t.Fatalf("partial line survived: file is %q, want the clean prefix %q", f.buf, good) - } - if !strings.HasSuffix(string(f.buf), "\n") { - t.Fatalf("file does not end on a newline: %q", f.buf) - } - - // The rolled-back file still parses one record per line, so a later verdict - // lands cleanly rather than fusing onto a fragment. - f.fail = false - if err := writeVerdict(f, append(second, '\n')); err != nil { - t.Fatalf("verdict after rollback: %v", err) - } - lines := strings.Split(strings.TrimRight(string(f.buf), "\n"), "\n") - if len(lines) != 2 { - t.Fatalf("got %d lines, want 2: %q", len(lines), f.buf) - } - for i, l := range lines { - var v analyze.Verdict - if err := json.Unmarshal([]byte(l), &v); err != nil { - t.Fatalf("line %d is not one JSON record: %v (%q)", i+1, err, l) - } - } -} - // TestAppendVerdictRefusesSymlink is the arbitrary-file-append regression: a // findings.jsonl planted as a symlink must not be followed. func TestAppendVerdictRefusesSymlink(t *testing.T) { @@ -1008,3 +920,93 @@ func TestAppendVerdictAcceptsUnchangedFinding(t *testing.T) { t.Fatalf("verdict not recorded as expected: %+v", verdicts) } } + +// --- the -kind dispatch ---------------------------------------------------- + +func TestParseKindFlag(t *testing.T) { + cases := []struct { + in, want string + wantErr bool + }{ + {"", KindFindings, false}, + {"findings", KindFindings, false}, + {"tests", KindTests, false}, + {"Findings", "", true}, + {"verdicts", "", true}, + } + for _, tc := range cases { + got, err := ParseKindFlag(tc.in) + if (err != nil) != tc.wantErr { + t.Fatalf("%q: err=%v wantErr=%v", tc.in, err, tc.wantErr) + } + if err == nil && got != tc.want { + t.Fatalf("%q: got %q, want %q", tc.in, got, tc.want) + } + } +} + +const draftFixture = `{"id":"T-001","finding":"F-001","session":"s","title":"Saving gives no confirmation","steps":["Open #general.","Click Save."],"expected":"The save is confirmed.","observed":"Nothing happens.","rationale_quote":"I clicked save and nothing happened","severity":3,"status":"proposed"} +` + +// TestRunDispatchesKindTests: one verb serves both record families, so a +// -kind tests run reaches the drafting layer's decision path and appends its +// record to tests.jsonl rather than findings.jsonl. +func TestRunDispatchesKindTests(t *testing.T) { + dir := writeSession(t) + if err := os.WriteFile(filepath.Join(dir, session.TestsFile), []byte(draftFixture), 0o644); err != nil { + t.Fatalf("write tests: %v", err) + } + before := findingLines(t, dir) + + var out bytes.Buffer + err := Run(Options{Dir: dir, Kind: KindTests, Test: "T-001", Decision: "accepted", Out: &out, Today: "2026-09-12"}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if got := out.String(); got != "recorded: T-001 accepted (2026-09-12)\n" { + t.Fatalf("echo = %q", got) + } + b, rerr := os.ReadFile(filepath.Join(dir, session.TestsFile)) + if rerr != nil { + t.Fatalf("read tests: %v", rerr) + } + if !strings.Contains(string(b), `{"kind":"decision","test":"T-001","decision":"accepted","at":"2026-09-12"}`) { + t.Fatalf("the decision did not reach tests.jsonl:\n%s", b) + } + if !strings.HasPrefix(string(b), draftFixture) { + t.Fatalf("the draft line was not preserved byte-for-byte:\n%s", b) + } + // findings.jsonl is a different record family and stays untouched. + if strings.Join(before, "\n") != strings.Join(findingLines(t, dir), "\n") { + t.Fatal("a tests-side decision modified findings.jsonl") + } +} + +// TestRunRefusesCrossFamilyFlags: a flag belonging to the other record family is +// a wrong invocation, not a silently ignored one — recording nothing while +// exiting 0 would let a script believe the decision landed. +func TestRunRefusesCrossFamilyFlags(t *testing.T) { + cases := []struct { + name string + opts Options + want string + }{ + {"verdict with kind tests", Options{Kind: KindTests, Finding: "F-001", Verdict: "confirmed"}, "-finding and -verdict apply to -kind findings"}, + {"test with kind findings", Options{Test: "T-001", Decision: "accepted"}, "-test, -decision and -edit apply to -kind tests"}, + {"edit with kind findings", Options{EditIn: strings.NewReader("{}")}, "-test, -decision and -edit apply to -kind tests"}, + {"unknown kind", Options{Kind: "verdicts"}, "invalid kind"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := writeSession(t) + opts := tc.opts + opts.Dir = dir + opts.Out = io.Discard + opts.Today = "2026-09-12" + err := Run(opts) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("got %v, want an error containing %q", err, tc.want) + } + }) + } +} diff --git a/internal/session/records.go b/internal/session/records.go new file mode 100644 index 0000000..0e9bbde --- /dev/null +++ b/internal/session/records.go @@ -0,0 +1,275 @@ +package session + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "syscall" +) + +// Append is one record appended to a session JSONL file. +// +// Two session artefacts hold a machine record plus appended human records — +// findings.jsonl (findings plus verdicts) and tests.jsonl (test drafts plus +// decisions) — and both are appended to under the same hazards: a planted +// symlink or FIFO at the path, a concurrent writer racing the measure-then-write +// sequence, an unterminated last line that a blind append would fuse onto, a +// short write leaving a newline-less fragment, and the read-side size invariants +// every reader scans to. AppendRecord holds that logic once so the two callers +// cannot drift apart; the vocabulary of each record family stays with its own +// package. +type Append struct { + Path string // the file to append to + Record []byte // the encoded record, without its newline + Label string // names the record in the line-limit error, e.g. "verdict for F-001" + Kind string // names the record kind in the file-limit error, e.g. "verdict" + Verify func(current io.Reader) error // optional re-check of the file's contents, run under the lock +} + +// AppendRecord appends a.Record as its own physical line without touching any +// existing line. +// +// It opens a.Path under the no-follow guard (O_APPEND|O_RDWR, so a planted +// symlink or non-regular file is refused rather than followed or blocked on) and +// takes an exclusive advisory lock across the whole probe → verify → write → +// rollback sequence. Two appenders to one session's file would otherwise race: A +// measures the end, B appends a full record past it, A's write fails part-way +// (ENOSPC — the case the rollback exists for), and A's truncate then cuts the +// file back below B's committed record, deleting it. The lock closes that +// window, so the length A rolls back to is the true end before A's own bytes. +// +// It pre-flights the record against MaxJSONLLine before the open (an unwritable +// record is a fact about the record alone) and, under the lock, the file against +// MaxJSONLBytes — the two read-side invariants every reader enforces, so +// a record is never persisted that would durably brick the file it lands in — +// runs a.Verify over the file's current contents (see Verify's own comment), +// frames the record with a leading newline when the file does not already end in +// one, writes it, truncates back to the pre-write length on a short write, and +// returns the Close error so a record is never reported written when its bytes +// did not reach disk (write-back deferred to close on NFS or a full device). +// +// a.Label and a.Kind name the record in the two size errors, so each caller's +// message reads in its own vocabulary; the file is named by a.Path's base. +func AppendRecord(a Append) error { + name := filepath.Base(a.Path) + // Hold the record to MaxJSONLLine, the shared read-side invariant every JSONL + // writer respects. A record carries caller-supplied text verbatim, and in an + // exchanged or hand-edited file that text can be just under the scanner cap — + // small enough that the existing lines load, large enough that this record's + // own framing tips its line over it. Appending it would durably brick the + // human-decision history these files exist to protect: every later load would + // fail with "token too long". + // + // Checked before the open, not after it: an unwritable record is a fact about + // the record alone, so the refusal must name it whether or not the file exists + // yet. Ordering it after the open reported a missing file — and its joined + // filesystem path — in place of the over-limit refusal the caller can act on. + if len(a.Record)+1 > MaxJSONLLine { + return fmt.Errorf("%s encodes to %d bytes, over the %d-byte %s line limit", + a.Label, len(a.Record)+1, MaxJSONLLine, name) + } + // O_RDWR rather than O_WRONLY because the record cannot be framed correctly + // without first reading the byte already at the end of the file. + f, err := OpenFileNoFollow(a.Path, os.O_APPEND|os.O_RDWR, 0o644) + if err != nil { + return err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + f.Close() + return err + } + // Hold the file to MaxJSONLBytes, the total-size invariant the readers + // enforce. A file written by an ingest step can legally sit right at the cap; + // without this check the next appended record would land it past the cap, and + // every later load would refuse it — including the record just appended, and + // any recorded before it. Measured under the lock, after the file is open, so + // a concurrent append cannot land between this check and the write below. The + // budget is len(Record)+2, not +1: writeRecord prepends a second leading + // newline when the file is non-empty and its last byte is not already one (an + // exchanged or hand-edited file can end unterminated), so budgeting only +1 + // let a file at exactly the cap minus (len+1) pass and still land one byte + // over. + info, err := f.Stat() + if err != nil { + f.Close() + return err + } + if info.Size()+int64(len(a.Record))+2 > MaxJSONLBytes { + f.Close() + return fmt.Errorf("%s is %d bytes; appending this %s would push it past the %d-byte JSONL file limit; refusing to write a session no command could read back", + name, info.Size(), a.Kind, MaxJSONLBytes) + } + // Verify re-reads the file the caller is about to append to, under the same + // lock, so the record is still the one the operator decided on: a caller that + // snapshots the file and then blocks on a human can have a concurrent + // truncate-and-rewrite slide different content under the same ids in that gap. + // The reader is a SectionReader over ReadAt, which leaves the file offset + // untouched, and the descriptor is O_APPEND, so the write below still lands at + // the true end of file. + if a.Verify != nil { + if err := a.Verify(io.NewSectionReader(f, 0, info.Size())); err != nil { + f.Close() + return err + } + } + rec := make([]byte, 0, len(a.Record)+2) + rec = append(rec, a.Record...) + rec = append(rec, '\n') + if err := writeRecord(f, rec); err != nil { + f.Close() + return err + } + return f.Close() +} + +// appendFile is the subset of *os.File that writeRecord needs; a fake satisfies +// it in tests to exercise the partial-write rollback. +type appendFile interface { + io.Writer + io.ReaderAt + Seek(offset int64, whence int) (int64, error) + Truncate(size int64) error +} + +// writeRecord frames rec so it lands as its own physical line and writes it, +// rolling the file back if the write only partly lands. +// +// A session JSONL file need not end in a newline: it may have been hand edited, +// produced by another tool, or left short by a crash part-way through an earlier +// write. Appending blindly would fuse the record onto that unterminated final +// line, producing one physical line holding two JSON objects — which makes not +// just those two records but the entire file unparseable to every reader. So +// probe the last byte and open a fresh line when it is not already one. +// (O_APPEND still puts the write at the end, whatever the seek offset; the seek +// is only to learn the size.) +// +// The write itself needs a rollback: os.File.Write gives no atomicity guarantee, +// so a full disk fills the remaining space, returns a short count, and leaves a +// truncated, newline-less fragment behind. That fragment would fuse with the next +// successful write into one malformed physical line and make the whole file +// unparseable. So on any write error the file is truncated back to the length it +// had immediately before the write. The size is re-measured at that point rather +// than reusing the offset the newline probe learned: the descriptor is O_APPEND, +// so the write lands at the true end of file, which a concurrent appender may +// have moved on since the probe. Truncating to the stale offset would delete that +// other writer's record instead of only our own partial bytes. +func writeRecord(f appendFile, rec []byte) error { + end, err := f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + if end > 0 { + var last [1]byte + if _, err := f.ReadAt(last[:], end-1); err != nil { + return err + } + if last[0] != '\n' { + rec = append([]byte{'\n'}, rec...) + } + } + before, err := f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + if _, err := f.Write(rec); err != nil { + // Best-effort roll back any partial bytes; surface the original error. + f.Truncate(before) + return err + } + return nil +} + +// Commit is a whole-file replacement of a session JSONL file. +type Commit struct { + Path string // the file to replace + Records [][]byte // each encoded record, without its newline + Guard func(current io.Reader) error // refuse the replacement by returning an error +} + +// CommitRecords replaces Path's contents with Records under the same no-follow +// guard and exclusive lock AppendRecord takes. +// +// Guard runs over the file's current contents before anything is truncated, and +// returning an error from it refuses the whole replacement: that is where a +// caller protects an appended human record from a re-ingest. Probing in one open +// and rewriting in another left a TOCTOU window — a concurrent appender commits +// a record (under AppendRecord's own lock) between the probe and the truncate, +// and the rewrite destroys it, precisely the record the guard exists to protect. +// Holding one exclusive lock across probe, truncate, and write forecloses the +// interleaving: an appender blocks until the commit completes, so its record is +// either visible to the guard (and the commit refused) or appended after the new +// contents. +// +// The whole set is encoded into one buffer before the file is truncated, so the +// truncate and the write are a single Write of pre-built bytes rather than a +// streamed series that a mid-way I/O error (ENOSPC) could leave half-flushed. +// That matters because the truncate has already destroyed the prior bytes: +// without the rollback a short write leaves a truncated, newline-less JSON +// fragment, which not only breaks every reader but blocks the tool's own +// recovery — the next ingest's guard parses every line and errors on the fragment +// before it can conclude the file is free to rewrite. On any write error the file +// is therefore truncated back to empty: an empty JSONL file is parseable (zero +// records) and re-ingestable, so the failure state does not foreclose its own +// repair. The Close error is returned for the same reason AppendRecord returns +// it. Callers pre-flight Records against MaxJSONLLine and MaxJSONLBytes before +// calling, because only they can name a record by its own id or its position in +// an answer. +func CommitRecords(c Commit) error { + f, err := OpenFileNoFollow(c.Path, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + f.Close() + return err + } + if c.Guard != nil { + if err := c.Guard(f); err != nil { + f.Close() + return err + } + } + if err := writeRecords(f, c.Records); err != nil { + f.Close() + return fmt.Errorf("write %s: %w", filepath.Base(c.Path), err) + } + // Close releases the lock with the descriptor. + if err := f.Close(); err != nil { + return fmt.Errorf("write %s: %w", filepath.Base(c.Path), err) + } + return nil +} + +// commitFile is the subset of *os.File that writeRecords needs; a fake satisfies +// it in tests to exercise the truncate-then-write rollback. +type commitFile interface { + io.Writer + Truncate(size int64) error + Seek(offset int64, whence int) (int64, error) +} + +// writeRecords replaces f's contents with records, one per line, rolling the file +// back to empty if the write only partly lands. See CommitRecords for why the set +// is buffered whole and why the failure state is an empty file. +func writeRecords(f commitFile, records [][]byte) error { + var buf bytes.Buffer + for _, r := range records { + buf.Write(r) + buf.WriteByte('\n') + } + if err := f.Truncate(0); err != nil { + return err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + if _, err := f.Write(buf.Bytes()); err != nil { + // Best-effort roll back to an empty (parseable, re-ingestable) file, then + // surface the original error. + f.Truncate(0) + return err + } + return nil +} diff --git a/internal/session/records_test.go b/internal/session/records_test.go new file mode 100644 index 0000000..059b85c --- /dev/null +++ b/internal/session/records_test.go @@ -0,0 +1,322 @@ +package session + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +// shortWriteFile is an appendFile whose Write persists a prefix and then errors, +// standing in for a full disk (write(2) fills the remaining space, returns a +// short count, and the next write returns ENOSPC — os.File.Write persists the +// truncated prefix before returning the error). Seek always reports the current +// length, so it also stands in for the O_APPEND descriptor writeRecord holds. +type shortWriteFile struct { + buf []byte + fail bool // when true, Write keeps only a prefix then returns an error +} + +func (f *shortWriteFile) Seek(offset int64, whence int) (int64, error) { + return int64(len(f.buf)), nil +} + +func (f *shortWriteFile) ReadAt(p []byte, off int64) (int, error) { + if off < 0 || off >= int64(len(f.buf)) { + return 0, io.EOF + } + return copy(p, f.buf[off:]), nil +} + +func (f *shortWriteFile) Truncate(size int64) error { + f.buf = f.buf[:size] + return nil +} + +func (f *shortWriteFile) Write(p []byte) (int, error) { + if f.fail { + half := len(p) / 2 + f.buf = append(f.buf, p[:half]...) + return half, errors.New("no space left on device") + } + f.buf = append(f.buf, p...) + return len(p), nil +} + +// failAfterWriter is a commitFile whose Write fails, recording whether the caller +// truncated back to 0 *after* the failed write — the rollback writeRecords must +// perform so a partial write never bricks a session JSONL file against its own +// recovery. The post-write ordering matters: writeRecords also truncates to 0 +// before writing, so only a truncate that follows the write attempt proves the +// rollback ran. +type failAfterWriter struct { + wrote bool + rolledBack bool +} + +func (w *failAfterWriter) Write(p []byte) (int, error) { + w.wrote = true + return 0, errors.New("no space left on device") +} + +func (w *failAfterWriter) Truncate(size int64) error { + if w.wrote && size == 0 { + w.rolledBack = true + } + return nil +} + +func (w *failAfterWriter) Seek(offset int64, whence int) (int64, error) { return 0, nil } + +// TestWriteRecordRollsBackPartialWrite is the ENOSPC regression on the append +// path (moved here from internal/review with the primitive it exercises): a short +// write that persists a newline-less prefix must be truncated away, so the file +// never retains a partial line that would fuse with the next record into one +// malformed physical record and make the whole file — the human-decision record +// the append path exists to protect — unparseable to every reader. +func TestWriteRecordRollsBackPartialWrite(t *testing.T) { + f := &shortWriteFile{} + first := []byte(`{"kind":"verdict","finding":"F-001","verdict":"confirmed","at":"2026-07-17"}`) + if err := writeRecord(f, append(first, '\n')); err != nil { + t.Fatalf("first record: %v", err) + } + good := string(f.buf) + + f.fail = true + second := []byte(`{"kind":"verdict","finding":"F-002","verdict":"rejected","at":"2026-07-17"}`) + if err := writeRecord(f, append(second, '\n')); err == nil { + t.Fatalf("expected a write error on a full disk") + } + if string(f.buf) != good { + t.Fatalf("partial line survived: file is %q, want the clean prefix %q", f.buf, good) + } + if !strings.HasSuffix(string(f.buf), "\n") { + t.Fatalf("file does not end on a newline: %q", f.buf) + } + + // The rolled-back file still parses one record per line, so a later record + // lands cleanly rather than fusing onto a fragment. + f.fail = false + if err := writeRecord(f, append(second, '\n')); err != nil { + t.Fatalf("record after rollback: %v", err) + } + lines := strings.Split(strings.TrimRight(string(f.buf), "\n"), "\n") + if len(lines) != 2 { + t.Fatalf("got %d lines, want 2: %q", len(lines), f.buf) + } +} + +// TestWriteRecordsRollsBackOnWriteError is the corrupt-on-failure regression for +// the commit path (moved here from internal/analyze with the primitive it +// exercises). writeRecords runs f.Truncate(0) before writing, so a short write +// (ENOSPC) would otherwise leave a truncated JSON fragment that not only breaks +// every reader but blocks the recovery path — the next ingest's guard errors on +// the fragment before it can conclude the file is free to rewrite. It must roll +// the file back to empty (parseable, re-ingestable) on any write error. +func TestWriteRecordsRollsBackOnWriteError(t *testing.T) { + w := &failAfterWriter{} + err := writeRecords(w, [][]byte{[]byte(`{"id":"F-001"}`)}) + if err == nil { + t.Fatal("writeRecords returned nil on a failing write; want the write error") + } + if !w.rolledBack { + t.Fatal("writeRecords did not truncate back to empty after the failed write; a partial line would survive and brick re-ingest") + } +} + +func appendFixture(t *testing.T, contents string) (dir, path string) { + t.Helper() + dir = t.TempDir() + path = filepath.Join(dir, FindingsFile) + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + t.Fatalf("seed write: %v", err) + } + return dir, path +} + +// TestAppendRecordFramesUnterminatedFile is the fused-line regression: a session +// JSONL file that does not end in a newline (hand edited, produced by another +// tool, or left short by a crash) must not have the new record appended onto its +// final line, which would make one physical line hold two JSON objects and the +// whole file unparseable. +func TestAppendRecordFramesUnterminatedFile(t *testing.T) { + _, path := appendFixture(t, `{"id":"F-001"}`) // no trailing newline + rec := []byte(`{"kind":"verdict","finding":"F-001"}`) + if err := AppendRecord(Append{Path: path, Record: rec, Label: "verdict for F-001", Kind: "verdict"}); err != nil { + t.Fatalf("AppendRecord: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + want := `{"id":"F-001"}` + "\n" + string(rec) + "\n" + if string(got) != want { + t.Fatalf("framing wrong:\n got %q\nwant %q", got, want) + } +} + +// TestAppendRecordRefusesOversizedLine holds the appended record to +// MaxJSONLLine, the read-side invariant every reader scans to. The refusal is +// pre-write and, since an unwritable record is a fact about the record alone, +// pre-open: the file need not exist for the caller to hear which limit was +// crossed. +func TestAppendRecordRefusesOversizedLine(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, FindingsFile) + rec := make([]byte, MaxJSONLLine) + err := AppendRecord(Append{Path: path, Record: rec, Label: "verdict for F-001", Kind: "verdict"}) + if err == nil || !strings.Contains(err.Error(), "line limit") { + t.Fatalf("expected an over-limit refusal, got %v", err) + } + if !strings.Contains(err.Error(), "verdict for F-001") { + t.Fatalf("refusal does not carry the caller's label: %v", err) + } + if !strings.Contains(err.Error(), FindingsFile) { + t.Fatalf("refusal does not name the file: %v", err) + } + if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { + t.Fatalf("the refusal created %s; want no write at all", path) + } +} + +// TestAppendRecordRefusesOversizedTotal is the write-side twin of ReadJSONL's +// total-size cap: a file written by an ingest step can legally sit right at +// MaxJSONLBytes, and appending even one small record would push it past the cap, +// durably bricking every record already in it. +func TestAppendRecordRefusesOversizedTotal(t *testing.T) { + pad := strings.Repeat("x", MaxJSONLBytes-20) + "\n" + _, path := appendFixture(t, pad) + rec := []byte(`{"kind":"verdict","finding":"F-001"}`) + err := AppendRecord(Append{Path: path, Record: rec, Label: "verdict for F-001", Kind: "verdict"}) + if err == nil || !strings.Contains(err.Error(), "JSONL file limit") { + t.Fatalf("expected an over-total refusal naming the JSONL file limit, got %v", err) + } + if !strings.Contains(err.Error(), "appending this verdict") { + t.Fatalf("refusal does not carry the caller's record kind: %v", err) + } + got, rerr := os.ReadFile(path) + if rerr != nil { + t.Fatalf("read: %v", rerr) + } + if string(got) != pad { + t.Fatalf("the file was modified despite the refusal") + } +} + +// TestAppendRecordVerifyRefusalWritesNothing covers the Verify seam: the closure +// runs under the append lock, over the file's current contents, and an error from +// it refuses the append outright. That is how a caller confirms the record still +// targets what the operator decided on after a concurrent rewrite. +func TestAppendRecordVerifyRefusalWritesNothing(t *testing.T) { + const seed = `{"id":"F-001"}` + "\n" + _, path := appendFixture(t, seed) + + var saw []byte + refusal := errors.New("the target changed since review started") + err := AppendRecord(Append{ + Path: path, + Record: []byte(`{"kind":"verdict","finding":"F-001"}`), + Label: "verdict for F-001", + Kind: "verdict", + Verify: func(current io.Reader) error { + b, rerr := io.ReadAll(current) + if rerr != nil { + return rerr + } + saw = b + return refusal + }, + }) + if !errors.Is(err, refusal) { + t.Fatalf("AppendRecord: got %v, want the Verify refusal", err) + } + if string(saw) != seed { + t.Fatalf("Verify saw %q, want the file's current contents %q", saw, seed) + } + got, rerr := os.ReadFile(path) + if rerr != nil { + t.Fatalf("read: %v", rerr) + } + if string(got) != seed { + t.Fatalf("a refused append still wrote: %q", got) + } +} + +// TestCommitRecordsReplacesContents is the happy path: the file is truncated and +// rewritten as one record per line, whatever it held before. +func TestCommitRecordsReplacesContents(t *testing.T) { + _, path := appendFixture(t, "stale\nlines\nthat must go\n") + err := CommitRecords(Commit{ + Path: path, + Records: [][]byte{[]byte(`{"id":"F-001"}`), []byte(`{"id":"F-002"}`)}, + }) + if err != nil { + t.Fatalf("CommitRecords: %v", err) + } + got, rerr := os.ReadFile(path) + if rerr != nil { + t.Fatalf("read: %v", rerr) + } + want := `{"id":"F-001"}` + "\n" + `{"id":"F-002"}` + "\n" + if string(got) != want { + t.Fatalf("commit wrote %q, want %q", got, want) + } +} + +// TestCommitRecordsGuardRefusalLeavesFileIntact is the protected-record +// regression: the guard reads the file's current contents before anything is +// truncated, and returning an error from it must leave every prior byte in place. +func TestCommitRecordsGuardRefusalLeavesFileIntact(t *testing.T) { + const seed = `{"id":"F-001"}` + "\n" + `{"kind":"verdict","finding":"F-001"}` + "\n" + _, path := appendFixture(t, seed) + + err := CommitRecords(Commit{ + Path: path, + Records: [][]byte{[]byte(`{"id":"F-009"}`)}, + Guard: func(current io.Reader) error { + b, rerr := io.ReadAll(current) + if rerr != nil { + return rerr + } + if bytes.Contains(b, []byte(`"kind":"verdict"`)) { + return fmt.Errorf("refusing to overwrite %s: it already holds verdict records", FindingsFile) + } + return nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "refusing to overwrite") { + t.Fatalf("expected the guard refusal, got %v", err) + } + got, rerr := os.ReadFile(path) + if rerr != nil { + t.Fatalf("read: %v", rerr) + } + if string(got) != seed { + t.Fatalf("a refused commit still rewrote the file: %q", got) + } +} + +// TestCommitRecordsRefusesSymlink is the arbitrary-file-truncation regression: a +// session artefact planted as a symlink must not be followed out of the session +// directory. +func TestCommitRecordsRefusesSymlink(t *testing.T) { + dir := t.TempDir() + outside := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(outside, []byte("original\n"), 0o600); err != nil { + t.Fatalf("seed victim: %v", err) + } + path := filepath.Join(dir, FindingsFile) + if err := os.Symlink(outside, path); err != nil { + t.Fatalf("symlink: %v", err) + } + if err := CommitRecords(Commit{Path: path, Records: [][]byte{[]byte(`{"id":"F-001"}`)}}); err == nil { + t.Fatal("CommitRecords followed a symlink; want refusal") + } + if b, _ := os.ReadFile(outside); string(b) != "original\n" { + t.Fatalf("victim file rewritten through symlink: %q", b) + } +} diff --git a/internal/session/session.go b/internal/session/session.go index 79b21ee..321f830 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -12,6 +12,7 @@ // transcript.jsonl word-aligned utterances (session-relative seconds) // timeline.jsonl merged, session-relative timeline // findings.jsonl analysis findings + appended verdicts +// tests.jsonl regression-test drafts + appended decisions // report.md human-readable session report package session @@ -55,6 +56,7 @@ const ( TranscriptFile = "transcript.jsonl" TimelineFile = "timeline.jsonl" FindingsFile = "findings.jsonl" + TestsFile = "tests.jsonl" ReportFile = "report.md" ) @@ -138,9 +140,10 @@ func (m Manifest) T0() (int64, error) { // untrusted session's JSONL files is bounded the same way: ReadJSONL caps both // a line (MaxJSONLLine) and the whole file (MaxJSONLBytes), and // analyze.ParseRecords — findings.jsonl's own scanner, not routed through -// ReadJSONL — carries the same pair of caps. analyze.Ingest caps the untrusted -// answer it validates at maxAnswerBytes, and the demo body caps what it -// accepts at capture time; neither reads a session's own JSONL files back. +// ReadJSONL — carries the same pair of caps, as does drafttests.ParseRecords +// for tests.jsonl. The two ingest boundaries cap the untrusted answer they +// validate at MaxAnswerBytes, and the demo body caps what it accepts at capture +// time; neither reads a session's own JSONL files back. const maxManifestBytes = 1 << 20 // 1 MiB // LoadManifest reads manifest.json from dir. @@ -458,6 +461,16 @@ const MaxJSONLLine = 4 << 20 // 4 MiB // not bound it. const MaxJSONLBytes = 16 << 20 // 16 MiB +// MaxAnswerBytes caps the untrusted model answer read at a validation boundary +// (analyze.Ingest, drafttests.Ingest), mirroring the bounded reads elsewhere +// (the demo server's 8 MiB body cap, the 4 MiB JSONL line cap). An answer is +// read from stdin or a file and is the one input the tool never trusts, so a +// multi-gigabyte one must not OOM the process before validation runs. It is +// generous for a genuine multi-record answer; anything larger is rejected, not +// buffered. It sits beside the JSONL caps because it bounds input at the same +// scale and for the same reason. +const MaxAnswerBytes = 16 << 20 // 16 MiB + // jsonlEncoder returns a json.Encoder configured exactly as WriteJSONL's own // encoders are, so a size measured against it predicts what WriteJSONL will // later check and write. HTML escaping is disabled: JSONL artefacts are never @@ -550,12 +563,14 @@ func ReadJSONL[T any](path string) ([]T, error) { // demo.appendRecords. That costs a second encoding pass over records that are // small structs; a durably unreadable session is the worse trade. // -// findings.jsonl never passes through here: analyze.commitFindings and -// review.AppendVerdict write it through their own locked descriptors instead, -// so neither can share this pre-flight. Both give their write path the matching -// MaxJSONLLine/MaxJSONLBytes checks ParseRecords (its read side) enforces — -// commitFindings via analyze.oversizedFindings, AppendVerdict via its own -// stat-then-append check. +// findings.jsonl and tests.jsonl never pass through here: they hold a machine +// record plus appended human records, so they are written through the locked +// descriptors of CommitRecords (a whole-file replacement, guarded) and +// AppendRecord (one appended line) instead, and neither can share this +// pre-flight. Both give their write path the matching MaxJSONLLine/MaxJSONLBytes +// checks their readers enforce — the commit side via its caller's own +// oversized-record pass, the append side via AppendRecord's stat-then-append +// check. func WriteJSONL[T any](path string, values []T) error { // Encode into one reusable buffer so the pre-flight pass holds a single // record, not the whole file, in memory. From 5fc65475bd1f5630a0ab633b1befeb0408d0b2bc Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 12 Sep 2026 06:42:14 +0100 Subject: [PATCH 2/2] feat: draft regression tests from confirmed findings (phase 2: CLI, docs, spec close) Assisted-by: Claude:claude-fable-5-1 --- .../brief/04-surfaces/08-draft-tests.md | 6 +- .../planned/itd-9-regression-test-drafting.md | 69 ---- .../shipped/itd-9-regression-test-drafting.md | 152 ++++++++ ...09120417480624-regression-test-drafting.md | 249 ++++++++++--- ...-agent-emits-verdict-json-in-shapes-tha.md | 14 + .github/workflows/ci.yml | 10 + AGENTS.md | 10 +- CHANGELOG.md | 5 +- docs/reference/cli.md | 75 +++- docs/reference/session-directory.md | 4 +- internal/analyze/ingest.go | 2 +- internal/cli/cli.go | 249 ++++++++++++- internal/cli/cli_test.go | 352 +++++++++++++++++- internal/drafttests/drafttests.go | 7 +- internal/drafttests/emit.go | 49 ++- internal/drafttests/emit_test.go | 109 +++++- internal/drafttests/ingest.go | 2 +- internal/drafttests/render_test.go | 2 +- internal/drafttests/review.go | 19 +- internal/drafttests/review_test.go | 40 +- 20 files changed, 1235 insertions(+), 190 deletions(-) delete mode 100644 .abcd/development/intents/planned/itd-9-regression-test-drafting.md create mode 100644 .abcd/development/intents/shipped/itd-9-regression-test-drafting.md rename .abcd/development/specs/{open => closed}/spc-2609120417480624-regression-test-drafting.md (76%) create mode 100644 .abcd/work/issues/open/iss-2609120532387596-abcd-s-intent-auditor-agent-emits-verdict-json-in-shapes-tha.md diff --git a/.abcd/development/brief/04-surfaces/08-draft-tests.md b/.abcd/development/brief/04-surfaces/08-draft-tests.md index 2fb8a79..a8297e7 100644 --- a/.abcd/development/brief/04-surfaces/08-draft-tests.md +++ b/.abcd/development/brief/04-surfaces/08-draft-tests.md @@ -31,7 +31,9 @@ See the schemas page ([`../05-internals/02-schemas.md`](../05-internals/02-schem `draft-tests` runs in exactly one mode, extending `analyze`'s emit-or-ingest rule by one: emit (neither `-ingest` nor `-render`), ingest (`-ingest`), or render -(`-render`). `-ingest` combines with neither `-out` nor `-render`. Emit reads +(`-render`). `-ingest` combines with neither `-out` nor `-render`, and `-window` is refused +outside emit mode (`-window applies to the emit mode only`) rather than silently +ignored. Emit reads `manifest.json`, `findings.jsonl`, and `timeline.jsonl`; ingest reads `manifest.json` and `findings.jsonl` only (drafts are validated against the *findings*, never re-derived from the timeline); render reads `manifest.json`, @@ -181,7 +183,7 @@ Both refusals write nothing and exit 1 (a well-formed invocation whose work cannot be done), naming the counts so the operator can see *why* they are empty: ``` -testimony: no confirmed findings to draft tests from (5 findings: 0 confirmed, 2 unverified, 1 duplicate, 1 rejected); confirm one with `testimony review -session sessions/x` first +testimony: no confirmed findings to draft tests from (5 findings: 0 confirmed, 2 unverified, 1 duplicate, 2 rejected); confirm one with `testimony review -session sessions/x` first testimony: no accepted test drafts to render (3 drafts: 0 accepted, 0 edited, 2 proposed, 1 rejected); accept one with `testimony review -session sessions/x -kind tests` first ``` diff --git a/.abcd/development/intents/planned/itd-9-regression-test-drafting.md b/.abcd/development/intents/planned/itd-9-regression-test-drafting.md deleted file mode 100644 index 253fc42..0000000 --- a/.abcd/development/intents/planned/itd-9-regression-test-drafting.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -id: itd-9 -slug: regression-test-drafting -spec_id: spc-2609120417480624 -kind: standalone -suggested_kind: null -reclassification_history: [] -builds_on: [] -severity: minor ---- - -# A Confirmed Bug Arrives With Its Test Already Written - -## Press Release - -> **Testimony drafts a regression test case from every confirmed finding.** Once a human has flipped a finding to `confirmed`, the drafting step turns it into a proposed test case: the steps reconstructed from the event window, the behaviour the participant expected, the behaviour they got, and their own words as the rationale. The draft goes to a person to accept, edit, or reject — nothing is written into a suite automatically. A session stops being a report that ages and becomes a test that keeps the bug from coming back. -> -> "A finding tells me something broke once; a test tells me it stays fixed," said Alice, the maintainer. "Getting the steps and the expected behaviour handed to me in the participant's own framing is most of the work of writing the test, and it's the part I always got subtly wrong from memory." - -## Why This Matters - -The pipeline's value decays at the end: a confirmed finding becomes an issue, the issue gets fixed, and the evidence that motivated it is never exercised again. Meanwhile the reproduction steps — which are the expensive, error-prone part of writing a regression test — are already sitting in the timeline as a precise event window with a spoken account of what the person expected. Drafting the test while that evidence is intact converts a one-off observation into a standing guarantee. - -Keeping the human in the loop mirrors the stance the rest of the pipeline takes. Findings are unverified until a person confirms them; a test drafted from a finding is likewise a proposal, not a commit. The step is deliberately downstream of verification, so only evidence a human already vouched for can become a test. - -## What's In Scope - -- Drafting a test case from a `confirmed` finding: reproduction steps derived from the finding's event window, expected versus observed behaviour, and the participant quote as rationale. -- A human accept / edit / reject pass over each drafted test, with the decision retained as the verdicts already are. -- Linking the drafted test back to its source finding and session, so a failing test leads to the evidence that motivated it. -- Emitting the draft in a form the docs-as-code manual test records can hold, so a session becomes evidence attached to a test run. - -## What's Out of Scope - -- Generating executable test code for the application under test; that needs its test framework and conventions, and is a further step. -- Running, filing, or committing tests automatically — the output is a draft for review. -- Drafting from unverified or rejected findings; only human-confirmed findings are eligible. -- Reference-capture findings (itd-4), which are design preferences rather than defects and have nothing to regress against. - -## Scope Conditions - -- The session holds a `findings.jsonl` with at least one finding whose current - verdict is `confirmed`; a session with none is staged loudly rather than - drafted from. -- `timeline.jsonl` is present, so each confirmed finding's event window - resolves — the reproduction steps are reconstructed from it. -- The host that answers the drafting request is the operator's chosen model, as - for `analyze`: the CLI emits the request, never calls a model, holds no keys, - and adds no network dependency. - -## Acceptance Criteria - -- **Given** a finding whose status is `confirmed`, **when** the drafting step runs, **then** a test case draft is produced containing reproduction steps from the event window, the expected and observed behaviour, and the participant's quote. -- **Given** a finding whose status is `unverified` or `rejected`, **when** the drafting step runs, **then** no test case is drafted for it. -- **Given** a drafted test case, **when** a human accepts or rejects it, **then** the decision is retained and the draft remains linked to its source finding and session. - -## Open Questions - -- Does the event window alone yield reproduction steps a developer can follow, or does a reliable repro need the keyframe channel as well? -- Where do accepted drafts live — the application's own repository, the docs-as-code test plan, or alongside the session? The architecture note leaves this open. -- Should a drafted test carry the finding's severity through, so triage order survives the hand-off? - -## Audit Notes - -_Empty. Populated by intent-fidelity-reviewer when intent moves to shipped/._ - -## Grounds - -- pursued: a confirmed finding's event window plus the participant's own words is enough for a host model to draft a followable regression test, and a human accept/edit/reject pass keeps every draft a proposal; what would show it wrong is drafts whose steps a developer cannot follow from the window alone, which the edited-decision rate on real sessions would reveal diff --git a/.abcd/development/intents/shipped/itd-9-regression-test-drafting.md b/.abcd/development/intents/shipped/itd-9-regression-test-drafting.md new file mode 100644 index 0000000..e4eb18b --- /dev/null +++ b/.abcd/development/intents/shipped/itd-9-regression-test-drafting.md @@ -0,0 +1,152 @@ +--- +id: itd-9 +slug: regression-test-drafting +spec_id: spc-2609120417480624 +kind: standalone +suggested_kind: null +reclassification_history: [] +builds_on: [] +severity: minor +--- + +# A Confirmed Bug Arrives With Its Test Already Written + +## Press Release + +> **Testimony drafts a regression test case from every confirmed finding.** Once a human has flipped a finding to `confirmed`, the drafting step turns it into a proposed test case: the steps reconstructed from the event window, the behaviour the participant expected, the behaviour they got, and their own words as the rationale. The draft goes to a person to accept, edit, or reject — nothing is written into a suite automatically. A session stops being a report that ages and becomes a test that keeps the bug from coming back. +> +> "A finding tells me something broke once; a test tells me it stays fixed," said Alice, the maintainer. "Getting the steps and the expected behaviour handed to me in the participant's own framing is most of the work of writing the test, and it's the part I always got subtly wrong from memory." + +## Why This Matters + +The pipeline's value decays at the end: a confirmed finding becomes an issue, the issue gets fixed, and the evidence that motivated it is never exercised again. Meanwhile the reproduction steps — which are the expensive, error-prone part of writing a regression test — are already sitting in the timeline as a precise event window with a spoken account of what the person expected. Drafting the test while that evidence is intact converts a one-off observation into a standing guarantee. + +Keeping the human in the loop mirrors the stance the rest of the pipeline takes. Findings are unverified until a person confirms them; a test drafted from a finding is likewise a proposal, not a commit. The step is deliberately downstream of verification, so only evidence a human already vouched for can become a test. + +## What's In Scope + +- Drafting a test case from a `confirmed` finding: reproduction steps derived from the finding's event window, expected versus observed behaviour, and the participant quote as rationale. +- A human accept / edit / reject pass over each drafted test, with the decision retained as the verdicts already are. +- Linking the drafted test back to its source finding and session, so a failing test leads to the evidence that motivated it. +- Emitting the draft in a form the docs-as-code manual test records can hold, so a session becomes evidence attached to a test run. + +## What's Out of Scope + +- Generating executable test code for the application under test; that needs its test framework and conventions, and is a further step. +- Running, filing, or committing tests automatically — the output is a draft for review. +- Drafting from unverified or rejected findings; only human-confirmed findings are eligible. +- Reference-capture findings (itd-4), which are design preferences rather than defects and have nothing to regress against. + +## Scope Conditions + +- The session holds a `findings.jsonl` with at least one finding whose current + verdict is `confirmed`; a session with none is staged loudly rather than + drafted from. +- `timeline.jsonl` is present, so each confirmed finding's event window + resolves — the reproduction steps are reconstructed from it. +- The host that answers the drafting request is the operator's chosen model, as + for `analyze`: the CLI emits the request, never calls a model, holds no keys, + and adds no network dependency. + +## Acceptance Criteria + +- **Given** a finding whose status is `confirmed`, **when** the drafting step runs, **then** a test case draft is produced containing reproduction steps from the event window, the expected and observed behaviour, and the participant's quote. +- **Given** a finding whose status is `unverified` or `rejected`, **when** the drafting step runs, **then** no test case is drafted for it. +- **Given** a drafted test case, **when** a human accepts or rejects it, **then** the decision is retained and the draft remains linked to its source finding and session. + +## Open Questions + +- Does the event window alone yield reproduction steps a developer can follow, or does a reliable repro need the keyframe channel as well? +- Where do accepted drafts live — the application's own repository, the docs-as-code test plan, or alongside the session? The architecture note leaves this open. +- Should a drafted test carry the finding's severity through, so triage order survives the hand-off? + +## Audit Notes + + +Fidelity review — receipt rcp-f14572859148 (verifier abcd:intent-auditor claude-opus-5[1m]). + +Provenance: abcd:intent-auditor@claude-opus-5[1m] · rubric_hash sha256:6e1ec6201a0891f1863b71cc9b5703ab8681f3f56252128f641c5ea6f81d3fd6 · prompt_hash sha256:462ac0436fbe2acebbb5155ba514eb06880b7ea1edb7a524953461d40e8436a5 +Input attestations: diff:origin/main..working tree@sha256:b6447e482347548dcdf5331618a74bf4b48dc29605cdffd69438d7c9878aa924; + +Acceptance rollup: MET 2 · MET_WITH_CONCERNS 1 · NOT_MET 0 · INCONCLUSIVE 0 + +Per-criterion verdicts: +- ac-1 — MET_WITH_CONCERNS: A confirmed finding reaches the drafting step with its event window attached (emit.go builds the request from Window over timeline.jsonl) and the ingested draft carries steps, expected, observed and the finding's quote byte-for-byte (validate enforces quote and severity equality); smoke run over examples/sample-session produced exactly such a draft. Concern: the drafting oracle is host-delegated, so the CLI emits a request and validates an answer rather than drafting in-process, and the boundary enforces only non-emptiness of steps — that the steps actually derive from the event window is instructed in the rubric, never verified. + evidence: internal/drafttests/emit.go:152 + evidence: internal/drafttests/window.go:37 + evidence: internal/drafttests/drafttests.go:60 + evidence: internal/drafttests/ingest.go:413 + evidence: internal/drafttests/ingest.go:387 + evidence: examples/sample-session/tests.jsonl:1 +- ac-2 — MET: Eligibility is enforced twice over the same effective-status computation: eligible() admits only findings whose current verdict is confirmed (so a rejected, unverified or duplicate finding is omitted from the emitted request) and validate refuses any ingested draft naming a finding not in that set; a run over the sample session emitted only F-001 and refused drafts of F-002 (unverified), F-003 (rejected) and F-005 (duplicate), writing no tests.jsonl. + evidence: internal/drafttests/drafttests.go:327 + evidence: internal/drafttests/emit.go:48 + evidence: internal/drafttests/ingest.go:371 + evidence: internal/drafttests/ingest_test.go:1 +- ac-3 — MET: A decision is an appended, non-destructive record written through session.AppendRecord — the draft line is never rewritten — and the draft's link fields are unreachable by any later write: Edit is a closed {title,steps,expected,observed} subset decoded with DisallowUnknownFields, and commitDrafts refuses to overwrite a tests.jsonl that already holds decisions. Smoke run: two decisions on T-001 appended as lines 2 and 3 with the draft line byte-identical, an edit naming "finding" refused, and a re-ingest refused. + evidence: internal/drafttests/review.go:390 + evidence: internal/drafttests/drafttests.go:86 + evidence: internal/drafttests/review.go:146 + evidence: internal/drafttests/ingest.go:157 + evidence: internal/session/records.go:31 + evidence: examples/sample-session/tests.jsonl:4 + +Gap audit: +- honoured: + - Drafting a test case from a confirmed finding: reproduction steps derived from the finding's event window, expected versus observed behaviour, and the participant quote as rationale. + evidence: internal/drafttests/window.go:37 + evidence: internal/drafttests/emit.go:152 + evidence: internal/drafttests/drafttests.go:60 + - A human accept / edit / reject pass over each drafted test, with the decision retained as the verdicts already are. + evidence: internal/drafttests/review.go:37 + evidence: internal/drafttests/review.go:390 + evidence: internal/session/records.go:31 + - Linking the drafted test back to its source finding and session, so a failing test leads to the evidence that motivated it. + evidence: internal/drafttests/ingest.go:376 + evidence: internal/drafttests/drafttests.go:86 + evidence: internal/drafttests/render.go:11 + - Emitting the draft in a form the docs-as-code manual test records can hold. + evidence: internal/drafttests/render.go:23 + evidence: internal/drafttests/testdata/tests.md:1 + evidence: docs/how-to/draft-regression-tests.md:1 + - Nothing is written into a suite automatically: every draft is born a proposal and the model's claimed status is laundered away. + evidence: internal/drafttests/ingest.go:121 + evidence: internal/drafttests/render.go:15 + - The CLI never calls a model, holds no keys, and adds no network dependency. + evidence: go.mod:1 + evidence: internal/drafttests/drafttests.go:18 + evidence: internal/drafttests/emit.go:36 +- diverged: + - "the drafting step turns it into a proposed test case" — delivered as a two-phase host-delegated exchange (emit a request, then ingest and validate an answer) rather than one in-process step; the operator must run a model between the two halves. + evidence: internal/drafttests/emit.go:36 + evidence: internal/drafttests/ingest.go:41 + evidence: internal/cli/cli.go:435 + - "the steps reconstructed from the event window" — the window is supplied and the rubric forbids inventing steps, but the validation boundary checks only that steps is non-empty and within bounds; step provenance from the window is never verified, so only quote and severity are evidence-locked. + evidence: internal/drafttests/emit.go:73 + evidence: internal/drafttests/ingest.go:387 + evidence: internal/drafttests/ingest.go:34 + - "Reference-capture findings (itd-4), which are design preferences rather than defects" out of scope — implemented as a mode != B guard that the code itself records as dead today, while a confirmed mode-A finding of type preference or idea stays eligible because type is deliberately not filtered. + evidence: internal/drafttests/drafttests.go:322 + evidence: internal/drafttests/drafttests.go:331 +- missing: (none) + +Scope-condition dispositions: +- cond-2609120430207432 — survived: Both write paths check eligibility before doing anything else and a session with no confirmed finding is refused loudly with the by-status tally and the review hint, writing nothing; a run with the sample's verdicts stripped exited 1 on emit and on ingest with "5 findings: 0 confirmed, 5 unverified, 0 duplicate, 0 rejected". + evidence: internal/drafttests/drafttests.go:397 + evidence: internal/drafttests/drafttests.go:405 + evidence: internal/drafttests/emit.go:49 + evidence: internal/drafttests/ingest.go:55 +- cond-2609120430209331 — narrowed: Only the emit path reads timeline.jsonl (and refuses with a "run merge first" hint when it is absent); ingest, review and render need no timeline at all, and a confirmed finding whose cited evidence ids match no entry does not fail — Window falls back to a time-centred span around f.T, which can be empty. + narrowing: Holds for `draft-tests` emit only — ingest/review/render ran successfully with timeline.jsonl deleted — and "the event window resolves" means a window is always produced, not that the finding's cited evidence actually resolved: unresolvable evidence silently degrades to [f.T-window, f.T+window]. + evidence: internal/drafttests/emit.go:56 + evidence: internal/drafttests/window.go:63 + evidence: internal/drafttests/ingest.go:35 + evidence: .github/workflows/ci.yml:124 +- cond-2609120430205995 — survived: The drafting layer is stdlib-plus-internal only and the module still declares no dependency; EmitRequest returns the request as text for the operator's chosen host and Ingest reads the answer from a file or stdin, so no model is called and no key is held on this path. + evidence: go.mod:1 + evidence: internal/drafttests/drafttests.go:1 + evidence: internal/drafttests/emit.go:3 + evidence: internal/cli/cli.go:482 +## Grounds + +- pursued: a confirmed finding's event window plus the participant's own words is enough for a host model to draft a followable regression test, and a human accept/edit/reject pass keeps every draft a proposal; what would show it wrong is drafts whose steps a developer cannot follow from the window alone, which the edited-decision rate on real sessions would reveal diff --git a/.abcd/development/specs/open/spc-2609120417480624-regression-test-drafting.md b/.abcd/development/specs/closed/spc-2609120417480624-regression-test-drafting.md similarity index 76% rename from .abcd/development/specs/open/spc-2609120417480624-regression-test-drafting.md rename to .abcd/development/specs/closed/spc-2609120417480624-regression-test-drafting.md index 4ac73d3..6c73414 100644 --- a/.abcd/development/specs/open/spc-2609120417480624-regression-test-drafting.md +++ b/.abcd/development/specs/closed/spc-2609120417480624-regression-test-drafting.md @@ -77,7 +77,14 @@ testimony review -session DIR -kind tests -test T-NNN -decision edited -edit FIL `draft-tests` runs in exactly one mode, mirroring `analyze`'s emit-or-ingest rule and extending it by one: emit (neither `-ingest` nor `-render`), ingest (`-ingest`), or render (`-render`). `-ingest` combines with neither `-out` nor -`-render`. `-out` pairs with emit or render. Every flag follows the CLI's +`-render` (each refused by its own message: `-out and -ingest cannot be combined`, +`-render and -ingest cannot be combined`). `-out` pairs with emit or render. +`-window` belongs to emit alone — ingest validates against the findings and +render reads only what is already on disk — so passing it with `-ingest` or +`-render` is refused (`-window applies to the emit mode only`) rather than +silently ignored, which would let a caller who meant to widen the window believe +they had. Set-ness is detected with `fs.Visit`, like the empty-value guards, so +the default never trips the check. Every flag follows the CLI's existing exit-2 gauntlet: `-session` required; an explicitly-empty `-out`, `-ingest`, or `-kind` refused as a wrong invocation (the unset-shell-variable case); a non-finite `-window` refused (the `report -window` precedent); no @@ -102,16 +109,24 @@ positional arguments (`rejectArgs`). `-finding`/`-verdict` are refused with `-kind tests`, and `-test`/`-decision`/ `-edit` with `-kind findings`, at exit 2 — a flag that belongs to the other -record family is a wrong invocation, not a silently ignored one. `-kind +record family is a wrong invocation, not a silently ignored one. The pairing is +checked twice on purpose: in `internal/cli`, where it takes the usage status and +is refused before the session is resolved or a file is read, and again in +`review.Run`, so the rule is a property of the API rather than of one caller's +invariants. `-kind` is parsed through `review.ParseKindFlag` against the closed +set `{findings, tests}` (`review.KindFindings`/`review.KindTests`), `-test` +through `drafttests.IsDraftID`, and `-decision` through +`drafttests.ParseDecisionFlag`. `-edit` is refused both when `-decision edited` +lacks it and when it accompanies any other decision. `-kind findings` is byte-for-byte the behaviour `review` has today. Reads by mode: emit reads `manifest.json`, `timeline.jsonl`, and `findings.jsonl`; ingest reads `manifest.json` and `findings.jsonl` only (drafts are validated against the *findings*, never re-derived from the timeline); render reads `manifest.json`, `findings.jsonl`, and `tests.jsonl`. Emit hints to -run `merge` first when the timeline is missing (reusing `analyze`'s -`loadTimeline`, so a duplicated entry id or an unknown `src` is refused there -too); every mode hints to run `analyze -ingest` first when there is no +run `merge` first when the timeline is missing (reusing `analyze`'s timeline +reader, which is exported as `analyze.LoadTimeline` for exactly this caller, so a +duplicated entry id or an unknown `src` is refused there too); every mode hints to run `analyze -ingest` first when there is no `findings.jsonl`, and ingest/render/`review -kind tests` hint to run `draft-tests -ingest` first when there is no `tests.jsonl`. @@ -134,6 +149,19 @@ evidence resolves to no entry — impossible after `analyze -ingest`, reachable a hand-edited `findings.jsonl` — falls back to `[f.T - window, f.T + window]`. Speech and event entries are both included: the utterances around the moment are what carry the *expected* behaviour, and the events are what carry the *steps*. +`Window` returns the entries in time order — it sorts its own result rather than +trusting the caller, so a hand-edited or exchanged `timeline.jsonl` cannot hand +the model a repro in the wrong order. + +**Where the steps stop.** The instructions state the boundary rather than leaving +it to be inferred: the steps end at the **last cited evidence event at or before +the finding's `t`**, and a cited evidence event *after* the finding's `t` belongs +in `observed`, as part of what the participant did in response. On the sample, +F-001's `t` is 22 s, so the steps end at `ev-003` (the first Save click at +19.2 s) and `ev-004` (the second click at 24.1 s) is observed behaviour. A live +run against a real host model found the unqualified phrasing ambiguous on exactly +this point — the second click had no stated home — which is why the rule is +written out. `-window` defaults to **10 seconds**, not `report`'s 2.5: `report`'s window joins an event to the utterance it accompanies, whereas a repro needs the lead-up @@ -168,20 +196,57 @@ Structure, in order, mirroring `analyze.EmitRequest`: 3. **Instructions** — one or more drafts per confirmed finding, in finding-id order; `steps` in time order, each one imperative action a developer can follow, naming the selector or route where the window names it, ending at the - moment the finding is anchored to; `expected` the behaviour the participant - expected, grounded in their utterances; `observed` what the system actually - did, grounded in the window's events and utterances; `title` one line naming - the defect. -4. **Rubric body** — the field definitions and the hard constraints restated as - the rules ingest enforces (quote copied byte-for-byte from the finding's - `quote`; `severity` and `session` copied unchanged; `finding` naming the - finding the draft came from; `steps` non-empty). + last cited evidence event at or before the finding's `t` (a later cited event + belongs in `observed`); `expected` the behaviour the participant expected, + grounded in their utterances; `observed` what the system actually did, + grounded in the window's events and utterances; `title` one line naming the + defect. The instructions also license **one** opening orientation step derived + from the `route` on the window's first event, with every other step required to + correspond to an entry actually in the window — see "the orientation step" + below. +4. **Rubric body** — the field definitions, how to read each finding record, and + the hard constraints restated as the rules ingest enforces (quote copied + byte-for-byte from the finding's `quote`; `severity` and `session` copied + unchanged; `finding` naming the finding the draft came from; `steps` + non-empty). Two fields of the record the model *reads* are defined there + because neither is a field it writes: `status` (see "the status field" below) + and `mode` (`A` is the application under test, `B` is reference capture of a + third-party app; only `A` is eligible, so every finding shown is `A`). 5. **Session context** — manifest `app`, `participant`, and the ordered `tasks`. 6. **Confirmed findings** — per eligible finding, in id order: a prose line - naming its id, `type`, `severity` and clock, then the finding's own JSON line - in a ```jsonl fence (so the `quote` bytes the model must copy are - unambiguous), then its event window as a ```jsonl fence of timeline entries in - time order. + naming its id, `type`, `severity`, clock, **and the date of the verdict that + confirmed it**, then the finding's own JSON line in a ```jsonl fence (so the + `quote` bytes the model must copy are unambiguous), then its event window as a + ```jsonl fence of timeline entries in time order. + + **The status field.** The finding's record is shown *verbatim as stored*, so + its `status` reads `unverified` — the birth state every finding this tool + writes carries. Against a heading that says every finding below is confirmed, + and hard constraints that say an unverified finding is ineligible, that reads + as a self-contradiction; a live run against a real host model reported it as + one. Two repairs were available: substitute the effective status into the + rendered line, or leave the record alone and explain it. **The record is left + alone and explained**, because the model must copy `quote` and `severity` out + of that line byte-for-byte — rewriting one of its fields would make the record + the model is shown differ from the record ingest validates against, which is + the shown-vs-validated gap this package closes everywhere else + (`indexTimeline`'s SafeText reasoning). So the per-finding header carries + `confirmed by human verdict on ` (the date from the same + `EffectiveStatus` computation eligibility uses, sanitised through `SafeInline`, + and the clause degrades to `confirmed by human verdict` when the verdict + carries no renderable date), and the rubric states that `status` is the birth + state, that it is not the finding's current status, and that the verdict + records that confirmed these findings are not shown. + + **The orientation step.** The worked example's first step ("Open #general…") + corresponds to no event in the sample window, so on its own it licensed an + invented step against the stance paragraph's "never invent a step" — the same + live run flagged this. Rather than weaken the example (every followable repro + has to say where it starts), the instructions state where that step may come + from: the `route` on the window's first event names where the participant + already is, **one** opening orientation step may be derived from it, and every + other step must correspond to an event or utterance that is actually in the + window. 7. **Required output shape + worked example** — > Answer with a single JSON document: `{"rubric":"testimony-testdraft/v1","tests":[ … ]}`. @@ -230,7 +295,7 @@ schema is closed (`DisallowUnknownFields`). | `observed` | string | yes | non-empty after `SafeText`+trim | | `rationale_quote` | string | yes | **equals** the source finding's `quote` (compared in `SafeText` form) | | `severity` | int | yes | **equals** the source finding's `severity` | -| `status` | string | no | **ignored on input and forced to `"proposed"`** on ingest, whatever the JSON says | +| `status` | string | no on input, always present on disk | **ignored on input and forced to `"proposed"`** on ingest, whatever the JSON says; the written record therefore always carries it, as `findings.jsonl`'s `status` does | The field is `finding`, not `finding_id`: the verdict record in `findings.jsonl` already names its referent `finding`, and the decision record below names its @@ -279,6 +344,10 @@ and ingest refuses any restatement that disagrees. 6. Commit through `session.CommitRecords` (below) with a guard that refuses to overwrite a `tests.jsonl` already holding any `kind:"decision"` line — the retained human record, protected exactly as `findings.jsonl`'s verdicts are. + `holdsDecisions(r io.Reader, path string) (bool, error)` mirrors + `analyze.holdsVerdicts` exactly, including its shape, and `commitDrafts` turns + a true into the refusal message so the `Guard func(io.Reader) error` seam stays + as narrow as the primitive declares it. 7. Print `validated N test drafts → (all proposed)`. An answer with an empty `tests` array (a bare `[]`, `{"tests":[]}`, or a @@ -398,14 +467,17 @@ type Append struct { Verify func(current io.Reader) error // optional re-check of the file's contents, run under the lock } -// AppendRecord appends a.Record as its own physical line: it opens Path under -// the no-follow guard (O_APPEND|O_RDWR), takes an exclusive advisory lock, -// pre-flights the record against MaxJSONLLine and the file against -// MaxJSONLBytes, runs a.Verify over the current contents, frames the record -// with a leading newline when the file does not already end in one, writes it, -// truncates back to the pre-write length on a short write, and returns the -// Close error so a record is never reported written when its bytes did not -// reach disk. +// AppendRecord appends a.Record as its own physical line: it pre-flights the +// record against MaxJSONLLine *before* the open (an unwritable record is a fact +// about the record alone, so the refusal must name it whether or not the file +// exists yet — ordering it after the open reports a missing file, and its joined +// path, in place of the limit the caller can act on), then opens Path under the +// no-follow guard (O_APPEND|O_RDWR), takes an exclusive advisory lock, +// pre-flights the file against MaxJSONLBytes, runs a.Verify over the current +// contents, frames the record with a leading newline when the file does not +// already end in one, writes it, truncates back to the pre-write length on a +// short write, and returns the Close error so a record is never reported written +// when its bytes did not reach disk. func AppendRecord(a Append) error // Commit is a whole-file replacement of a session JSONL file. @@ -420,7 +492,12 @@ type Commit struct { // the whole set into one buffer before truncating, writes it as a single Write, // rolls the file back to empty on a short write (an empty JSONL file is // parseable and re-ingestable, so the failure state does not foreclose its own -// repair), and returns the Close error. +// repair), and returns the Close error. Commit carries no Label — a whole-file +// replacement names no single record — so a write or Close failure is wrapped +// with the file's base name (`write findings.jsonl: …`, `write tests.jsonl: …`) +// rather than the caller's own phrasing. Callers encode their records with +// json.Marshal, the same encoder their oversized-record pre-flight measures +// with, so the bytes written are exactly the bytes that passed the check. func CommitRecords(c Commit) error ``` @@ -428,11 +505,14 @@ Callers after the extraction: - `review.AppendVerdict` → `session.AppendRecord` with `Label: "verdict for " + SafeText(v.Finding)`, `Kind: "verdict"`, and - `Verify` wrapping the existing `verifyTarget` logic. `review.writeVerdict` and - its `verdictFile` interface are deleted; their behaviour and their two size + `Verify` wrapping the existing `verifyTarget` logic (whose signature narrows + from `*os.File` to the `io.Reader` the primitive hands it). `review.writeVerdict` + and its `verdictFile` interface are deleted; their behaviour and their two size error messages move verbatim (the `Label`/`Kind` parameters and `filepath.Base(Path)` reproduce today's strings byte-for-byte, so review's - existing message assertions keep passing unchanged). + existing message assertions keep passing unchanged). The one string that does + change is the commit path's write/Close wrapper, from `write findings: …` to + `write findings.jsonl: …`; nothing asserts it. - `analyze.commitFindings`/`writeFindings` → `session.CommitRecords` with `Guard: holdsVerdicts` and its existing refusal message. The `findingsFile` interface is deleted. @@ -484,10 +564,26 @@ participant `P1`). 2 of 3 drafts accepted. **Rationale (participant, [00:22]):** “I clicked save and nothing happened” ``` -Every inserted value goes through `session.SafeInline` — the one shared home for -the escape set that `report.md` and the emitted request already use — so an -attacker-authored draft cannot forge Markdown structure, an active link, or an -image beacon in a document the operator pastes into their own repository. The +Every inserted value goes through the one shared home for the escape set that +`report.md` and the emitted request already use, so an attacker-authored draft +cannot forge Markdown structure, an active link, or an image beacon in a document +the operator pastes into their own repository. Which of its two forms applies +depends on the context, exactly as in `report.md`: a value rendered as prose goes +through `session.SafeInline`, while the four rendered **inside a code span** +(`session`, `app`, `participant`, and the finding id) go through the +backtick-stripping form `report.mdCode` uses. A backslash escape does not apply +inside a code span — `SafeInline` would escape a backtick to `\`` and the +backtick would still close the span, leaving the tail as active markup — so the +span content has its backticks stripped instead. Ordinary input is byte-identical +under either form. + +The Markdown block above shows the plan as a **viewer renders it**. In the source +bytes, a step such as `Click the Save button ([data-testid=save-btn]).` is written +`Click the Save button \(\[data-testid=save-btn\]\).` — `SafeInline` escapes the +bracket and parenthesis triggers, which is what stops an `[x](http://…)` payload +in a draft field from becoming a live link, and is what `report.md` already does +to the identical text. The counts sentence is emitted as one physical line; it is +wrapped above only to fit the page. The clock is rendered `[MM:SS]` from the *finding's* `t`, with a leading `-` for a negative time, matching `report`. Render writes nothing into the session directory unless `-out` names a path there; the artefact is a hand-off copy, so @@ -505,28 +601,53 @@ Both refusals write nothing and exit 1 (a well-formed invocation whose work cannot be done), naming the counts so the operator can see *why* they are empty: ``` -testimony: no confirmed findings to draft tests from (5 findings: 0 confirmed, 2 unverified, 1 duplicate, 1 rejected); confirm one with `testimony review -session sessions/x` first +testimony: no confirmed findings to draft tests from (5 findings: 0 confirmed, 2 unverified, 1 duplicate, 2 rejected); confirm one with `testimony review -session sessions/x` first testimony: no accepted test drafts to render (3 drafts: 0 accepted, 0 edited, 2 proposed, 1 rejected); accept one with `testimony review -session sessions/x -kind tests` first ``` The first applies to emit **and** ingest (with no eligible finding there is -nothing a draft could legally reference). The second keeps `-out FILE` from -truncating an existing test plan into an empty document, which is the same -reasoning behind `analyze -ingest`'s empty-answer refusal. +nothing a draft could legally reference), and on ingest it fires *before a byte of +the answer is read*: every draft would fail the same rule, so the operator should +read the one fact that explains them rather than a wall of per-draft errors. The +second keeps `-out FILE` from truncating an existing test plan into an empty +document, which is the same reasoning behind `analyze -ingest`'s empty-answer +refusal. Each wraps a package sentinel (`ErrNoConfirmedFindings`, +`ErrNoAcceptedDrafts`) so a caller can tell a staged-empty session from a genuine +failure; both map to exit 1, the status a well-formed invocation whose work cannot +be done already takes. + +The counts are rendered with an unconditional plural noun, so a single-record +session reads `1 findings` / `1 drafts`. That is deliberate: the strings above are +the contract, and `analyze -ingest`'s own `validated N findings` has read the same +way since itd-2. Pluralising here alone would make the two commands disagree. ### Package layout & session constants -- **`internal/drafttests`** (new) — `Draft` and `Decision` types, `RubricVersion`, - `Load`/`ParseRecords`, `EffectiveStatus`, `SameIdentity`, `Window`, - `EmitRequest`, `Ingest`, `Render`, `Review` (the walk and the single-decision - path), `AppendDecision`, and the unexported `validate`/`oversizedDrafts`/ - `holdsDecisions`. Imports `analyze`, `session`, `timeline`. -- **`internal/review`** — gains the `-kind` dispatch (`Options.Kind`); its - findings path is unchanged except that `AppendVerdict` now calls - `session.AppendRecord`. +- **`internal/drafttests`** (new) — `Draft`, `Decision`, `Edit` and `Status` + types, `RubricVersion`, `IsDraftID`, `ParseDecisionFlag`, `Load`/`ParseRecords`, + `EffectiveStatus`, `Edit.Apply`, `SameIdentity`, `Window`, `EmitRequest`, + `Ingest`, `Render`, `Review`/`ReviewOptions` (the walk and the single-decision + path), `ParseEdit`, `AppendDecision`, the loud-staging sentinels + `ErrNoConfirmedFindings`/`ErrNoAcceptedDrafts`, and the unexported + `validate`/`oversizedDrafts`/`holdsDecisions`/`commitDrafts`. Imports `analyze`, + `session`, `timeline`. `ReviewOptions` mirrors `review.Options` field for field + except that the replacement fields arrive as `EditIn io.Reader` rather than a + decoded object: the CLI opens `-edit FILE` (or stdin for `-`) through + `session.OpenFileNoFollowRead` exactly as it opens `-ingest`, and `ParseEdit` + decodes it, so the no-follow guard sits at the same layer for both paths. + `Status` carries the winning decision's `Edit`, which is how the render reaches + "the last `edited` decision's edit" without a second pass over the decisions. +- **`internal/review`** — gains the `-kind` dispatch (`Options.Kind`, with + `Test`/`Decision`/`EditIn` alongside `Finding`/`Verdict`), the closed-set parser + `ParseKindFlag` and the `KindFindings`/`KindTests` constants, and the + cross-family flag refusals; `Run` delegates `-kind tests` to + `drafttests.Review`. Its findings path is otherwise unchanged except that + `AppendVerdict` now calls `session.AppendRecord`. - **`internal/analyze`** — `commitFindings` now calls `session.CommitRecords`; `maxAnswerBytes` moves to `session.MaxAnswerBytes` (16 MiB) beside - `MaxJSONLLine`/`MaxJSONLBytes`, where the shared caps already live. + `MaxJSONLLine`/`MaxJSONLBytes`, where the shared caps already live; and + `loadTimeline` is exported as `LoadTimeline` so `drafttests` can read the + timeline under the same refusals rather than duplicating them. - **`internal/session`** — gains `TestsFile = "tests.jsonl"`, `MaxAnswerBytes`, `Append`/`AppendRecord`, and `Commit`/`CommitRecords`. - **`internal/cli`** — the `draft-tests` case, `review`'s new flags, and the @@ -559,9 +680,11 @@ event window, the expected and observed behaviour, and the participant's quote.* **required**, and ingest refuses a draft missing any of them or whose quote is not the finding's quote byte-for-byte. - *Tests:* `TestEmitCarriesConfirmedFindingsAndWindows`, - `TestWindowSpansEvidenceWidenedByWindow`, `TestIngestRequiresSteps`, - `TestIngestRequiresExpectedAndObserved`, - `TestIngestRejectsQuoteThatIsNotTheFindingsQuote`, and the round-trip golden. + `TestWindowSpansEvidenceWidenedByWindow`, and — as subtests of the one + rule-per-case table `TestIngestValidationFailures` — + `absent_steps`, `empty_steps`, `whitespace-only_step`, `empty_expected`, + `empty_observed` and `quote_off_by_one_byte`; plus the round-trip golden + (`TestRoundTripGolden`). - **Flagged, as itd-2's AC3 was:** the CLI guarantees that a draft *contains* steps and that the request it came from carried *only* the event window. It cannot verify that a given step was in fact derived from the window — `steps` @@ -581,12 +704,12 @@ drafting step runs, then no test case is drafted for it.* hand-written or stale answer cannot smuggle one in. Effective status comes from `analyze.EffectiveStatus`, so a later verdict overriding an earlier one is honoured. -- *Tests:* `TestEmitOmitsUnverifiedRejectedAndDuplicateFindings`, - `TestIngestRejectsDraftOfUnverifiedFinding`, - `TestIngestRejectsDraftOfRejectedFinding`, - `TestIngestRejectsDraftOfDuplicateFinding`, - `TestEligibilityHonoursLastVerdict` (confirmed-then-rejected excluded, - rejected-then-confirmed included), `TestEmitRefusesWithNoConfirmedFindings`. +- *Tests:* `TestEmitOmitsUnverifiedRejectedAndDuplicateFindings`; the + `TestIngestValidationFailures` subtests `unverified_finding`, + `rejected_finding`, `duplicate_finding`, `mode_B_finding` and + `unknown_finding`; `TestEligibilityHonoursLastVerdict` + (confirmed-then-rejected excluded, rejected-then-confirmed included); and + `TestEmitRefusesWithNoConfirmedFindings`. **AC3** — *Given a drafted test case, when a human accepts or rejects it, then the decision is retained and the draft remains linked to its source finding and @@ -600,10 +723,10 @@ session.* `AppendDecision` re-checks the target under its lock. - *Tests:* `TestDecisionIsAppendedAndDraftLinesUnchanged` (byte-for-byte), `TestEditCannotNameFindingOrSessionOrSeverityOrQuoteOrID`, - `TestEffectiveStatusLastDecisionWins`, - `TestIngestRejectsSessionMismatch`, + `TestEffectiveStatusLastDecisionWins`, the `TestIngestValidationFailures` + subtest `session_mismatch`, `TestAppendDecisionRefusesWhenDraftChangedUnderTheLock`, and the interactive - walk tests for `a`/`e`/`r`. + walk tests for `a`/`e`/`r` (`TestInteractiveWalk`). **Scope bullet 4** — *Emitting the draft in a form the docs-as-code manual test records can hold.* Met by `draft-tests -render` (one Markdown test-case block per @@ -794,8 +917,12 @@ edit another, render the plan, and read it; fix what it exposes before the PR. 5 render the test plan and where to put it. Closes with pointers to the two reference pages. - `docs/README.md` — the new how-to added to the How-to guides line. -- `README.md` — "Status and roadmap": `draft-tests` moved into "working today", - and the regression-test bullet out of "Coming next". +- `docs/reference/cli.md` also updates its "Session directory inference" section + where it enumerates the commands that infer (five → six), since `draft-tests` + adopts `resolveSession` like the rest of the pipeline. +- `README.md` — "Status and roadmap": `draft-tests` moved into "working today". + ("Coming next" carries no regression-test bullet to remove — the three bullets + there are codebase mapping, reference capture, and the macOS app.) **Durable record (`.abcd/development/`, not user-facing).** diff --git a/.abcd/work/issues/open/iss-2609120532387596-abcd-s-intent-auditor-agent-emits-verdict-json-in-shapes-tha.md b/.abcd/work/issues/open/iss-2609120532387596-abcd-s-intent-auditor-agent-emits-verdict-json-in-shapes-tha.md new file mode 100644 index 0000000..862e9ae --- /dev/null +++ b/.abcd/work/issues/open/iss-2609120532387596-abcd-s-intent-auditor-agent-emits-verdict-json-in-shapes-tha.md @@ -0,0 +1,14 @@ +--- +schema_version: 1 +id: "iss-2609120532387596" +slug: "abcd-s-intent-auditor-agent-emits-verdict-json-in-shapes-tha" +severity: "minor" +category: "observation" +source: "user-observation" +found_during: "itd-9-fidelity-audit" +origin: researcher-authored +production_mode: hand-written +found_at: ".abcd/.work.local/reviews" +--- + +abcd's intent-auditor agent emits verdict JSON in shapes that 'abcd intent audit ingest' dead-letters: two runs in one session added a 'quote' field to evidence entries and used {file,line} instead of {ref}; the ingest's DisallowUnknownFields is right, but the agent definition should carry the exact evidence shape (or the ingest should print the expected schema on dead-letter) so a host does not have to hand-convert diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d0e93a..2b9c775 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,11 @@ jobs: - name: Pipeline smoke test run: | set -euo pipefail + # draft-tests -render first: it reads manifest.json, findings.jsonl and + # tests.jsonl only, so it needs no merged timeline and proves the bundled + # tests.jsonl still renders the accepted plan before merge writes anything. + ./testimony draft-tests -render -session examples/sample-session | tee /tmp/tests-plan.md + grep -q "T-001" /tmp/tests-plan.md ./testimony merge -session examples/sample-session ./testimony report -session examples/sample-session test -s examples/sample-session/timeline.jsonl @@ -141,6 +146,11 @@ jobs: # updating them here and in release.yml. grep -q "\*\*Utterances:\*\* 10 · \*\*Events:\*\* 10" examples/sample-session/report.md grep -q "data-testid=save-btn" examples/sample-session/report.md + # The drafting request needs the merged timeline (each confirmed finding + # carries its event window), so it runs after merge: exit 0 proves the + # sample still has a confirmed finding to draft from. + ./testimony draft-tests -session examples/sample-session > /tmp/tests-request.md + grep -q "testimony-testdraft/v1" /tmp/tests-request.md # The counts line above catches events going missing from the merge, # but not a windowing/attachment regression: report's header counts are # raw entry counts, computed before the join, so they stay "10 · 10" diff --git a/AGENTS.md b/AGENTS.md index 6dd6d64..565465b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,11 +50,13 @@ Testimony captures usability evidence, on the record. ## Current state -v0.4.0. A Go CLI (`testimony`, standard library only) whose seven pipeline +v0.4.0. A Go CLI (`testimony`, standard library only) whose eight pipeline commands are all implemented and dispatched from `internal/cli` behind the `cmd/testimony` entry point: `record` and `demo` -(capture), `transcribe`, `merge`, `report`, and the analysis layer `analyze` -and `review` — plus `version` and `help`. The model work is host-delegated — +(capture), `transcribe`, `merge`, `report`, the analysis layer `analyze` +and `review`, and the regression-test drafting layer `draft-tests` (with +`review -kind tests` for its human pass) — plus `version` and `help`. The model +work is host-delegated — the CLI never calls a model, holds no keys, and adds no network dependency. The user-facing documentation is [`docs/README.md`](docs/README.md); the exact command and file contracts are [`docs/reference/cli.md`](docs/reference/cli.md) and @@ -73,8 +75,10 @@ go vet ./... # static checks go test ./... # unit tests go test -race ./... # race-enabled go test -run TestEventsNearWindow ./internal/timeline/ # a single test +./testimony draft-tests -render -session examples/sample-session # needs no timeline ./testimony merge -session examples/sample-session # pipeline smoke: ./testimony report -session examples/sample-session # writes timeline.jsonl + report.md +./testimony draft-tests -session examples/sample-session # emit the drafting request (after merge) sh -n install.sh && bash -n install.sh # installer syntax ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 06e72b0..4519ca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,10 @@ break an existing invocation is called out in the entry that records it. write). `analyze -ingest` and `testimony review` write through them with no change in behaviour: the two size refusals and the verdict-overwrite guard are byte-for-byte the messages they always were, and the drafting layer shares the - primitives rather than carrying a second copy of a subtle write path. + primitives rather than carrying a second copy of a subtle write path. One + message does change — a failure while writing `findings.jsonl` is now prefixed + `write findings.jsonl:` rather than `write findings:`, so every failure on that + path names the file the same way. - `transcribe` prints an elapsed-time status line every 5 seconds while the ASR engine is still running, instead of staying silent between the offset line and completion — a CPU-only `whisperx`/`whisper-cli` run can take diff --git a/docs/reference/cli.md b/docs/reference/cli.md index c94865d..e9ccefd 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -16,7 +16,7 @@ Running `testimony` with no command, or with an unknown command, prints the usag ## Session directory inference -The five pipeline commands — `transcribe`, `merge`, `report`, `analyze`, and `review` — take their session directory from `-session DIR`. When `-session` is omitted and the current directory itself holds a Testimony session `manifest.json`, that directory is the session: the command operates on it exactly as `-session .` does, and prints one line to stderr naming what it inferred (`merge: using session . (inferred from the current directory)`) before it starts work, so the implicit choice is visible in the output of the run. The line goes to stderr, never stdout, so `analyze`'s emitted request stays a clean pipe. An explicit `-session` always wins, is used verbatim, and prints no such line — the current directory is not consulted at all. +The six pipeline commands — `transcribe`, `merge`, `report`, `analyze`, `draft-tests`, and `review` — take their session directory from `-session DIR`. When `-session` is omitted and the current directory itself holds a Testimony session `manifest.json`, that directory is the session: the command operates on it exactly as `-session .` does, and prints one line to stderr naming what it inferred (`merge: using session . (inferred from the current directory)`) before it starts work, so the implicit choice is visible in the output of the run. The line goes to stderr, never stdout, so `analyze`'s emitted request stays a clean pipe. An explicit `-session` always wins, is used verbatim, and prints no such line — the current directory is not consulted at all. The marker is a session manifest, not merely the file name. `manifest.json` is one of the most common file names in software, so the file must be a regular file (a directory or a symlink at that name is not a marker) and, when it parses, must carry the `session` field every `testimony` session has (see [`manifest.json`](session-directory.md#manifestjson)). A `manifest.json` that belongs to something else leaves the command refusing rather than writing into a directory that is not a session: @@ -177,20 +177,71 @@ Emit behaviour: writes a single self-contained prompt — the rubric version hea Ingest behaviour: reads the answer from `FILE` (or stdin when `-`), accepting a top-level object with a `findings` array (optionally a `rubric`, which must be a known version) or a bare array. Ingest is the sole validation boundary and never trusts the model. Each finding is decoded with unknown fields disallowed, then checked against every schema rule (see [session directory reference](session-directory.md#findingsjsonl)): id format and uniqueness, `t` within the session, the `type`, `severity`, and `mode` enums, non-empty `evidence` of at most 64 ids with every id real and at least one spoken `utt-*` anchor, a `quote` that is a verbatim substring of one *cited* evidence utterance, and any `ui` selector/route matching a real event. Validation is transactional — all errors are reported at once and nothing is written on any failure. On success every finding is forced to `status: unverified`, `findings.jsonl` is written, and the command prints `validated N findings → (all unverified)`. An answer with no findings (a bare `[]`, `{"findings":[]}`, or a truncated file) is refused rather than written, so it cannot erase a prior `findings.jsonl`; an answer whose findings would together push `findings.jsonl` past the session's 16 MiB total-size limit is refused the same way (see [`session-directory.md`](session-directory.md)). Ingest refuses to overwrite a `findings.jsonl` that already holds verdict records — counting any `kind:"verdict"` line, even one whose value is outside the closed enum. +## `testimony draft-tests` + +The regression-test drafting layer. Like `analyze`, `draft-tests` never calls a model, holds no keys, and adds no network dependency: it *emits* a self-contained drafting request that any assistant (or a human) runs, then *ingests* and validates the JSON answer into `tests.jsonl`. A third mode *renders* the accepted drafts as Markdown test cases. The step sits downstream of verification, so only a finding a person already confirmed can be drafted from. + +``` +testimony draft-tests [-session DIR] [-window 10] [-out FILE] # emit the request +testimony draft-tests [-session DIR] -ingest FILE # validate the answer → tests.jsonl +testimony draft-tests [-session DIR] -render [-out FILE] # render the accepted drafts +``` + +| Flag | Default | Meaning | +|---|---|---| +| `-session` | *(inferred)* | session directory; when omitted, the current directory if it holds a Testimony session `manifest.json` (see [session directory inference](#session-directory-inference)) | +| `-window` | `10` | emit mode: the event-window half-width in seconds around each finding's cited evidence | +| `-out` | *(stdout)* | emit or render mode: write the document to `FILE` instead of stdout | +| `-ingest` | *(off)* | ingest mode: validate the answer JSON at `FILE` (or `-` for stdin) into `tests.jsonl` | +| `-render` | *(off)* | render mode: write Markdown test cases for the accepted drafts | + +`draft-tests` runs in exactly one mode: emit (neither `-ingest` nor `-render`), ingest (`-ingest`), or render (`-render`). `-ingest` combines with neither `-out` nor `-render`; `-out` pairs with emit or render. `-window` belongs to emit alone — ingest validates against the findings and render reads only what is on disk — so passing it with `-ingest` or `-render` is a usage error rather than a silently ignored flag. A non-finite `-window` is a usage error too, as it is for `report`; a negative one is legitimate and narrows the window. + +Emit reads `manifest.json`, `findings.jsonl`, and `timeline.jsonl`; ingest reads `manifest.json` and `findings.jsonl` only, because drafts are validated against the *findings* rather than re-derived from the timeline; render reads `manifest.json`, `findings.jsonl`, and `tests.jsonl`. Emit hints to run `merge` first when the timeline is missing; every mode hints to run `analyze -ingest` first when there is no `findings.jsonl`, and ingest, render, and `review -kind tests` hint to run `draft-tests -ingest` first when there is no `tests.jsonl`. + +**Eligibility.** A finding may be drafted from when its effective status is `confirmed` and its `mode` is not `B`. Effective status is the same computation `review` and `report` use, so a later verdict overriding an earlier one is honoured: a finding confirmed and then rejected is not eligible, and one rejected and then confirmed is. `unverified`, `rejected`, and `duplicate` findings are never eligible — including a `duplicate` whose target is confirmed, since the canonical finding carries the evidence. The finding's `type` is *not* filtered: the request carries it so the model can calibrate, and a draft with nothing to regress against is what the reject decision is for. + +Emit behaviour: writes a single self-contained prompt — the rubric version header (`testimony-testdraft/v1`), the proposal stance, the per-field instructions, the rubric body (the field definitions, how to read each finding record, and the hard constraints ingest enforces), the session context (session, app, participant, tasks), then, per eligible finding in id order, a prose header naming its id, type, severity, clock and the date of the verdict that confirmed it, its own record verbatim in a ```jsonl fence, and its event window in a second fence, followed by the required output shape with a worked example. Nothing in the session directory is mutated. With `-out FILE` the prompt goes to a file and the command prints `wrote `; otherwise it prints to stdout. + +The **event window** is the only material the steps may be reconstructed from. For each finding it is every timeline entry whose time falls between the earliest cited evidence entry's start minus `-window` and the latest cited entry's end plus `-window`, in time order, speech and events together — the utterances carry the expected behaviour and the events carry the steps. The default half-width is 10 seconds rather than `report`'s 2.5, because a reproduction needs the lead-up and the aftermath and not only the moment: on the bundled sample, 2.5 seconds excludes the one utterance in which the participant states what they expected. A finding whose evidence resolves to no entry falls back to the window around its own `t`. The steps are stated to end at the last cited evidence event at or before the finding's `t`; a cited event after it belongs in `observed`. + +Ingest behaviour: reads the answer from `FILE` (or stdin when `-`), accepting a top-level object with a `tests` array (optionally a `rubric`, which must be a known version) or a bare array. Ingest is the sole validation boundary and never trusts the model. Each draft is decoded with unknown fields disallowed, then checked against every schema rule (see [session directory reference](session-directory.md#testsjsonl)): id format and uniqueness, a `finding` that is currently confirmed and not mode `B`, a `session` equal to the manifest's, a non-empty `title` of at most 200 characters, non-empty `steps` of at most 32 non-empty entries, non-empty `expected` and `observed`, a `rationale_quote` equal to the source finding's `quote` byte for byte, and a `severity` equal to the source finding's. Validation is transactional — all errors are reported at once and nothing is written on any failure. On success every draft is forced to `status: proposed`, `tests.jsonl` is written, and the command prints `validated N test drafts → (all proposed)`. An answer with no drafts (a bare `[]`, `{"tests":[]}`, or a truncated file) is refused rather than written, so it cannot erase a prior `tests.jsonl`; so is an answer whose drafts would together push `tests.jsonl` past the session's 16 MiB total-size limit. Ingest refuses to overwrite a `tests.jsonl` that already holds decision records — counting any `kind:"decision"` line, even one whose value is outside the closed enum. + +Render behaviour: writes one Markdown test-case block per draft whose effective status is `accepted` or `edited`, in id order, with the latest `edited` decision's fields applied over the draft. `proposed` and `rejected` drafts are omitted: a proposal is not a test, and a rejected draft stays in `tests.jsonl` for the record rather than for the plan. Each block names the source finding and session, the decision and its date, the numbered steps, the expected and observed behaviour, and the participant's quote as the rationale. With `-out FILE` the plan goes to a file and the command prints `wrote `; otherwise it prints to stdout, which is the default because Testimony never writes into the application's repository — where a docs-as-code test plan lives is the operator's choice. + +**Loud staging.** Two states are refused at exit 1 — a well-formed invocation whose work cannot be done — with the counts by status and nothing written: + +``` +testimony: no confirmed findings to draft tests from (5 findings: 0 confirmed, 2 unverified, 1 duplicate, 2 rejected); confirm one with `testimony review -session sessions/x` first +testimony: no accepted test drafts to render (3 drafts: 0 accepted, 0 edited, 2 proposed, 1 rejected); accept one with `testimony review -session sessions/x -kind tests` first +``` + +The first applies to emit and to ingest, and on ingest it fires before a byte of the answer is read: with no eligible finding, every draft in the answer would fail the same rule. The second keeps `-out FILE` from truncating an existing test plan into an empty document. + ## `testimony review` -Records a human verdict on each candidate finding, appended to `findings.jsonl` without ever rewriting a finding in place — the finding's birth state and the full verdict history are retained as the precision measure. +The one human-decision verb, across both record families. With `-kind findings` (the default) it records a verdict on each candidate finding; with `-kind tests` it records an accept / edit / reject decision on each drafted regression test. Either way the decision is *appended* — to `findings.jsonl` or to `tests.jsonl` — without ever rewriting the machine record in place, so the record's birth state and the full decision history are retained as the precision measure. ``` -testimony review [-session DIR] +testimony review [-session DIR] [-kind findings|tests] testimony review [-session DIR] -finding F-NNN -verdict confirmed|rejected|duplicate-of-F-NNN +testimony review [-session DIR] -kind tests -test T-NNN -decision accepted|rejected +testimony review [-session DIR] -kind tests -test T-NNN -decision edited -edit FILE ``` | Flag | Default | Meaning | |---|---|---| | `-session` | *(inferred)* | session directory; when omitted, the current directory if it holds a Testimony session `manifest.json` (see [session directory inference](#session-directory-inference)) | -| `-finding` | *(interactive)* | non-interactive: the finding to judge (`F-NNN`) | -| `-verdict` | *(interactive)* | non-interactive: `confirmed`, `rejected`, or `duplicate-of-F-NNN` | +| `-kind` | `findings` | which record family to review: `findings` or `tests` | +| `-finding` | *(interactive)* | non-interactive: the finding to judge (`F-NNN`), `-kind findings` only | +| `-verdict` | *(interactive)* | non-interactive: `confirmed`, `rejected`, or `duplicate-of-F-NNN`, `-kind findings` only | +| `-test` | *(interactive)* | non-interactive: the test draft to decide (`T-NNN`), `-kind tests` only | +| `-decision` | *(interactive)* | non-interactive: `accepted`, `edited`, or `rejected`, `-kind tests` only | +| `-edit` | *(off)* | with `-decision edited`: the replacement fields as a JSON object at `FILE` (or `-` for stdin) | + +A flag belonging to the other record family is a usage error, not a silently ignored value: `-finding` or `-verdict` with `-kind tests`, and `-test`, `-decision` or `-edit` with `-kind findings`, each exit 2. So do an unknown `-kind`, a `-test` that is not `T-NNN`, a `-decision` outside the enum, `-decision edited` without `-edit`, and `-edit` alongside any other decision. + +### `-kind findings` (the default) Behaviour: loads findings and existing verdicts (hinting to run `analyze -ingest` first when there is no `findings.jsonl`) and computes each finding's effective status (every finding starts `unverified`; the last verdict for a finding wins). @@ -198,6 +249,20 @@ Interactive (`review -session DIR`): walks the `unverified` findings in id order Non-interactive (`-finding F-003 -verdict confirmed`, or `-verdict duplicate-of-F-002`): validates that the finding exists, the verdict parses, and any duplicate target exists and differs; appends one verdict record and prints a one-line confirmation. A verdict may be appended even when one already exists (append-only correction; the latest wins), unless appending it would push `findings.jsonl` past the session's 16 MiB total-size limit, which both interactive and non-interactive `review` refuse (see [`session-directory.md`](session-directory.md)). The stored verdict enum is exactly `confirmed | rejected | duplicate`; `duplicate-of-F-NNN` is stored as `verdict: "duplicate"` with `of: "F-NNN"`. +### `-kind tests` + +Behaviour: loads the test drafts and existing decisions (hinting to run `draft-tests -ingest` first when there is no `tests.jsonl`) and computes each draft's effective status — every draft starts `proposed`, decision records apply in file order, and the last one for a draft wins. A decision naming an unknown draft, or carrying a value outside the closed enum, is ignored rather than applied, so a draft never vanishes from both the walk and the plan. + +Interactive (`review -session DIR -kind tests`): walks the `proposed` drafts in id order, printing each draft's id, its source finding with that finding's type, severity and clock, the title, the numbered steps, the expected and observed behaviour, and the participant's quote, then prompting `[a]ccept [e]dit [r]eject [s]kip [q]uit`. `e` asks for each editable field in turn showing the current value; a blank answer keeps it, and `steps` are read one per line until a blank line. A pass through the prompts that changes nothing prints `no changes; recorded as accepted.` and records `accepted`, because an `edited` decision with an empty edit records a change that did not happen. The character-device gate is the same as the findings walk's. + +Non-interactive (`-kind tests -test T-001 -decision accepted`): validates that the draft exists and the decision parses, appends one decision record, and prints `recorded: T-001 accepted ()`. `-decision edited` requires `-edit FILE` (or `-` for stdin), a JSON object holding the replacement fields — a subset of `title`, `steps`, `expected`, and `observed` with at least one member, each held to the draft's own rule for that field: + +```json +{"title":"Saving a display name gives no confirmation","steps":["Open #general.","Click Save."]} +``` + +The `edit` object is closed: one naming `id`, `finding`, `session`, `severity`, or `rationale_quote` is an error rather than a silently dropped key, so no edit can re-point a draft at different evidence. The only way to change the link is to reject the draft and ingest a new one. A decision may be appended even when one already exists (append-only correction; the latest wins), and the accepted and edited drafts are what [`draft-tests -render`](#testimony-draft-tests) puts in the test plan. + ## `testimony version` Prints `testimony ` — the version stamped at release, or `dev`. diff --git a/docs/reference/session-directory.md b/docs/reference/session-directory.md index dfe2252..13c6b89 100644 --- a/docs/reference/session-directory.md +++ b/docs/reference/session-directory.md @@ -157,7 +157,7 @@ A finding's effective status starts `unverified`; verdict records apply in file The regression-test drafting layer's output, written by `testimony draft-tests -ingest` and appended to by `testimony review -kind tests`. Two record kinds share the file, one per line: a **draft** line (no `kind` field) and a **decision** line (`kind: "decision"`). Decisions are appended, never written in place, so a draft's original state and the full decision history are retained. Blank lines are ignored. -Ingest validates every draft against `findings.jsonl` and is the sole validation boundary — it never trusts the model. Unknown fields are rejected (the shape is closed), and `status` is forced to `"proposed"` on ingest regardless of the answer JSON, so a draft can never be born accepted. A draft may only ever reference a finding whose effective status is `confirmed`. +Ingest validates every draft against `findings.jsonl` and is the sole validation boundary — it never trusts the model. Each draft object is closed: an unknown field in one is rejected rather than dropped, as is an unknown field in a decision's `edit` object. The top-level answer container is not closed — a key beside `rubric` and `tests` is tolerated, mirroring `analyze` — so strictness lands on the records themselves. `status` is forced to `"proposed"` on ingest regardless of the answer JSON, so a draft can never be born accepted. A draft may only ever reference a finding whose effective status is `confirmed`. **Draft record** @@ -172,7 +172,7 @@ Ingest validates every draft against `findings.jsonl` and is the sole validation | `observed` | string | yes | what the system did; non-empty | | `rationale_quote` | string | yes | **equal** to the source finding's `quote`, byte for byte — the drafting step carries evidence forward and never introduces any | | `severity` | integer | yes | **equal** to the source finding's `severity`, so triage order survives the hand-off unaltered | -| `status` | string | no | always `"proposed"` on ingest, whatever the answer claims | +| `status` | string | yes | always `"proposed"` on ingest, whatever the answer claims (the answer may omit it; the written record always carries it) | ```json {"id":"T-001","finding":"F-001","session":"sample-session","title":"Saving gives no confirmation","steps":["Open #general in the settings prototype.","Change the display name to Alice.","Click the Save button ([data-testid=save-btn])."],"expected":"The save is confirmed on screen — a toast, or the button briefly disabled.","observed":"Nothing visibly changes, so there is no way to tell the save landed.","rationale_quote":"I clicked save and nothing happened","severity":3,"status":"proposed"} diff --git a/internal/analyze/ingest.go b/internal/analyze/ingest.go index 12c0679..862def3 100644 --- a/internal/analyze/ingest.go +++ b/internal/analyze/ingest.go @@ -169,7 +169,7 @@ func commitFindings(dir string, findings []Finding) error { for _, f := range findings { b, err := json.Marshal(f) if err != nil { - return fmt.Errorf("write findings: %w", err) + return fmt.Errorf("write %s: %w", session.FindingsFile, err) } records = append(records, b) } diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 8338ad6..461cb12 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -4,6 +4,7 @@ package cli import ( "flag" "fmt" + "io" "math" "os" "path/filepath" @@ -12,6 +13,7 @@ import ( "github.com/REPPL/Testimony/internal/analyze" "github.com/REPPL/Testimony/internal/demo" + "github.com/REPPL/Testimony/internal/drafttests" "github.com/REPPL/Testimony/internal/record" "github.com/REPPL/Testimony/internal/report" "github.com/REPPL/Testimony/internal/review" @@ -36,15 +38,20 @@ Usage: testimony report [-session DIR] [-window 2.5] render timeline.jsonl as a Markdown report testimony analyze [-session DIR] [-out FILE] emit the analysis request (rubric + timeline) on stdout or to FILE testimony analyze [-session DIR] -ingest FILE validate answer JSON (FILE or "-") → findings.jsonl (all findings unverified) - testimony review [-session DIR] interactively record verdicts on unverified findings (stdin must be a character device) + testimony draft-tests [-session DIR] [-window 10] [-out FILE] emit the regression-test drafting request (rubric + confirmed findings + event windows) + testimony draft-tests [-session DIR] -ingest FILE validate answer JSON (FILE or "-") → tests.jsonl (all drafts proposed) + testimony draft-tests [-session DIR] -render [-out FILE] render the accepted drafts as Markdown test cases + testimony review [-session DIR] [-kind findings|tests] interactively record verdicts on unverified findings, or decisions on proposed test drafts (stdin must be a character device) testimony review [-session DIR] -finding F-NNN -verdict confirmed|rejected|duplicate-of-F-NNN + testimony review [-session DIR] -kind tests -test T-NNN -decision accepted|rejected + testimony review [-session DIR] -kind tests -test T-NNN -decision edited -edit FILE testimony version testimony help A session directory is described in docs/reference/session-directory.md. -Omitting -session on transcribe, merge, report, analyze, or review uses the -current directory when it holds a Testimony session manifest.json (one with a -session field), and names the inferred session on stderr. +Omitting -session on transcribe, merge, report, analyze, draft-tests, or review +uses the current directory when it holds a Testimony session manifest.json (one +with a session field), and names the inferred session on stderr. ` // Run executes the CLI and returns a process exit code. @@ -394,23 +401,156 @@ func Run(args []string) int { fmt.Print(prompt) return 0 + case "draft-tests": + fs := flag.NewFlagSet("draft-tests", flag.ExitOnError) + dir := fs.String("session", "", "session directory") + window := fs.Float64("window", 10, "emit mode: event-window half-width around a finding's evidence, seconds") + out := fs.String("out", "", "emit/render mode: write to FILE instead of stdout") + ingest := fs.String("ingest", "", "validate answer JSON at FILE (or \"-\" for stdin) into tests.jsonl") + render := fs.Bool("render", false, "render the accepted drafts as Markdown test cases") + fs.Parse(rest) + if err := rejectArgs(fs); err != nil { + return usageErr(err) + } + outSet, ingestSet, windowSet := false, false, false + fs.Visit(func(f *flag.Flag) { + switch f.Name { + case "out": + outSet = true + case "ingest": + ingestSet = true + case "window": + windowSet = true + } + }) + // An explicitly-empty -ingest or -out is a wrong invocation (an unset shell + // variable spliced into the flag, say), not a valid path — analyze's + // identical guard above. Left unchecked, an empty -ingest falls through the + // mode check below into emit at exit 0 (the answer is never validated), and + // an empty -out falls through to stdout at exit 0 instead of writing a file. + if ingestSet && *ingest == "" { + return usageErr(fmt.Errorf("draft-tests: -ingest must not be empty")) + } + if outSet && *out == "" { + return usageErr(fmt.Errorf("draft-tests: -out must not be empty")) + } + // draft-tests runs in exactly one mode, extending analyze's emit-or-ingest + // rule by one: emit (neither), ingest (-ingest), or render (-render). + // -ingest writes tests.jsonl and produces no document, so neither -out nor + // -render has any meaning alongside it; a caller who combined them meant one + // of the two modes and must be told which they cannot have. + if *ingest != "" { + if *out != "" { + return usageErr(fmt.Errorf("draft-tests: -out and -ingest cannot be combined")) + } + if *render { + return usageErr(fmt.Errorf("draft-tests: -render and -ingest cannot be combined")) + } + } + // -window sizes the event window emit puts in the request; ingest validates + // against the findings and render reads only what is already on disk, so in + // neither mode does it do anything. Silently ignored, it lets a caller who + // meant to widen the window believe they had — refusing names the mode they + // are actually in, the same class as the -out/-ingest combination above. + if windowSet && (*ingest != "" || *render) { + return usageErr(fmt.Errorf("draft-tests: -window applies to the emit mode only")) + } + // A non-finite window is not a window at all, and Window has no way to + // notice: every comparison against NaN is false, so a NaN window emits an + // empty event window for every finding — a request whose steps cannot be + // grounded in anything — while +Inf emits the whole timeline as every + // finding's window. Either way the drafting request misstates its own + // evidence and the command exits 0. A negative window is legitimate (it + // narrows the window), so only finiteness is required — the report -window + // precedent. + if math.IsNaN(*window) || math.IsInf(*window, 0) { + return usageErr(fmt.Errorf("draft-tests: -window must be a finite number of seconds, got %v", *window)) + } + // Resolved last of the invocation checks (see report above): a run refused + // for another flag must not first announce an inferred session. + sess, err := resolveSession(fs, *dir) + if err != nil { + return usageErr(err) + } + if *ingest != "" { + in := os.Stdin + if *ingest != "-" { + // Read the answer through the no-follow guard, like analyze -ingest: the + // operator naturally saves the model's answer beside the session, and a + // received session can ship a FIFO at that name (plain os.Open blocks in + // open(2) for ever) or a symlink out of the directory. + f, err := session.OpenFileNoFollowRead(*ingest) + if err != nil { + return fail(err) + } + defer f.Close() + in = f + } + // Both loud-staging refusals (no confirmed finding to draft from, no + // accepted draft to render) are well-formed invocations whose work cannot + // be done, so they take the runtime status here rather than the usage one. + drafts, err := drafttests.Ingest(sess, in) + if err != nil { + return fail(err) + } + fmt.Printf("validated %d test drafts → %s (all proposed)\n", + len(drafts), filepath.Join(sess, session.TestsFile)) + return 0 + } + var doc string + if *render { + doc, err = drafttests.Render(sess) + } else { + doc, err = drafttests.EmitRequest(sess, *window) + } + if err != nil { + return fail(err) + } + if *out != "" { + // Write through the no-follow guard, matching analyze -out: the operator + // naturally directs -out at a path beside the session, and a received + // session can ship a symlink there that plain os.WriteFile would follow, + // truncating an arbitrary operator-writable file outside the session. + if err := session.WriteFileNoFollow(*out, []byte(doc), 0o644); err != nil { + return fail(err) + } + fmt.Printf("wrote %s\n", *out) + return 0 + } + fmt.Print(doc) + return 0 + case "review": fs := flag.NewFlagSet("review", flag.ExitOnError) dir := fs.String("session", "", "session directory") + kind := fs.String("kind", review.KindFindings, "which record family to review: findings | tests") finding := fs.String("finding", "", "non-interactive: the finding to judge (F-NNN)") verdict := fs.String("verdict", "", "non-interactive: confirmed | rejected | duplicate-of-F-NNN") + test := fs.String("test", "", "non-interactive (-kind tests): the test draft to decide (T-NNN)") + decision := fs.String("decision", "", "non-interactive (-kind tests): accepted | edited | rejected") + edit := fs.String("edit", "", "with -decision edited: the replacement fields as a JSON object at FILE (or \"-\" for stdin)") fs.Parse(rest) if err := rejectArgs(fs); err != nil { return usageErr(err) } f, v := strings.TrimSpace(*finding), strings.TrimSpace(*verdict) + tst, dec := strings.TrimSpace(*test), strings.TrimSpace(*decision) findingSet, verdictSet := false, false + kindSet, testSet, decisionSet, editSet := false, false, false, false fs.Visit(func(fl *flag.Flag) { switch fl.Name { case "finding": findingSet = true case "verdict": verdictSet = true + case "kind": + kindSet = true + case "test": + testSet = true + case "decision": + decisionSet = true + case "edit": + editSet = true } }) // An explicitly-empty -finding or -verdict is a wrong invocation (an @@ -425,6 +565,69 @@ func Run(args []string) int { if verdictSet && v == "" { return usageErr(fmt.Errorf("review: -verdict must not be empty")) } + // The tests side gets the identical unset-shell-variable guard on each of its + // own flags: set-but-empty is indistinguishable from omitted, so it would + // otherwise fall through to the interactive walk at exit 0 rather than refuse + // the caller's mistake. -edit is a path, so it is not trimmed (matching + // -ingest/-out); -kind, -test and -decision are identifiers and are. + if kindSet && strings.TrimSpace(*kind) == "" { + return usageErr(fmt.Errorf("review: -kind must not be empty")) + } + if testSet && tst == "" { + return usageErr(fmt.Errorf("review: -test must not be empty")) + } + if decisionSet && dec == "" { + return usageErr(fmt.Errorf("review: -decision must not be empty")) + } + if editSet && *edit == "" { + return usageErr(fmt.Errorf("review: -edit must not be empty")) + } + // The record family is a closed set, so an unknown one is a wrong invocation + // rather than a silently-ignored value that would run the findings walk under + // a name the caller did not mean. + recordKind, err := review.ParseKindFlag(strings.TrimSpace(*kind)) + if err != nil { + return usageErr(fmt.Errorf("review: %w", err)) + } + // A flag belonging to the other record family is a wrong invocation, not a + // silently ignored one: a caller who typed -verdict against -kind tests meant + // a decision this walk cannot record, and recording nothing while exiting 0 + // would let a script believe it landed. review.Run refuses the same pairings, + // so the rule holds for any caller; refusing here gives it the usage status + // and does so before the session is resolved or any file is read. + if recordKind == review.KindTests { + if f != "" || v != "" { + return usageErr(fmt.Errorf("review: -finding and -verdict apply to -kind findings, not -kind tests")) + } + } else if tst != "" || dec != "" || *edit != "" { + return usageErr(fmt.Errorf("review: -test, -decision and -edit apply to -kind tests, not -kind findings")) + } + // The -test/-decision pairing, the draft id's syntax, the decision enum, and + // -edit's pairing are all invocation facts, so they are refused here at the + // usage status rather than from inside the package after the drafts load. + if tst != "" && dec == "" { + return usageErr(fmt.Errorf("review: -decision is required with -test")) + } + if dec != "" && tst == "" { + return usageErr(fmt.Errorf("review: -test is required with -decision")) + } + if tst != "" && !drafttests.IsDraftID(tst) { + return usageErr(fmt.Errorf("review: invalid -test %q (want T-NNN)", tst)) + } + if dec != "" { + if _, err := drafttests.ParseDecisionFlag(dec); err != nil { + return usageErr(fmt.Errorf("review: %w", err)) + } + } + // An "edited" decision with no replacement fields is not representable, and + // an -edit alongside any other decision would be silently discarded, so each + // half of the pairing is refused from the flags alone. + if dec == "edited" && *edit == "" { + return usageErr(fmt.Errorf("review: -edit is required with -decision edited")) + } + if *edit != "" && dec != "edited" { + return usageErr(fmt.Errorf("review: -edit applies only to -decision edited")) + } // The -finding/-verdict pairing and the verdict's syntax are invocation // facts, so they are refused here at the usage status — reported from // review.Run they exited 1, and only after the findings load, so a wrong @@ -461,14 +664,34 @@ func Run(args []string) int { if err != nil { return usageErr(err) } + // Read the replacement fields through the no-follow guard, like every other + // operator-named path on a session surface: the edit is naturally saved beside + // the session, and a received session can ship a FIFO or a symlink at that name. + var editIn io.Reader + if *edit != "" { + if *edit == "-" { + editIn = os.Stdin + } else { + ef, err := session.OpenFileNoFollowRead(*edit) + if err != nil { + return fail(err) + } + defer ef.Close() + editIn = ef + } + } if err := review.Run(review.Options{ - Dir: sess, - Finding: f, - Verdict: v, - In: os.Stdin, - Out: os.Stdout, - IsTTY: isCharDevice(os.Stdin), - Today: time.Now().Format("2006-01-02"), + Dir: sess, + Kind: recordKind, + Finding: f, + Verdict: v, + Test: tst, + Decision: dec, + EditIn: editIn, + In: os.Stdin, + Out: os.Stdout, + IsTTY: isCharDevice(os.Stdin), + Today: time.Now().Format("2006-01-02"), }); err != nil { return fail(err) } @@ -511,8 +734,8 @@ func rejectArgs(fs *flag.FlagSet) error { // resolveSession returns the session directory a pipeline command operates on: // the explicit -session flag when it is given, otherwise the current directory // when that directory itself holds a manifest.json. It is the single resolution -// point for transcribe, merge, report, analyze, and review, so the five commands -// cannot drift in what they accept. +// point for transcribe, merge, report, analyze, draft-tests, and review, so the +// six commands cannot drift in what they accept. // // Inference covers the exact current directory only — never a parent, the way // git searches upward for .git — because a command that operated on an ancestor diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index ebd1fdd..9d6b1a0 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -174,6 +174,7 @@ func TestStrayPositionalIsAUsageError(t *testing.T) { {"transcribe", "-session", dir, "junk", "-offset", "99"}, {"analyze", "-session", dir, "junk", "-out", "x", "-ingest", "-"}, {"review", "-session", dir, "junk", "-finding", "F-001", "-verdict", "confirmed"}, + {"draft-tests", "-session", dir, "junk", "-render"}, {"record", "-out", t.TempDir(), "junk", "-participant", "P9"}, {"version", "junk"}, {"help", "junk"}, @@ -254,6 +255,28 @@ func TestInvalidFlagValuesExitTwo(t *testing.T) { {[]string{"review", "-session", dir, "-finding", "", "-verdict", ""}, `review: -finding must not be empty`}, {[]string{"review", "-session", dir, "-finding", "F-001", "-verdict", ""}, `review: -verdict must not be empty`}, {[]string{"review", "-session", dir, "-finding", "F-001", "-verdict", "duplicate-of-F-001"}, `review: -finding cannot be a duplicate of itself`}, + {[]string{"draft-tests", "-session", dir, "-ingest", ""}, `draft-tests: -ingest must not be empty`}, + {[]string{"draft-tests", "-session", dir, "-out", ""}, `draft-tests: -out must not be empty`}, + {[]string{"draft-tests", "-session", dir, "-out", "f.md", "-ingest", "-"}, `draft-tests: -out and -ingest cannot be combined`}, + {[]string{"draft-tests", "-session", dir, "-render", "-ingest", "-"}, `draft-tests: -render and -ingest cannot be combined`}, + {[]string{"draft-tests", "-session", dir, "-window", "NaN"}, `draft-tests: -window must be a finite number of seconds`}, + {[]string{"draft-tests", "-session", dir, "-window", "+Inf"}, `draft-tests: -window must be a finite number of seconds`}, + {[]string{"draft-tests", "-session", dir, "-window", "20", "-ingest", "-"}, `draft-tests: -window applies to the emit mode only`}, + {[]string{"draft-tests", "-session", dir, "-window", "20", "-render"}, `draft-tests: -window applies to the emit mode only`}, + {[]string{"review", "-session", dir, "-kind", ""}, `review: -kind must not be empty`}, + {[]string{"review", "-session", dir, "-kind", "tests", "-test", ""}, `review: -test must not be empty`}, + {[]string{"review", "-session", dir, "-kind", "tests", "-decision", ""}, `review: -decision must not be empty`}, + {[]string{"review", "-session", dir, "-kind", "tests", "-edit", ""}, `review: -edit must not be empty`}, + {[]string{"review", "-session", dir, "-kind", "verdicts"}, `review: invalid kind "verdicts"`}, + {[]string{"review", "-session", dir, "-kind", "tests", "-test", "T-01", "-decision", "accepted"}, `review: invalid -test "T-01"`}, + {[]string{"review", "-session", dir, "-kind", "tests", "-test", "T-001", "-decision", "maybe"}, `review: invalid decision "maybe"`}, + {[]string{"review", "-session", dir, "-kind", "tests", "-test", "T-001"}, `review: -decision is required with -test`}, + {[]string{"review", "-session", dir, "-kind", "tests", "-decision", "accepted"}, `review: -test is required with -decision`}, + {[]string{"review", "-session", dir, "-kind", "tests", "-test", "T-001", "-decision", "edited"}, `review: -edit is required with -decision edited`}, + {[]string{"review", "-session", dir, "-kind", "tests", "-test", "T-001", "-decision", "accepted", "-edit", "e.json"}, `review: -edit applies only to -decision edited`}, + {[]string{"review", "-session", dir, "-kind", "tests", "-finding", "F-001", "-verdict", "confirmed"}, `review: -finding and -verdict apply to -kind findings, not -kind tests`}, + {[]string{"review", "-session", dir, "-test", "T-001", "-decision", "accepted"}, `review: -test, -decision and -edit apply to -kind tests, not -kind findings`}, + {[]string{"review", "-session", dir, "-edit", "e.json"}, `review: -test, -decision and -edit apply to -kind tests, not -kind findings`}, } for _, c := range cases { var code int @@ -274,7 +297,9 @@ func TestInvalidFlagValuesExitTwo(t *testing.T) { // documented invocation surface: record's -commit flag and the help command // are part of docs/reference/cli.md but were absent from `testimony help`. func TestUsageListsEveryFlagAndCommand(t *testing.T) { - for _, want := range []string{"-commit HASH", "testimony help"} { + for _, want := range []string{"-commit HASH", "testimony help", + "testimony draft-tests", "-window 10", "-kind findings|tests", "-decision edited -edit FILE", + "transcribe, merge, report, analyze, draft-tests, or review"} { if !strings.Contains(usage, want) { t.Errorf("usage text does not mention %q", want) } @@ -293,7 +318,7 @@ func TestMissingSessionIsAUsageError(t *testing.T) { // manifest.json, which is no longer merely incidental now that its // absence is what sends these invocations down the refusal path. chdir(t, t.TempDir()) - for _, cmd := range []string{"merge", "report", "transcribe", "analyze", "review"} { + for _, cmd := range []string{"merge", "report", "transcribe", "analyze", "draft-tests", "review"} { var code int stderr := captureStderr(t, func() { code = Run([]string{cmd}) }) if code != 2 { @@ -683,3 +708,326 @@ func TestRefusedInvocationAnnouncesNoSession(t *testing.T) { } } } + +// --- draft-tests ------------------------------------------------------------ + +// draftableSession writes a session the drafting layer can work on: a manifest, +// a two-entry timeline, and a findings.jsonl whose F-001 carries a confirmed +// verdict — the one state `draft-tests` is allowed to draft from. +func draftableSession(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := session.SaveManifest(dir, session.Manifest{Session: "s", App: "app", Participant: "P1"}); err != nil { + t.Fatalf("SaveManifest: %v", err) + } + tl := `{"t":0,"src":"speech","id":"utt-001","payload":{"speaker":"P1","t1":5,"text":"I clicked save and nothing happened"}}` + "\n" + + `{"t":1,"src":"event","id":"ev-001","payload":{"kind":"click","selector":"[data-testid=save-btn]","route":"#general"}}` + "\n" + if err := os.WriteFile(filepath.Join(dir, session.TimelineFile), []byte(tl), 0o644); err != nil { + t.Fatalf("write timeline: %v", err) + } + fnd := `{"id":"F-001","t":0,"type":"bug","severity":3,"mode":"A","quote":"I clicked save and nothing happened","evidence":["utt-001","ev-001"],"status":"unverified"}` + "\n" + + `{"kind":"verdict","finding":"F-001","verdict":"confirmed","at":"2026-09-12"}` + "\n" + if err := os.WriteFile(filepath.Join(dir, session.FindingsFile), []byte(fnd), 0o644); err != nil { + t.Fatalf("write findings: %v", err) + } + return dir +} + +// goodCLIAnswer is a schema-clean answer for draftableSession's F-001. +const goodCLIAnswer = `{"rubric":"testimony-testdraft/v1","tests":[` + + `{"id":"T-001","finding":"F-001","session":"s","title":"Saving gives no confirmation",` + + `"steps":["Open #general.","Click Save."],"expected":"The save is confirmed.",` + + `"observed":"Nothing visibly changes.","rationale_quote":"I clicked save and nothing happened",` + + `"severity":3,"status":"accepted"}]}` + +// TestDraftTestsLoudStagingExitsOne pins both loud-staging refusals at exit 1 — +// a well-formed invocation whose work cannot be done — with the counts by status +// on stderr, and nothing written. A caller cannot tell these from a mistyped flag +// if they share exit 2, and cannot tell them from success if they exit 0. +func TestDraftTestsLoudStagingExitsOne(t *testing.T) { + // No confirmed finding: emit and ingest both refuse, naming the counts. + dir := draftableSession(t) + fnd := `{"id":"F-001","t":0,"type":"bug","severity":3,"quote":"I clicked save and nothing happened","evidence":["utt-001"],"status":"unverified"}` + "\n" + if err := os.WriteFile(filepath.Join(dir, session.FindingsFile), []byte(fnd), 0o644); err != nil { + t.Fatalf("write findings: %v", err) + } + // The ingest case passes "-" deliberately: the refusal comes before a byte of + // the answer is read, so it fires with nothing on stdin at all. + for _, args := range [][]string{ + {"draft-tests", "-session", dir}, + {"draft-tests", "-session", dir, "-ingest", "-"}, + } { + var code int + stderr := captureStderr(t, func() { code = Run(args) }) + if code != 1 { + t.Errorf("%v: exit %d, want 1 (runtime refusal)", args, code) + } + want := "testimony: no confirmed findings to draft tests from (1 findings: 0 confirmed, 1 unverified, 0 duplicate, 0 rejected); confirm one with `testimony review -session " + dir + "` first" + if !strings.Contains(stderr, want) { + t.Errorf("%v: want %q on stderr, got %q", args, want, stderr) + } + } + if _, err := os.Stat(filepath.Join(dir, session.TestsFile)); !os.IsNotExist(err) { + t.Errorf("a refused draft-tests wrote %s (err=%v)", session.TestsFile, err) + } + + // No accepted draft: render refuses, and -out writes nothing, so an existing + // test plan cannot be truncated into an empty document. + dir2 := draftableSession(t) + seed := filepath.Join(t.TempDir(), "answer.json") + if err := os.WriteFile(seed, []byte(goodCLIAnswer), 0o644); err != nil { + t.Fatalf("write answer: %v", err) + } + if code := Run([]string{"draft-tests", "-session", dir2, "-ingest", seed}); code != 0 { + t.Fatalf("seed ingest: exit %d", code) + } + out := filepath.Join(t.TempDir(), "plan.md") + if err := os.WriteFile(out, []byte("PRIOR PLAN\n"), 0o644); err != nil { + t.Fatalf("seed plan: %v", err) + } + var code int + stderr := captureStderr(t, func() { code = Run([]string{"draft-tests", "-session", dir2, "-render", "-out", out}) }) + if code != 1 { + t.Errorf("render with no accepted draft: exit %d, want 1", code) + } + want := "testimony: no accepted test drafts to render (1 drafts: 0 accepted, 0 edited, 1 proposed, 0 rejected); accept one with `testimony review -session " + dir2 + " -kind tests` first" + if !strings.Contains(stderr, want) { + t.Errorf("render refusal: want %q on stderr, got %q", want, stderr) + } + if b, err := os.ReadFile(out); err != nil || string(b) != "PRIOR PLAN\n" { + t.Errorf("a refused render truncated the prior plan: %q (err=%v)", b, err) + } +} + +// TestDraftTestsHintsMissingArtefacts: each mode names the command that produces +// what it is missing, rather than surfacing a bare filesystem error. +func TestDraftTestsHintsMissingArtefacts(t *testing.T) { + cases := []struct { + name string + prep func(t *testing.T, dir string) + args func(dir string) []string + want string + }{ + { + "no tests.jsonl to render", + func(t *testing.T, dir string) {}, + func(dir string) []string { return []string{"draft-tests", "-session", dir, "-render"} }, + "no tests.jsonl (run `testimony draft-tests -ingest` first)", + }, + { + "no tests.jsonl to review", + func(t *testing.T, dir string) {}, + func(dir string) []string { + return []string{"review", "-session", dir, "-kind", "tests", "-test", "T-001", "-decision", "accepted"} + }, + "no tests.jsonl (run `testimony draft-tests -ingest` first)", + }, + { + "no findings.jsonl", + func(t *testing.T, dir string) { + if err := os.Remove(filepath.Join(dir, session.FindingsFile)); err != nil { + t.Fatalf("remove findings: %v", err) + } + }, + func(dir string) []string { return []string{"draft-tests", "-session", dir} }, + "no findings.jsonl (run `testimony analyze -ingest` first)", + }, + { + "no timeline.jsonl", + func(t *testing.T, dir string) { + if err := os.Remove(filepath.Join(dir, session.TimelineFile)); err != nil { + t.Fatalf("remove timeline: %v", err) + } + }, + func(dir string) []string { return []string{"draft-tests", "-session", dir} }, + "run `testimony merge` first", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + dir := draftableSession(t) + c.prep(t, dir) + var code int + stderr := captureStderr(t, func() { code = Run(c.args(dir)) }) + if code != 1 { + t.Errorf("exit %d, want 1", code) + } + if !strings.Contains(stderr, c.want) { + t.Errorf("want %q on stderr, got %q", c.want, stderr) + } + }) + } +} + +// TestDraftTestsRoundTripThroughTheCLI drives the whole drafting layer the way an +// operator does: emit the request, ingest a known-good answer, record all three +// decisions, and render the plan — asserting the printed lines, that the draft +// line survives every decision byte-for-byte, and that findings.jsonl is never +// written to. +func TestDraftTestsRoundTripThroughTheCLI(t *testing.T) { + dir := draftableSession(t) + findingsBefore, err := os.ReadFile(filepath.Join(dir, session.FindingsFile)) + if err != nil { + t.Fatalf("read findings: %v", err) + } + + // Emit: to stdout, and the inference notice never contaminates it. + req := captureStdout(t, func() { + if code := Run([]string{"draft-tests", "-session", dir}); code != 0 { + t.Errorf("emit: exit %d, want 0", code) + } + }) + for _, want := range []string{ + "Testimony regression-test drafting rubric: testimony-testdraft/v1", + "Finding F-001 — bug, severity 3, at [00:00], confirmed by human verdict on 2026-09-12:", + "Event window:", + } { + if !strings.Contains(req, want) { + t.Fatalf("emitted request is missing %q:\n%s", want, req) + } + } + + // Emit to a file. + reqPath := filepath.Join(t.TempDir(), "request.md") + stdout := captureStdout(t, func() { + if code := Run([]string{"draft-tests", "-session", dir, "-out", reqPath}); code != 0 { + t.Errorf("emit -out: exit %d, want 0", code) + } + }) + if want := "wrote " + reqPath; !strings.Contains(stdout, want) { + t.Errorf("emit -out: want %q on stdout, got %q", want, stdout) + } + if b, err := os.ReadFile(reqPath); err != nil || !strings.Contains(string(b), "testimony-testdraft/v1") { + t.Errorf("emit -out wrote no request (err=%v)", err) + } + + // Ingest a known-good answer from a file. + answerPath := filepath.Join(t.TempDir(), "answer.json") + if err := os.WriteFile(answerPath, []byte(goodCLIAnswer), 0o644); err != nil { + t.Fatalf("write answer: %v", err) + } + stdout = captureStdout(t, func() { + if code := Run([]string{"draft-tests", "-session", dir, "-ingest", answerPath}); code != 0 { + t.Errorf("ingest: exit %d, want 0", code) + } + }) + if want := "validated 1 test drafts → " + filepath.Join(dir, session.TestsFile) + " (all proposed)"; !strings.Contains(stdout, want) { + t.Errorf("ingest: want %q on stdout, got %q", want, stdout) + } + draftLine := testsDraftLine(t, dir) + // The answer claimed "accepted"; ingest launders it. + if !strings.Contains(draftLine, `"status":"proposed"`) { + t.Errorf("ingest did not force the draft to proposed: %q", draftLine) + } + + // Three decisions, each appended. + editPath := filepath.Join(t.TempDir(), "edit.json") + if err := os.WriteFile(editPath, []byte(`{"title":"Saving a display name gives no confirmation"}`), 0o644); err != nil { + t.Fatalf("write edit: %v", err) + } + today := time.Now().Format("2006-01-02") + for _, c := range []struct { + args []string + want string + }{ + {[]string{"review", "-session", dir, "-kind", "tests", "-test", "T-001", "-decision", "accepted"}, "recorded: T-001 accepted (" + today + ")"}, + {[]string{"review", "-session", dir, "-kind", "tests", "-test", "T-001", "-decision", "edited", "-edit", editPath}, "recorded: T-001 edited (" + today + ")"}, + } { + stdout = captureStdout(t, func() { + if code := Run(c.args); code != 0 { + t.Errorf("%v: exit %d, want 0", c.args, code) + } + }) + if !strings.Contains(stdout, c.want) { + t.Errorf("%v: want %q on stdout, got %q", c.args, c.want, stdout) + } + } + + // Render: the latest edit is applied, and the draft line is untouched. + plan := captureStdout(t, func() { + if code := Run([]string{"draft-tests", "-session", dir, "-render"}); code != 0 { + t.Errorf("render: exit %d, want 0", code) + } + }) + for _, want := range []string{ + "# Regression tests — s", + "## T-001 — Saving a display name gives no confirmation", + "- **Source:** finding `F-001` (bug, severity 3) in session `s`, at [00:00]", + "- **Decision:** edited (" + today + ")", + "1. Open #general.", + "**Expected:** The save is confirmed.", + "“I clicked save and nothing happened”", + "1 of 1 drafts accepted.", + } { + if !strings.Contains(plan, want) { + t.Fatalf("rendered plan is missing %q:\n%s", want, plan) + } + } + if got := testsDraftLine(t, dir); got != draftLine { + t.Errorf("the draft line changed across the decisions:\n got %q\nwant %q", got, draftLine) + } + + // A rejection removes it from the plan again. + stdout = captureStdout(t, func() { + if code := Run([]string{"review", "-session", dir, "-kind", "tests", "-test", "T-001", "-decision", "rejected"}); code != 0 { + t.Errorf("reject: exit %d, want 0", code) + } + }) + if want := "recorded: T-001 rejected (" + today + ")"; !strings.Contains(stdout, want) { + t.Errorf("reject: want %q on stdout, got %q", want, stdout) + } + if code := Run([]string{"draft-tests", "-session", dir, "-render"}); code != 1 { + t.Errorf("render after the rejection: exit %d, want 1", code) + } + + // findings.jsonl is a different record family and is never written to. + after, err := os.ReadFile(filepath.Join(dir, session.FindingsFile)) + if err != nil { + t.Fatalf("read findings: %v", err) + } + if string(after) != string(findingsBefore) { + t.Error("the drafting pipeline modified findings.jsonl") + } +} + +// testsDraftLine returns the single draft (non-decision) line of tests.jsonl, so +// the append-only property can be asserted byte-for-byte across decisions. +func testsDraftLine(t *testing.T, dir string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(dir, session.TestsFile)) + if err != nil { + t.Fatalf("read tests: %v", err) + } + for _, l := range strings.Split(strings.TrimRight(string(b), "\n"), "\n") { + if !strings.Contains(l, `"kind":"decision"`) { + return l + } + } + t.Fatalf("tests.jsonl holds no draft line: %q", b) + return "" +} + +// TestDraftTestsInfersSession: draft-tests joins the commands that take their +// session from the current directory, announcing the inference on stderr and +// keeping it off stdout, where the emitted request has to stay a clean pipe. +func TestDraftTestsInfersSession(t *testing.T) { + dir := draftableSession(t) + chdir(t, dir) + var code int + var req string + stderr := captureStderr(t, func() { + req = captureStdout(t, func() { code = Run([]string{"draft-tests"}) }) + }) + if code != 0 { + t.Fatalf("draft-tests with an inferred session: exit %d, want 0", code) + } + if want := "draft-tests: using session . (inferred from the current directory)"; !strings.Contains(stderr, want) { + t.Errorf("want %q on stderr, got %q", want, stderr) + } + if !strings.Contains(req, "testimony-testdraft/v1") { + t.Errorf("the inferred run emitted no request: %q", req) + } + if strings.Contains(req, "inferred") { + t.Errorf("the inference notice reached stdout: %q", req) + } +} diff --git a/internal/drafttests/drafttests.go b/internal/drafttests/drafttests.go index d03fa67..15a7ebb 100644 --- a/internal/drafttests/drafttests.go +++ b/internal/drafttests/drafttests.go @@ -391,9 +391,10 @@ func clock(sec float64) string { // ErrNoConfirmedFindings marks the refusal that stages an empty drafting step // loudly: a session whose findings are all unverified, rejected, or duplicates // has nothing a draft could legally reference, so emit and ingest both refuse, -// name the finding count by status, and write nothing. It is a sentinel so the -// CLI can tell a well-formed invocation whose work cannot be done from a genuine -// failure. +// name the finding count by status, and write nothing. It is a sentinel so a +// caller can tell a well-formed invocation whose work cannot be done from a +// genuine failure; the CLI maps both to exit 1, the status such a refusal +// already takes, and does not branch on it. var ErrNoConfirmedFindings = errors.New("no confirmed findings to draft tests from") // ErrNoAcceptedDrafts is its render-side twin: a plan with no accepted or edited diff --git a/internal/drafttests/emit.go b/internal/drafttests/emit.go index ed82711..d222de8 100644 --- a/internal/drafttests/emit.go +++ b/internal/drafttests/emit.go @@ -72,7 +72,14 @@ func EmitRequest(dir string, window float64) (string, error) { b.WriteString("Draft one or more test cases per confirmed finding, in finding-id order. For each draft:\n\n") b.WriteString("- **`steps`** — the reproduction, in time order. Each entry is one imperative " + "action a developer can follow, naming the selector or the route where the window names " + - "it, and the sequence ends at the moment the finding is anchored to.\n") + "it. The sequence **ends at the last cited evidence event at or before the finding's `t`**: " + + "that event is the action the finding is anchored to. A cited evidence event *after* the " + + "finding's `t` is not a step — it is part of what the participant did in response, so it " + + "belongs in `observed`.\n") + b.WriteString("- The `route` on the window's first event names where the participant already " + + "is when the window opens. You may derive **one** opening orientation step from it (for " + + "example \"Open #general\"); every other step must correspond to an event or an utterance " + + "that is actually in the window.\n") b.WriteString("- **`expected`** — the behaviour the participant expected, grounded in their own utterances in the window.\n") b.WriteString("- **`observed`** — what the system actually did, grounded in the window's events and utterances.\n") b.WriteString("- **`title`** — one line naming the defect.\n\n") @@ -87,6 +94,14 @@ func EmitRequest(dir string, window float64) (string, error) { b.WriteString("- `expected`, `observed` — non-empty prose.\n") b.WriteString("- `rationale_quote` — the finding's own `quote`, copied byte for byte.\n") b.WriteString("- `severity` — the finding's own `severity`, copied unchanged.\n\n") + b.WriteString("Reading each finding record below — two of its fields are about the record, not about your draft:\n\n") + b.WriteString("- `status` is the finding's **birth state**, and it reads `unverified` on every " + + "finding this tool writes: a finding is born a candidate. It is *not* the finding's current " + + "status. The human verdict records that confirmed these findings live alongside them and are " + + "not shown here; every finding below is confirmed, and its header names the date the verdict " + + "was recorded.\n") + b.WriteString("- `mode` is the capture mode: `A` is the application under test, `B` is reference " + + "capture of a third-party app. Only mode `A` findings are eligible, so every finding below is mode `A`.\n\n") b.WriteString("Hard constraints (each is enforced when your answer is ingested):\n\n") b.WriteString("- `rationale_quote` must **equal** the source finding's `quote` — byte for byte, not a re-derivation from the utterance. The drafting step carries evidence forward; it never introduces any.\n") b.WriteString("- `severity` must equal the source finding's `severity`. Triage order is a human product and is not yours to choose.\n") @@ -136,11 +151,20 @@ func EmitRequest(dir string, window float64) (string, error) { b.WriteString("## Confirmed findings\n\n") b.WriteString("Each finding below is confirmed by a human and eligible. Its own record is given " + - "first (copy `quote` and `severity` from it byte for byte), then its event window: the " + - "timeline entries around it, in time order, which are the only source for the steps.\n\n") + "first, verbatim as it is stored (copy `quote` and `severity` from it byte for byte; its " + + "`status` is the birth state, see the rubric above), then its event window: the timeline " + + "entries around it, in time order, which are the only source for the steps.\n\n") + // The verdict date comes from the same effective-status computation eligibility + // does, so the header cannot claim a confirmation the eligible set did not agree + // with. It is rendered rather than the record's own status field being rewritten: + // the model must copy `quote` and `severity` out of this line byte for byte, and + // substituting one field would make the record it is shown differ from the record + // ingest validates against — the shown-vs-validated gap this package closes + // everywhere else. + eff := analyze.EffectiveStatus(findings, verdicts) for _, f := range confirmed { - fmt.Fprintf(&b, "Finding %s — %s, severity %d, at [%s]:\n\n", - session.SafeInline(f.ID), safeOrDash(f.Type), f.Severity, clock(f.T)) + fmt.Fprintf(&b, "Finding %s — %s, severity %d, at [%s], %s:\n\n", + session.SafeInline(f.ID), safeOrDash(f.Type), f.Severity, clock(f.T), confirmedOn(eff[f.ID])) line, err := json.Marshal(f) if err != nil { return "", err @@ -197,3 +221,18 @@ func safeOrDash(s string) string { } return t } + +// confirmedOn renders the human verdict behind an eligible finding, so the +// request never shows a record whose `status` field reads "unverified" without +// saying in the same line what actually made it eligible. The date is +// attacker-authorable (a verdict in an exchanged session is unvalidated text) and +// renders outside any code fence, so it goes through SafeInline; a verdict +// carrying no date — or one that renders as nothing — drops the clause rather +// than printing a dangling "on". +func confirmedOn(st analyze.Status) string { + at := session.SafeInline(st.At) + if strings.TrimSpace(at) == "" { + return "confirmed by human verdict" + } + return "confirmed by human verdict on " + at +} diff --git a/internal/drafttests/emit_test.go b/internal/drafttests/emit_test.go index 09616e0..c0393d4 100644 --- a/internal/drafttests/emit_test.go +++ b/internal/drafttests/emit_test.go @@ -35,7 +35,7 @@ func TestEmitCarriesConfirmedFindingsAndWindows(t *testing.T) { "1. Change your display name and save it", "2. Try the appearance settings", "## Confirmed findings", - "Finding F-001 — bug, severity 3, at [00:22]:", + "Finding F-001 — bug, severity 3, at [00:22], confirmed by human verdict on 2026-09-12:", "Event window:", "## Answer", `{"rubric":"testimony-testdraft/v1","tests":[ … ]}`, @@ -270,3 +270,110 @@ func TestEmitPlaceholdersInvisibleOnlyManifestFields(t *testing.T) { t.Fatalf("a blank task was numbered:\n%s", got) } } + +// TestEmitResolvesTheStatusContradiction is the self-contradicting-request +// regression a host model found when it answered the real thing. The per-finding +// record is shown verbatim as stored, so its `status` field reads "unverified" — +// the birth state every finding this tool writes carries — while the section +// heading says every finding below is confirmed and the hard constraints say an +// unverified finding is ineligible. Left unexplained, the request contradicts +// itself and the model has to guess which claim to believe. The record is still +// shown byte-for-byte (the model copies `quote` and `severity` out of it, so +// rewriting a field would make the shown record differ from the one ingest +// validates against); the header names the verdict that confirmed it and the +// rubric says what `status` is. +func TestEmitResolvesTheStatusContradiction(t *testing.T) { + dir := writeSession(t) + got, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + // The record is unaltered: status still reads its birth state. + if !strings.Contains(got, `"status":"unverified"`) { + t.Fatalf("the finding record was rewritten rather than explained:\n%s", got) + } + // The header says what made it eligible, with the verdict's own date. + if !strings.Contains(got, "confirmed by human verdict on 2026-09-12:") { + t.Fatalf("the per-finding header does not name the verdict behind it:\n%s", got) + } + // The rubric says what the status field is, and what mode is. + for _, want := range []string{ + "`status` is the finding's **birth state**", + "It is *not* the finding's current status.", + "`mode` is the capture mode", + } { + if !strings.Contains(got, want) { + t.Fatalf("the rubric does not explain %q:\n%s", want, got) + } + } +} + +// TestEmitVerdictDateFallsBackWhenAbsent: a verdict record in an exchanged or +// hand-edited findings.jsonl need not carry a date, and one that renders as +// nothing must drop the clause rather than print a dangling "on". +func TestEmitVerdictDateFallsBackWhenAbsent(t *testing.T) { + findings := `{"id":"F-001","t":22,"type":"bug","severity":3,"mode":"A","quote":"I clicked save and nothing happened","evidence":["utt-004"],"status":"unverified"} +{"kind":"verdict","finding":"F-001","verdict":"confirmed","at":" "} +` + dir := writeSession(t, session.FindingsFile, findings) + got, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + if !strings.Contains(got, "confirmed by human verdict:") { + t.Fatalf("a dateless verdict did not fall back cleanly:\n%s", got) + } + if strings.Contains(got, "verdict on :") || strings.Contains(got, "verdict on ") { + t.Fatalf("a dateless verdict printed a dangling \"on\":\n%s", got) + } +} + +// TestEmitStatesWhereTheStepsEnd is the first of two ambiguity regressions from +// the same live run. "ends at the moment the finding is anchored to" did not say +// which anchor, and a cited evidence event *after* the finding's t (the second +// Save click at ev-004 in the sample) had no stated home — so the model could +// legitimately have made it a step or dropped it. The rule is stated: steps end +// at the last cited evidence event at or before the finding's t, and a later one +// belongs in observed. +func TestEmitStatesWhereTheStepsEnd(t *testing.T) { + dir := writeSession(t) + got, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + for _, want := range []string{ + "ends at the last cited evidence event at or before the finding's `t`**", + "it belongs in `observed`", + } { + if !strings.Contains(got, want) { + t.Fatalf("the instructions do not state the steps-end rule (%q):\n%s", want, got) + } + } +} + +// TestEmitLicensesOneOrientationStep is the second. The worked example's first +// step ("Open #general…") corresponds to no event in the sample window, so the +// example licensed an invented step against the stance paragraph's "never invent +// a step". Rather than weaken the example — every real repro needs to say where +// it starts — the request states where that step may legitimately come from, and +// bounds it to one. +func TestEmitLicensesOneOrientationStep(t *testing.T) { + dir := writeSession(t) + got, err := EmitRequest(dir, 10) + if err != nil { + t.Fatalf("EmitRequest: %v", err) + } + for _, want := range []string{ + "The `route` on the window's first event names where the participant already is", + "You may derive **one** opening orientation step from it", + "every other step must correspond to an event or an utterance that is actually in the window", + } { + if !strings.Contains(got, want) { + t.Fatalf("the instructions do not license the orientation step (%q):\n%s", want, got) + } + } + // The worked example's opening step is the one the licence covers. + if !strings.Contains(got, `"Open #general in the settings prototype."`) { + t.Fatalf("the worked example lost its orientation step:\n%s", got) + } +} diff --git a/internal/drafttests/ingest.go b/internal/drafttests/ingest.go index 61daf7a..0f966cb 100644 --- a/internal/drafttests/ingest.go +++ b/internal/drafttests/ingest.go @@ -147,7 +147,7 @@ func commitDrafts(dir string, drafts []Draft) error { for _, d := range drafts { b, err := json.Marshal(d) if err != nil { - return fmt.Errorf("write test drafts: %w", err) + return fmt.Errorf("write %s: %w", session.TestsFile, err) } records = append(records, b) } diff --git a/internal/drafttests/render_test.go b/internal/drafttests/render_test.go index 325988d..8272ada 100644 --- a/internal/drafttests/render_test.go +++ b/internal/drafttests/render_test.go @@ -231,7 +231,7 @@ func TestRoundTripGolden(t *testing.T) { if err != nil { t.Fatalf("EmitRequest: %v", err) } - if !strings.Contains(req, "Finding F-001 — bug, severity 3, at [00:22]:") { + if !strings.Contains(req, "Finding F-001 — bug, severity 3, at [00:22], confirmed by human verdict on 2026-07-17:") { t.Fatalf("the emitted request does not carry F-001:\n%s", req) } diff --git a/internal/drafttests/review.go b/internal/drafttests/review.go index 609324e..ddab1ff 100644 --- a/internal/drafttests/review.go +++ b/internal/drafttests/review.go @@ -311,6 +311,11 @@ func promptEdit(w io.Writer, r *bufio.Reader, d Draft) (*Edit, error) { fmt.Fprintln(w, " steps (one per line, blank line ends; a blank first line keeps them):") var steps []string + // typed counts every step the operator actually entered; steps stops growing + // one past the cap. The two differ precisely when the input is over-long, and + // the refusal below reports typed — reporting len(steps) would say "33 + // entries" to someone who typed forty, naming a number they never chose. + typed := 0 for { fmt.Fprint(w, " ") line, lerr := readLine(r) @@ -321,14 +326,20 @@ func promptEdit(w io.Writer, r *bufio.Reader, d Draft) (*Edit, error) { if s == "" { break } - // Read to the terminating blank line whatever the count, but stop growing - // one past the cap: the overflow is then reported by checkEdit, naming the - // limit, rather than silently truncated at 32 with the rest of the - // operator's typing left to be consumed as the next prompt's answer. + typed++ + // Read to the terminating blank line whatever the count, so the rest of the + // operator's typing is never left behind to be consumed as the next + // prompt's answer, but hold the slice one past the cap so an absurd paste + // cannot grow it without bound. if len(steps) <= maxSteps { steps = append(steps, s) } } + // Checked here rather than left to checkEdit, which can only see the truncated + // slice. The message is checkEdit's, so both paths refuse in one voice. + if typed > maxSteps { + return nil, fmt.Errorf("edit: steps lists %d entries, exceeding the limit of %d", typed, maxSteps) + } if len(steps) > 0 { e.Steps = &steps } diff --git a/internal/drafttests/review_test.go b/internal/drafttests/review_test.go index 9cb724b..459a2b7 100644 --- a/internal/drafttests/review_test.go +++ b/internal/drafttests/review_test.go @@ -545,22 +545,30 @@ func TestEditRoundTripsThroughJSON(t *testing.T) { // against the limit rather than silently truncated at 32 with the rest of the // operator's typing consumed as the next prompt's answer. func TestInteractiveEditRefusesTooManySteps(t *testing.T) { - dir := ingestThree(t) - lines := []string{"e", ""} // edit, then a blank title (keep it) - for i := 0; i <= maxSteps; i++ { - lines = append(lines, fmt.Sprintf("Step %d.", i+1)) - } - lines = append(lines, "", "", "") // end the steps, then blank expected and observed - var out bytes.Buffer - if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader(strings.Join(lines, "\n") + "\n"), Out: &out, IsTTY: true, Today: "2026-09-12"}); err != nil { - t.Fatalf("Review: %v", err) - } - want := fmt.Sprintf("steps lists %d entries, exceeding the limit of %d", maxSteps+1, maxSteps) - if !strings.Contains(out.String(), want) { - t.Fatalf("expected the over-long steps refusal (%q):\n%s", want, out.String()) - } - if _, decisions, _ := Load(dir); len(decisions) != 0 { - t.Fatal("a refused edit recorded a decision") + // The count in the message is what the operator typed, not where the slice + // stopped growing: the prompt reads to the terminating blank line whatever the + // count but holds the slice one past the cap, so reporting len(steps) told + // someone who typed forty steps that they had listed 33. + for _, typed := range []int{maxSteps + 1, 40} { + t.Run(fmt.Sprintf("%d steps", typed), func(t *testing.T) { + dir := ingestThree(t) + lines := []string{"e", ""} // edit, then a blank title (keep it) + for i := 0; i < typed; i++ { + lines = append(lines, fmt.Sprintf("Step %d.", i+1)) + } + lines = append(lines, "", "", "") // end the steps, then blank expected and observed + var out bytes.Buffer + if err := Review(ReviewOptions{Dir: dir, In: strings.NewReader(strings.Join(lines, "\n") + "\n"), Out: &out, IsTTY: true, Today: "2026-09-12"}); err != nil { + t.Fatalf("Review: %v", err) + } + want := fmt.Sprintf("steps lists %d entries, exceeding the limit of %d", typed, maxSteps) + if !strings.Contains(out.String(), want) { + t.Fatalf("expected the over-long steps refusal (%q):\n%s", want, out.String()) + } + if _, decisions, _ := Load(dir); len(decisions) != 0 { + t.Fatal("a refused edit recorded a decision") + } + }) } }