From 9750d06ea7886b8480c64f9f36a14910ab1b7565 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Fri, 16 Oct 2026 10:31:53 -0700 Subject: [PATCH 1/3] fix(report): estimate a cloud candidate's violations by searching, not by default A cloud candidate's estimatedViolations of zero usually meant the agent never searched, while the recipe called that zero honest. A new 'Estimating violations' section asks for a fast, good-faith estimate of whether the pattern is happening now: a handful of searches, starting from the real shape of a past fix, searching for the problem rather than the protection, separating syntactic from data-flow patterns, spot-checking hits, and writing the low end of the range. Zero stays a valid answer once the agent has looked. --- packages/cli/src/agent/report.md | 36 ++++++++++++++++++++---- packages/cli/src/schemas/report-facts.ts | 4 +-- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/agent/report.md b/packages/cli/src/agent/report.md index 58387bd3..a4146362 100644 --- a/packages/cli/src/agent/report.md +++ b/packages/cli/src/agent/report.md @@ -46,7 +46,8 @@ later and deletes one on request. - `history`: what that window held. Merged pull or merge requests, review comments, the ones an agent helped write, and how many recurring review patterns were rule-shaped versus judgment calls. - Zeros are honest when there was nothing to mine. + A zero is honest only for a source you read and found empty, not + for one you did not read. - `rules`: one entry per rule the draft listed, with a title a reader understands and its `surfaced` counts, counted as described under `## Counting surfaced`. @@ -55,10 +56,9 @@ later and deletes one on request. - `cloud`: runtime-shaped candidates you judged better solved by a runtime rule and did not author. Count their `surfaced` the same way. Give each one `estimatedViolations`: how many instances of - its pattern you estimate are in the code today, from what you saw - while mining. No rule has run, so this is an estimate, and it is - published as one. Zero is honest if you proposed the candidate - from review alone and saw no instance in the code. + its pattern you estimate are in the code today, made as described + under `## Estimating violations`. No rule has run, so this is an + estimate, and it is published as one. - `redaction`: what you removed yourself, by kind, and any fixture you choose not to publish. Names of people and customers are yours to judge; the CLI's patterns cannot see them. @@ -135,6 +135,32 @@ sessions compare: however well it shows what the rule is worth. The facts have no field for it; leave it out rather than count it as review. +## Estimating violations + +A cloud candidate's `estimatedViolations` answers one question: how +likely is this pattern happening in the code now? Zero is a fine +answer, but only after you have looked. Not having searched is not +evidence that the code is clean. Make the estimate in good faith, and +quickly: + +- **Spend a handful of searches, not an audit.** Do not read whole + files or start other agents for it. +- **Start from the real shape.** If a commit fixed the pattern, look + at what it changed and search for that, not for a form you guessed. + A pattern fixed more than once is likely to remain somewhere. +- **Search for the problem, not the protection.** A search for the + helper that escapes, or the guard that checks, finds the places + already right and says nothing about the places still missing it. +- **Know which kind of pattern it is.** A syntactic one (a call, an + import, a setting) can be searched for directly, and the count is + close to exact. A data-flow one (input reaching an output + unprotected) cannot: find where the data comes out, then the places + that lack the protection. Those are suspects, not confirmed + instances. +- **Check two or three hits by hand,** and scale the count by how + many held up. +- **Write the low end of your range.** + ## Facts schema The `--from` JSON file conforms to: diff --git a/packages/cli/src/schemas/report-facts.ts b/packages/cli/src/schemas/report-facts.ts index 56832b13..5e3c9a86 100644 --- a/packages/cli/src/schemas/report-facts.ts +++ b/packages/cli/src/schemas/report-facts.ts @@ -82,7 +82,7 @@ const cloudCandidateFacts = z ...outcome, language: z.string().min(1).max(40), estimatedViolations: count.describe( - "How many instances of this pattern you estimate are in the code today, from what you saw while mining. An estimate, not a measurement: no rule has run" + "How many instances of this pattern you estimate are in the code today, from a quick search for the problem itself. An estimate, not a measurement: no rule has run. A zero means you searched and found none" ), fail: fixture.optional().describe("Code the rule would flag"), pass: fixture.optional().describe("Code the rule would accept"), @@ -123,7 +123,7 @@ export const reportFactsSchema = z }) .strict() .describe( - "What the window's history held. Zeros are honest when there was nothing to mine" + "What the window's history held. A zero is honest only for a source you read and found empty" ), rules: z .array(localRuleFacts) From 807a88e42e467d4d380f4c85d42159f5924bbd0e Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Fri, 16 Oct 2026 10:52:45 -0700 Subject: [PATCH 2/3] fix(report): bump the report topic to v5 The recipe body changed, so a cached v4 copy needs a signal that its guidance on estimating cloud violations is superseded. --- packages/cli/src/agent/report.md | 2 +- packages/cli/test/report.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/agent/report.md b/packages/cli/src/agent/report.md index a4146362..021efde9 100644 --- a/packages/cli/src/agent/report.md +++ b/packages/cli/src/agent/report.md @@ -1,4 +1,4 @@ -# Topic: report (CLI v%(CLI_VERSION)s / topic v4) +# Topic: report (CLI v%(CLI_VERSION)s / topic v5) ## Goal Publish a Taskless report: an unlisted page on tskl.es that shows what diff --git a/packages/cli/test/report.test.ts b/packages/cli/test/report.test.ts index 937b24d2..1ca6784c 100644 --- a/packages/cli/test/report.test.ts +++ b/packages/cli/test/report.test.ts @@ -878,7 +878,7 @@ describe("taskless report", () => { it("counts only review comments, and asks for a candidate estimate, not pricing", async () => { const { stdout, exitCode } = await run(["agent", "report"]); expect(exitCode).toBe(0); - expect(stdout).toMatch(/^# Topic: report {5}\(CLI v.+ \/ topic v4\)/m); + expect(stdout).toMatch(/^# Topic: report {5}\(CLI v.+ \/ topic v5\)/m); const flat = stdout.replaceAll(/\s+/g, " "); const counting = flat.slice(flat.indexOf("## Counting surfaced")); expect(counting).toContain("**Only review comments count.**"); From 8dea6da7de5e96440b2316fef4f994359c26465b Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Fri, 16 Oct 2026 11:07:25 -0700 Subject: [PATCH 3/3] test(report): pin the estimating-violations guidance An agent once searched for the escape helper, found where escaping already happened, and published zero. Pin the section that asks for a search for the problem itself, and that the old 'saw no instance' zero is gone. --- packages/cli/test/report.test.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/cli/test/report.test.ts b/packages/cli/test/report.test.ts index 1ca6784c..c2d508b6 100644 --- a/packages/cli/test/report.test.ts +++ b/packages/cli/test/report.test.ts @@ -890,6 +890,8 @@ describe("taskless report", () => { flat.indexOf("**Generate.**") ); expect(supply).toContain("`estimatedViolations`"); + expect(supply).toContain("`## Estimating violations`"); + expect(supply).not.toContain("saw no instance"); expect(supply).not.toContain("contextTokens"); expect(supply).not.toContain("callsPerCatch"); const show = flat.slice( @@ -899,6 +901,27 @@ describe("taskless report", () => { expect(show).toContain("which are **your estimate**"); }); + // An agent grepped for the escape helper, found where escaping already + // happens, and published zero for a pattern fixed four times. Zero stays a + // valid answer, but only after a search for the problem itself. + it("asks for a searched estimate of a cloud candidate's violations", async () => { + const { stdout, exitCode } = await run(["agent", "report"]); + expect(exitCode).toBe(0); + const flat = stdout.replaceAll(/\s+/g, " "); + const estimating = flat.slice( + flat.indexOf("## Estimating violations"), + flat.indexOf("## Facts schema") + ); + expect(estimating).toContain( + "Zero is a fine answer, but only after you have looked." + ); + expect(estimating).toContain( + "**Search for the problem, not the protection.**" + ); + expect(estimating).toContain("**Start from the real shape.**"); + expect(estimating).toContain("**Write the low end of your range.**"); + }); + // "Show it exactly as printed" read as verbatim, so agents fenced the // summary and its link stopped being clickable. The summary already says it // can be deleted, so the recipe no longer asks for that line again.