diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aaef38..0f7203e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,11 @@ Please choose versions by [Semantic Versioning](http://semver.org/). - feat: `--reharvest` re-parses cached transcripts and rewrites each row's harvested fields, invoking no review. A harvester fix otherwise leaves every scored row wrong with no cheap way to correct it: the config identity deliberately excludes `bench/run.py`, so a re-run serves the same stale rows from cache, and a live re-run costs the price of the whole pass. Applying it to the curated-1 pass took findings 128 → 133 and dropped items 5 → 0 for zero review spend. This only works because reviews are now captured with `--output-format stream-json` — before that the stored output was the reviewer's last message, and replay could not recover what was never captured - test: 207 → 212. Extensionless files attribute; dotted paths still do; prose and clock times (`see step 3:`, `at 12:30`, `v1.2.3:`) do not become paths; known rule ids still do not; and the unattributable message does not claim a discard +## Unreleased + +- feat: `bench/prs.json` becomes `curated-1` — **20 PRs across 15 repos**, replacing the 5-PR `dev-1` development fixture. Size spread 6 → 2024 lines; 17 merge-commit and 3 squash entries, so both diff-reconstruction paths are exercised. `dev-1` existed to build the runner against, not to carry a score: measured 3-sigma on it was ~119% of the mean, meaning two configurations had to differ by more than the entire average finding count before the gap registered +- test: the manifest fixture test pins `curated-1`/20 (a manifest change is part of the config identity, so it must be a conscious edit rather than something a scored run discovers), plus a new test asserting the spread the set was curated **for** — small and large PRs present, 10+ distinct repos, and both merge strategies + ## v0.39.1 - fix: the report page shows what the gates removed. `v0.39.0` made the gates drop an unattributable finding and grade a review on the sections it did carry, recording both on the row — but neither reached the page, so a run whose reviews were largely rejected on shape rendered identically to one that scored cleanly. The Per-PR table gains `dropped items` and `missing sections` columns, the Runs table's PR count becomes scored-over-expected (`4/5`, not `4`), and an effective-fixture line under Per-PR states how many of the manifest's PRs produced a scored row. Every number is counted from the rows, never asserted diff --git a/bench/prs.json b/bench/prs.json index aff90e3..ae9ba9c 100644 --- a/bench/prs.json +++ b/bench/prs.json @@ -1,7 +1,7 @@ { - "version": "dev-1", - "description": "Development fixture for the code-review benchmark. Deliberately NOT representative — chosen to exercise the runner across languages, sizes and merge strategies while the harness is built. The curated 15-20 PR set is separate.", - "created": "2026-08-06", + "version": "curated-1", + "description": "Curated benchmark PR set: 20 merged PRs across 13 bborbe repos. Grown from the 5-PR dev-1 fixture so the noise floor can shrink enough for SC4 (discrimination) to be attemptable. PRs in bborbe/coding are deliberately excluded: scoring the reviewer against the repo whose rules define it is self-referential.", + "created": "2026-08-09", "verified": "2026-08-06", "prs": [ { @@ -83,6 +83,246 @@ "deletions": 13, "role": "python-coverage", "notes": "Vulnerability and secret scanning gates. Exercises the 13 python rules." + }, + { + "id": "vault-cli#68", + "owner": "bborbe", + "repo": "vault-cli", + "number": 68, + "language": "go", + "merge_strategy": "merge-commit", + "merge_sha": "3e30d6a947c4dfd07a100c4ed8c45d64bbefa790", + "base_sha": "f59662a4f78f06fd28ff6a90d6b6014567fd5843", + "head_sha": "ba7ab5baf194c2b2656fdeb3c57c7f9149bbae41", + "changed_files": 2, + "additions": 4, + "deletions": 4, + "role": "unreviewed", + "notes": "fix: remove go.mod exclude directive blocking remote install \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "backup#15", + "owner": "bborbe", + "repo": "backup", + "number": 15, + "language": "go", + "merge_strategy": "merge-commit", + "merge_sha": "78e3ea7adc83d8867fb0a19da2c3c557c6f7c7ba", + "base_sha": "6fae61c0855c800a6f348487cfc87d0d67996525", + "head_sha": "cbc0c69e659b1648e6e2ff354cf0e9c38f0a8121", + "changed_files": 3, + "additions": 9, + "deletions": 3, + "role": "unreviewed", + "notes": "Fix npm registry host and make it overridable \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "bw#41", + "owner": "bborbe", + "repo": "bw", + "number": 41, + "language": "python", + "merge_strategy": "merge-commit", + "merge_sha": "9d2c24811dac1e0b4a7cf6f7feb5c67996733890", + "base_sha": "2c9afe09b20723f69daa9d0e3532f14221dec1d7", + "head_sha": "2f3b061c63efc4bf4a3275843a25c22dc0b7fb85", + "changed_files": 2, + "additions": 23, + "deletions": 2, + "role": "unreviewed", + "notes": "fix(k3s): raise ARP neighbour table limits to stop table overflow \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "dark-factory#71", + "owner": "bborbe", + "repo": "dark-factory", + "number": 71, + "language": "go", + "merge_strategy": "merge-commit", + "merge_sha": "57d159405cd2cd05b7f0152e8591e1f4ef8bf835", + "base_sha": "e2c735b5e1d93497eabf72f01a12bdcf89132247", + "head_sha": "4ad54f58bc26d3b471e795a6d6bb65b0cd50a420", + "changed_files": 2, + "additions": 18, + "deletions": 6, + "role": "unreviewed", + "notes": "fix: stop prompt-auditor offering fixes it cannot apply \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "tts-mcp#16", + "owner": "bborbe", + "repo": "tts-mcp", + "number": 16, + "language": "python", + "merge_strategy": "merge-commit", + "merge_sha": "d6cb9fd8178bd693264fdc28ae89f7f2b47ae50c", + "base_sha": "2341be9202eddd255fdbf249645206c2135d86ed", + "head_sha": "e75300e7c33cf413af9418b7d9c8b4001cc75a2e", + "changed_files": 2, + "additions": 46, + "deletions": 14, + "role": "unreviewed", + "notes": "Skill owns spoken-output volume; add narrate mode \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "tts-mcp#3", + "owner": "bborbe", + "repo": "tts-mcp", + "number": 3, + "language": "python", + "merge_strategy": "merge-commit", + "merge_sha": "75ec087324067df97c47eafdba7cb3eede0d19c9", + "base_sha": "a5ed826d76ce505d837f6011e5edeef2b4fea0c6", + "head_sha": "c462f4a79c6611bf440a6ae5b2f51c78f62c5f54", + "changed_files": 3, + "additions": 40, + "deletions": 16, + "role": "unreviewed", + "notes": "Split the voice selftest out of /voice activation \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "distill#6", + "owner": "bborbe", + "repo": "distill", + "number": 6, + "language": "go", + "merge_strategy": "merge-commit", + "merge_sha": "1b9de9efb53d64bb45da168678491157065d03a7", + "base_sha": "efe960985a64364b00261dc05258e9ea7a5f1f3a", + "head_sha": "9beb1231e5066788c8c6a4b73d628497c248602c", + "changed_files": 4, + "additions": 251, + "deletions": 0, + "role": "unreviewed", + "notes": "docs: CLAUDE.md adoption runbook + ship the edit-protection hook \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "vault-ui#39", + "owner": "bborbe", + "repo": "vault-ui", + "number": 39, + "language": "python", + "merge_strategy": "merge-commit", + "merge_sha": "2294ec1a56ee3797c4294e60bcbf38ef76eb8756", + "base_sha": "a581a6aa360f9ea06b5baf6b546013407912e6e1", + "head_sha": "2a94147d58322c80f91c8f6d91e0de5da5577666", + "changed_files": 11, + "additions": 341, + "deletions": 6, + "role": "unreviewed", + "notes": "Add activity-age badge to task cards \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "recurring-task-creator#30", + "owner": "bborbe", + "repo": "recurring-task-creator", + "number": 30, + "language": "go", + "merge_strategy": "merge-commit", + "merge_sha": "5c9f8ecbcbdd153b16e032cfd9115dde938b2e73", + "base_sha": "6b0fc0fdad7798025c1dfe1fea6a7e58e91e7515", + "head_sha": "3f42ce626126a7dfe52886336ca59b69e7ef03e1", + "changed_files": 6, + "additions": 369, + "deletions": 14, + "role": "unreviewed", + "notes": "fix: anchor PeriodOffset shifts to day-1 to stop month-end token overflow \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "tts-mcp#10", + "owner": "bborbe", + "repo": "tts-mcp", + "number": 10, + "language": "python", + "merge_strategy": "merge-commit", + "merge_sha": "a55de3d67bb11e0a8d768c625392bb63082b1a0a", + "base_sha": "75ec087324067df97c47eafdba7cb3eede0d19c9", + "head_sha": "c6f136af37f33a035708f27e59de4d78f68db6e6", + "changed_files": 4, + "additions": 253, + "deletions": 220, + "role": "unreviewed", + "notes": "Bump mcp/ transitive deps and make CI actually cover the relay \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "discord-assistant#5", + "owner": "bborbe", + "repo": "discord-assistant", + "number": 5, + "language": "node", + "merge_strategy": "merge-commit", + "merge_sha": "4b0d3af36d886b6d8242ac772d4130f63329569b", + "base_sha": "b66b8002d00e9ea5e527ada51ce0390130ba5c53", + "head_sha": "b910f40ba6b2711ee7780ed8cc9365b27d3d4c8d", + "changed_files": 7, + "additions": 513, + "deletions": 12, + "role": "unreviewed", + "notes": "feat: wait to be addressed before answering in a call \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "tts-mcp#13", + "owner": "bborbe", + "repo": "tts-mcp", + "number": 13, + "language": "python", + "merge_strategy": "merge-commit", + "merge_sha": "e7ee655c2ffcea8ff97095aa56f629617a0bc711", + "base_sha": "cd18826db9a96c3e3a0098bd1190856b39ba06b0", + "head_sha": "59f9fe8588e48c0b03fee1de910641b6ce14c4df", + "changed_files": 17, + "additions": 915, + "deletions": 233, + "role": "unreviewed", + "notes": "Add pluggable TTS engine (voxtral | qwen3) \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "github-releaser-agent#8", + "owner": "bborbe", + "repo": "github-releaser-agent", + "number": 8, + "language": "go", + "merge_strategy": "merge-commit", + "merge_sha": "98ab8544a11efd5ec1dda7f7aaf524617d8fcfd0", + "base_sha": "e6b692828ea8a404e538adc3d40b6efd56819b9f", + "head_sha": "51736bf1889d6711b0585fdf3a40b7ba8534cb50", + "changed_files": 12, + "additions": 1983, + "deletions": 41, + "role": "unreviewed", + "notes": "feat(planning): close nothing-to-release tasks instead of escalating \u2014 sourced 2026-08-09, role pending adjudication" + }, + { + "id": "discord-assistant#1", + "owner": "bborbe", + "repo": "discord-assistant", + "number": 1, + "language": "node", + "merge_strategy": "squash", + "merge_sha": "aa4ea180a3c26ada0f7f4f1fd618c300ec58d7b2", + "base_sha": "4e4c21197a6f9e794608075357265cfcfcc3df08", + "head_sha": "0123af7a9b003b46a4145ef763b1586d853de940", + "changed_files": 4, + "additions": 98, + "deletions": 1, + "role": "squash-coverage", + "notes": "survive gateway network faults and restart on exit \u2014 one of only 2 squash-merged PRs in the org" + }, + { + "id": "helm#3", + "owner": "bborbe", + "repo": "helm", + "number": 3, + "language": "go", + "merge_strategy": "squash", + "merge_sha": "584cb9f23f556287b82b0bfb64489bea9a6a40e1", + "base_sha": "15a6b84b57b37b5de151a568bc9c879a5b91d9f8", + "head_sha": "e634f85ab0ccdda380405ec3482732ce1ee1ff18", + "changed_files": 2, + "additions": 13, + "deletions": 3, + "role": "squash-coverage", + "notes": "trust ben-s-go-updater[bot] in github-pr-reviewer TRUSTED_AUTHORS \u2014 one of only 2 squash-merged PRs in the org" } ] } diff --git a/bench/test_config.py b/bench/test_config.py index 7996d47..82c3010 100644 --- a/bench/test_config.py +++ b/bench/test_config.py @@ -196,10 +196,35 @@ def test_load_manifest_rejects_traversal_owner(self): self.assertIn(case.get("owner") or case.get("repo"), str(ctx.exception)) def test_load_manifest_accepts_real_fixture(self): - """load_manifest on the frozen bench/prs.json succeeds and returns dev-1 with 5 entries.""" + """load_manifest on the shipped bench/prs.json returns curated-1 with 20 entries. + + Pinned on purpose: the version and count are part of the config identity, + so a manifest change must be a conscious edit here rather than something + a scored run discovers. `dev-1`/5 was the 5-PR development fixture; its + 3-sigma was ~119% of the mean, which is why it could not carry a score. + """ + m = run.load_manifest(run.BENCH_DIR / "prs.json") + self.assertEqual(m["version"], "curated-1") + self.assertEqual(len(m["prs"]), 20) + + def test_curated_manifest_holds_the_spread_it_was_curated_for(self): + """The fixture's whole purpose is spread; assert it rather than trust it.""" m = run.load_manifest(run.BENCH_DIR / "prs.json") - self.assertEqual(m["version"], "dev-1") - self.assertEqual(len(m["prs"]), 5) + + sizes = [p["additions"] + p["deletions"] for p in m["prs"]] + self.assertLessEqual(min(sizes), 20, "no small PR in the set") + self.assertGreaterEqual(max(sizes), 1000, "no large PR in the set") + + repos = {p["repo"] for p in m["prs"]} + self.assertGreaterEqual(len(repos), 10, f"too few distinct repos: {repos}") + + strategies = {p["merge_strategy"] for p in m["prs"]} + # Both diff-reconstruction paths must be exercised: a merge-commit uses + # `^1..^2`, a squash has one parent and needs the manifest's + # base_sha..head_sha. A fixture carrying only one shape leaves the other + # branch untested. + self.assertIn("squash", strategies) + self.assertIn("merge-commit", strategies) class TestPluginResolution(unittest.TestCase):