From d8032c6d2e4f3529dd11366c76b049c45b1751a9 Mon Sep 17 00:00:00 2001 From: bompus Date: Sun, 18 Oct 2026 23:00:11 -0600 Subject: [PATCH 1/3] eval: gate retrieval changes by replaying saved agent queries replay-queries.mjs runs a corpus of real codegraph_explore queries against a base and a candidate build and fails when an answer loses its source section, a rendered file or a pattern. harvest-queries.mjs collects those queries from Claude Code and Codex transcripts, keeping only queries on repositories named as public. A changed answer is read and justified in the PR; the agent A/B moves to new languages and frameworks and once per release batch. --- AGENTS.md | 2 + __tests__/query-replay.test.ts | 68 ++++++++ docs/AGENTS.md | 13 +- scripts/agent-eval/harvest-queries.mjs | 123 ++++++++++++++ scripts/agent-eval/replay-queries.mjs | 100 ++++++++++++ scripts/agent-eval/replay/codegraph.json | 194 +++++++++++++++++++++++ 6 files changed, 499 insertions(+), 1 deletion(-) create mode 100644 __tests__/query-replay.test.ts create mode 100644 scripts/agent-eval/harvest-queries.mjs create mode 100644 scripts/agent-eval/replay-queries.mjs create mode 100644 scripts/agent-eval/replay/codegraph.json diff --git a/AGENTS.md b/AGENTS.md index 86faadb2d0..8a783c0d7b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,6 +94,8 @@ Defined in `src/types.ts`. Both extractors and resolvers must use these exact st **Required** for every new language/framework: validate on small/medium/large real repos with >=3 flow prompts; deterministic `scripts/agent-eval/probe-explore.mjs` probes, then agent A/B (`scripts/agent-eval/run-all.sh` / `ab-new-vs-baseline.sh`). Pass bar: ~0 Read/Grep within the explore-call budget, faster than without-codegraph, no control-repo regression. +**Every other retrieval change** (explore ranking, budgets, rendering, search) is gated by replaying saved agent queries, not by an agent A/B: `node scripts/agent-eval/replay-queries.mjs --tasks scripts/agent-eval/replay/.json --base --cand --out ` (about 20 s). Read every changed or regressed answer and say in the PR why each is acceptable. Run one agent A/B per release batch, before a build is promoted or released, and add its new query shapes to the corpus with `harvest-queries.mjs`. Details: `docs/AGENTS.md` § Query replay. + Full methodology (feedback metrics, CLI contamination guard, Sonnet/`--effort high` model policy, daemon pre-warm), the Excalidraw worked example, and coverage matrix live in: - `docs/AGENTS.md` (nested; also loaded when cwd is under `docs/`) - `docs/design/dynamic-dispatch-coverage-playbook.md` diff --git a/__tests__/query-replay.test.ts b/__tests__/query-replay.test.ts new file mode 100644 index 0000000000..150859539a --- /dev/null +++ b/__tests__/query-replay.test.ts @@ -0,0 +1,68 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; +// @ts-expect-error untyped ESM script +import { callsInLine, repoSlug } from '../scripts/agent-eval/harvest-queries.mjs'; +// @ts-expect-error untyped ESM script +import { compareRow } from '../scripts/agent-eval/replay-queries.mjs'; + +const row = (files: string[], over: Record = {}) => ({ + id: 'q', chars: 100, sourceAt: 10, files, mustMatchAt: [], ...over, +}); + +describe('replay comparison', () => { + it('passes the same files in another order', () => { + const r = compareRow(row(['a.ts', 'b.ts']), row(['b.ts', 'a.ts'])); + expect(r.regressed).toBe(false); + expect(r.changed).toBe(false); + }); + + it('flags a lost file, and names it when the query asked for it', () => { + const r = compareRow(row(['a.ts', 'src/b.ts']), row(['a.ts']), 'explain src/b.ts'); + expect(r.regressed).toBe(true); + expect(r.lostFiles).toEqual(['src/b.ts']); + expect(r.lostNamed).toEqual(['src/b.ts']); + }); + + it('flags a lost source section and a pattern the base found', () => { + expect(compareRow(row(['a.ts']), row([], { sourceAt: -1 })).lostSource).toBe(true); + expect(compareRow(row([], { mustMatchAt: [5] }), row([], { mustMatchAt: [-1] })).lostMatch).toBe(1); + }); + + it('counts a gained file as a change without loss', () => { + const r = compareRow(row(['a.ts']), row(['a.ts', 'c.ts'])); + expect(r).toMatchObject({ regressed: false, changed: true, gainedFiles: ['c.ts'] }); + }); +}); + +describe('query harvest', () => { + it('reads Claude tool_use, Codex function calls and Codex code-mode calls', () => { + expect(callsInLine({ message: { content: [{ type: 'tool_use', name: 'mcp__codegraph__codegraph_explore', input: { query: 'A b', projectPath: '/x' } }] } })) + .toEqual([{ query: 'A b', projectPath: '/x' }]); + expect(callsInLine({ payload: { type: 'function_call', name: 'mcp__codegraph__codegraph_explore', arguments: '{"query":"C"}' } })) + .toEqual([{ query: 'C', projectPath: undefined }]); + const input = 'text(await tools.mcp__codegraph__codegraph_explore({projectPath:"/y",query:"D \\"e\\"",maxFiles:8}));'; + expect(callsInLine({ payload: { type: 'custom_tool_call', input } })).toEqual([{ query: 'D "e"', projectPath: '/y' }]); + expect(callsInLine({ message: { content: [{ type: 'tool_use', name: 'Read', input: { query: 'no' } }] } })).toEqual([]); + }); + + it('maps GitHub remotes to owner/repo', () => { + expect(repoSlug('git@github.com:Owner/Repo.git\n')).toBe('owner/repo'); + expect(repoSlug('https://github.com/owner/repo')).toBe('owner/repo'); + expect(repoSlug('https://gitlab.com/owner/repo.git')).toBeNull(); + }); +}); + +describe('committed replay corpora', () => { + const dir = resolve(import.meta.dirname, '../scripts/agent-eval/replay'); + for (const name of readdirSync(dir).filter((f) => f.endsWith('.json'))) { + it(`${name} is pinned, unique and has no local paths`, () => { + const text = readFileSync(join(dir, name), 'utf8'); + const corpus = JSON.parse(text); + expect(corpus.commit).toMatch(/^[0-9a-f]{7,40}$/); + const ids = corpus.tasks.map((t: { id: string }) => t.id); + expect(new Set(ids).size).toBe(ids.length); + expect(text).not.toMatch(/\/home\/|\/Users\/|[A-Z]:\\\\|~\//); + }); + } +}); diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 9a2a6d0769..efd0e00e1f 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -10,7 +10,7 @@ For each **language × framework**, validate on **small, medium, and large** rea 1. **Pick the canonical flow** for the framework ("how does X reach Y": state→render, request→handler→view, query→SQL, action→reducer→store…). 2. **Deterministic probes** (`scripts/agent-eval/probe-{node,explore}.mjs` against the built `dist/`): `codegraph_explore` with the flow's symbol names connects from→to end-to-end with no break (its Flow section shows the path); **no node explosion** (`select count(*) from nodes` stable before/after re-index); synthesized-edge **precision** spot-check (`select … where provenance='heuristic'`). -3. **Agent A/B** (`scripts/agent-eval/run-all.sh ""`): with vs without codegraph, **≥2 runs/arm** (run-to-run variance is large — never conclude from n=1). Record **duration, total tool calls, Read, Grep**. Optional forced-Read-0 sufficiency proof via the block-read hook (`scripts/agent-eval/hook-settings.json`); export `CODEGRAPH_REPO_ROOT` as this checkout's absolute path before launching Claude. +3. **Agent A/B** (`scripts/agent-eval/run-all.sh ""`), for a new language or framework and once per release batch (a change to existing retrieval uses the query replay below instead): with vs without codegraph, **≥2 runs/arm** (run-to-run variance is large — never conclude from n=1). Record **duration, total tool calls, Read, Grep**. Optional forced-Read-0 sufficiency proof via the block-read hook (`scripts/agent-eval/hook-settings.json`); export `CODEGRAPH_REPO_ROOT` as this checkout's absolute path before launching Claude. - **Every run also reports three feedback metrics** — residual context occupancy, explore sufficiency (what the agent did NEXT after each explore), and allocation efficiency (share of returned bytes the answer cited) — under each run, plus a side-by-side arm table (`compare-arms.mjs`). Entry point: `docs/benchmarks/agent-eval-feedback-metrics.md`. Reading them: `Read a file we returned` is an allocation miss, `Read a file we did NOT return`/`Grep` is recall; allocation efficiency is **relative** (attribution is by citation) so it is only valid between builds on the same question; occupancy *shares* are Claude Code / 200k and don't transfer to another host — the arm ratio does. - **The `codegraph` CLI is blocked in every arm** (`no-cli-shim.sh`: sanitized PATH + a PreToolUse hook, shared by both harnesses). Without it 14 of 15 without-arm runs in one 7-repo pass reached codegraph through Bash. Check the contamination row before believing any number: `CLI calls that RETURNED output` > 0 invalidates the run (in a new-vs-baseline A/B it silently drops calls from all three metrics, since a CLI explore is not a tool call). - **Model selection and availability.** The Claude harness defaults to `--model sonnet --effort high` for floor-model validation. If that model is unavailable through quota, an outage or an unsupported route, use the next available eligible model and effort from the operator's current benchmark-backed bug-finding order. Primary, secondary and tertiary choices change with benchmark results; read the current selection rather than fixing fallback names here. Record the selected model, effort, ranking source and fallback reason. @@ -21,6 +21,17 @@ For each **language × framework**, validate on **small, medium, and large** rea Full playbook + per-mechanism design: `docs/design/dynamic-dispatch-coverage-playbook.md` and `docs/design/callback-edge-synthesis.md`. +### Query replay (gate for changes to existing retrieval) + +A change to explore ranking, budgets, rendering or search replays real agent queries against the base and candidate builds instead of running an agent A/B. The past change-gating A/Bs mostly agreed with the deterministic probes; the defects only agents caught came from query shapes the probes lacked, and replaying those agents' queries reproduced each one. The corpus keeps those shapes. + +1. Check out the corpus repository at the corpus's `commit` and index it with either build (`codegraph init -y`). The replay refuses a checkout at another commit. +2. `node scripts/agent-eval/replay-queries.mjs --tasks scripts/agent-eval/replay/.json --base --cand --out ` (each build is a checkout with `dist/` built). It prints a table of changed answers and exits 1 when an answer lost its source section, a file the base rendered (marked when the query names the file) or a `mustMatch` pattern. Answers are saved under `/base` and `/cand`. +3. Read every changed and regressed answer. A lost file can be a fix (a peripheral file swapped for a better one); say so per query in the PR. Same-build replay is identical, so any change comes from the candidate. +4. Once per release batch, before a build is promoted or released, run one agent A/B (`ab-new-vs-baseline.sh`) on a flow question the corpus covers least, then harvest its queries: `node scripts/agent-eval/harvest-queries.mjs --public ...`. It keeps queries only on repositories named with `--public` and writes no paths; still read every prompt before committing, since a public query can quote a private name. `__tests__/query-replay.test.ts` fails a committed corpus that contains a local path. + +Measured on `replay/codegraph.json` (46 queries, this repository at `2a4af7c3`): same build 46 of 46 identical in 9 s; a build capped at two files per answer flagged 39; the root `__tests__/` test-path fix (#500) changed 5 answers and flagged 1, where test files the query did not name were swapped for other test files. + ### Worked example — Excalidraw (TS/React, medium, 643 files) This table records the historical trace-era evaluation; current validation uses diff --git a/scripts/agent-eval/harvest-queries.mjs b/scripts/agent-eval/harvest-queries.mjs new file mode 100644 index 0000000000..e0fa3189ab --- /dev/null +++ b/scripts/agent-eval/harvest-queries.mjs @@ -0,0 +1,123 @@ +#!/usr/bin/env node +// Collect the codegraph_explore queries real agent sessions asked, for replay +// with replay-queries.mjs. +// +// node scripts/agent-eval/harvest-queries.mjs --public [--public ...] [--since 2026-09-01] ... +// +// Reads Claude Code transcripts (assistant tool_use blocks) and Codex rollouts +// (MCP tool calls and code-mode `tools.mcp__codegraph__codegraph_explore({...})` +// calls). Each call's project is its projectPath, else the session's working +// directory, mapped to a repository through `git remote get-url origin`. Only +// queries on a repository named by --public are kept; everything else is +// dropped and counted, so a private project never reaches the output. Output +// on stdout: `{ repos: { "": { tasks: [{ id, prompt }] } } }`, +// deduplicated, with no paths. Review every prompt before committing it: a +// query on a public repository can still quote a private name. +import { execFileSync } from 'node:child_process'; +import { createReadStream, readdirSync, statSync } from 'node:fs'; +import { createInterface } from 'node:readline'; +import { join } from 'node:path'; +import { createHash } from 'node:crypto'; + +/** Explore calls in one transcript line: [{ query, projectPath? }]. */ +export function callsInLine(entry) { + const calls = []; + const blocks = entry?.message?.content; + if (Array.isArray(blocks)) { + for (const b of blocks) { + if (b?.type === 'tool_use' && /codegraph_explore$/.test(b.name ?? '') && typeof b.input?.query === 'string') { + calls.push({ query: b.input.query, projectPath: b.input.projectPath }); + } + } + } + const p = entry?.payload; + if (p?.type === 'function_call' && /codegraph_explore$/.test(p.name ?? '')) { + try { + const args = JSON.parse(p.arguments); + if (typeof args.query === 'string') calls.push({ query: args.query, projectPath: args.projectPath }); + } catch { /* not JSON arguments */ } + } + if (p?.type === 'custom_tool_call' && typeof p.input === 'string') { + for (const m of p.input.matchAll(/codegraph_explore\(\{([^}]*)\}\)/g)) { + const field = (name) => { + const f = new RegExp(`\\b${name}\\s*:\\s*("(?:[^"\\\\]|\\\\.)*")`).exec(m[1]); + try { return f ? JSON.parse(f[1]) : undefined; } catch { return undefined; } + }; + const query = field('query'); + if (typeof query === 'string') calls.push({ query, projectPath: field('projectPath') }); + } + } + return calls; +} + +/** `owner/repo` from a GitHub remote URL, or null. */ +export function repoSlug(url) { + const m = /github\.com[:/]([^/\s]+)\/([^/\s]+?)(?:\.git)?\/?$/.exec(url.trim()); + return m ? `${m[1]}/${m[2]}`.toLowerCase() : null; +} + +function* jsonlFiles(path) { + const st = statSync(path, { throwIfNoEntry: false }); + if (!st) return; + if (st.isFile()) { if (path.endsWith('.jsonl')) yield path; return; } + for (const name of readdirSync(path)) yield* jsonlFiles(join(path, name)); +} + +async function main(argv) { + const pub = new Set(); + const inputs = []; + let since = ''; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--public') pub.add(argv[++i].toLowerCase()); + else if (argv[i] === '--since') since = argv[++i]; + else inputs.push(argv[i]); + } + if (!pub.size || !inputs.length) { + console.error('usage: harvest-queries.mjs --public [--public ...] [--since YYYY-MM-DD] ...'); + return 2; + } + const slugOf = new Map(); + const slugFor = (dir) => { + if (!slugOf.has(dir)) { + let slug = null; + try { + slug = repoSlug(execFileSync('git', ['-C', dir, 'remote', 'get-url', 'origin'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] })); + } catch { /* gone or not a repository */ } + slugOf.set(dir, slug); + } + return slugOf.get(dir); + }; + const repos = {}; + const seen = new Set(); + let kept = 0, dropped = 0, duplicate = 0; + for (const input of inputs) { + for (const file of jsonlFiles(input)) { + let cwd; + // Line by line: a long transcript can exceed the largest string a runtime allows. + for await (const line of createInterface({ input: createReadStream(file, 'utf8'), crlfDelay: Infinity })) { + if (!line.includes('codegraph_explore') && !line.includes('"cwd"')) continue; + let entry; + try { entry = JSON.parse(line); } catch { continue; } + cwd = entry.cwd ?? entry.payload?.cwd ?? cwd; + if (since && (entry.timestamp ?? '') < since) continue; + for (const call of callsInLine(entry)) { + const dir = call.projectPath ?? cwd; + const slug = dir ? slugFor(dir) : null; + if (!slug || !pub.has(slug)) { dropped++; continue; } + const prompt = call.query.trim().replace(/\s+/g, ' '); + const key = `${slug}\0${prompt.toLowerCase()}`; + if (seen.has(key)) { duplicate++; continue; } + seen.add(key); + const id = createHash('sha256').update(prompt).digest('hex').slice(0, 10); + (repos[slug] ??= { tasks: [] }).tasks.push({ id: `q-${id}`, prompt }); + kept++; + } + } + } + } + console.log(JSON.stringify({ repos }, null, 2)); + console.error(`kept ${kept}, duplicates ${duplicate}, dropped ${dropped} (not on a --public repository)`); + return 0; +} + +if (import.meta.url === `file://${process.argv[1]}`) main(process.argv.slice(2)).then((code) => process.exit(code)); diff --git a/scripts/agent-eval/replay-queries.mjs b/scripts/agent-eval/replay-queries.mjs new file mode 100644 index 0000000000..2ae3b4f632 --- /dev/null +++ b/scripts/agent-eval/replay-queries.mjs @@ -0,0 +1,100 @@ +#!/usr/bin/env node +// Replay saved agent queries against two builds and diff the answers. +// +// node scripts/agent-eval/replay-queries.mjs --tasks --base --cand --out [--ids globs] +// +// is an indexed checkout at the corpus's pinned commit (the corpus file +// names it). Each build answers every query through probe-explore.mjs in its +// own process. A query regresses when the candidate's answer loses its Source +// Code section, drops a source file the base rendered (worst when the query +// names that file), or misses a mustMatch pattern the base found. Prints a Markdown table of changed queries and a +// summary line; exits 1 on any regression, 0 otherwise. Answers are saved +// under /base and /cand for reading the diff; keep off /tmp. +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +/** How one query's answer moved from base to candidate. */ +export function compareRow(base, cand, prompt = '') { + const lostFiles = base.files.filter((f) => !cand.files.includes(f)); + const lostNamed = lostFiles.filter((f) => prompt.includes(f)); + const gainedFiles = cand.files.filter((f) => !base.files.includes(f)); + const lostSource = base.sourceAt >= 0 && cand.sourceAt < 0; + const lostMatch = base.mustMatchAt.filter((at, i) => at >= 0 && (cand.mustMatchAt[i] ?? -1) < 0).length; + const regressed = lostSource || lostFiles.length > 0 || lostMatch > 0; + const changed = regressed || gainedFiles.length > 0 || base.chars !== cand.chars; + return { id: base.id, regressed, changed, lostSource, lostFiles, lostNamed, gainedFiles, lostMatch, baseChars: base.chars, candChars: cand.chars }; +} + +function probe(build, repo, tasks, ids, out) { + const script = fileURLToPath(new URL('./probe-explore.mjs', import.meta.url)); + const args = [script, repo, '--tasks', tasks, '--build', build, '--out', out]; + if (ids) args.push('--ids', ids); + const res = spawnSync(process.execPath, args, { encoding: 'utf8', maxBuffer: 64 << 20, stdio: ['ignore', 'pipe', 'inherit'] }); + if (res.status !== 0) throw new Error(`probe-explore with ${build} exited ${res.status}`); + const rows = new Map(); + for (const line of res.stdout.split('\n')) { + if (!line.startsWith('{')) continue; + const row = JSON.parse(line); + if (row.id !== 'memory') rows.set(row.id, row); + } + return rows; +} + +function main() { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { tasks: { type: 'string' }, base: { type: 'string' }, cand: { type: 'string' }, out: { type: 'string' }, ids: { type: 'string' } }, + }); + const [repoArg] = positionals; + if (!repoArg || !values.tasks || !values.base || !values.cand || !values.out) { + console.error('usage: replay-queries.mjs --tasks --base --cand --out [--ids globs]'); + return 2; + } + const repo = resolve(repoArg); + const out = resolve(values.out); + const corpus = JSON.parse(readFileSync(values.tasks, 'utf8')); + if (corpus.commit) { + const head = spawnSync('git', ['-C', repo, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).stdout.trim(); + if (!head.startsWith(corpus.commit)) { + console.error(`${repo} is at ${head.slice(0, 12)}, the corpus is pinned to ${corpus.commit}; answers would not be comparable`); + return 2; + } + } + const base = probe(resolve(values.base), repo, values.tasks, values.ids, join(out, 'base')); + const cand = probe(resolve(values.cand), repo, values.tasks, values.ids, join(out, 'cand')); + const prompts = new Map(corpus.tasks.map((t) => [t.id, t.prompt])); + const rows = [...base.values()].map((b) => { + const c = cand.get(b.id); + if (!c) throw new Error(`candidate has no answer for ${b.id}`); + return compareRow(b, c, prompts.get(b.id)); + }); + const changed = rows.filter((r) => r.changed); + const regressed = rows.filter((r) => r.regressed); + if (changed.length) { + console.log('| Query | Result | Chars (base → cand) | Files lost | Files gained |'); + console.log('|---|---|---:|---|---|'); + for (const r of changed) { + const result = !r.regressed ? 'changed' + : r.lostSource ? 'REGRESSED: no source' + : r.lostNamed.length ? 'REGRESSED: named file lost' + : r.lostMatch ? 'REGRESSED: pattern missing' + : 'REGRESSED: file lost'; + console.log(`| ${r.id} | ${result} | ${r.baseChars} → ${r.candChars} | ${r.lostFiles.join(', ') || '-'} | ${r.gainedFiles.join(', ') || '-'} |`); + } + console.log(''); + } + console.log(`${rows.length} queries: ${rows.length - changed.length} identical, ${changed.length - regressed.length} changed without loss, ${regressed.length} regressed. Answers: ${out}`); + return regressed.length ? 1 : 0; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + process.exit(main()); + } catch (error) { + console.error(`[replay-queries] ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } +} diff --git a/scripts/agent-eval/replay/codegraph.json b/scripts/agent-eval/replay/codegraph.json new file mode 100644 index 0000000000..cf2066f16e --- /dev/null +++ b/scripts/agent-eval/replay/codegraph.json @@ -0,0 +1,194 @@ +{ + "$comment": [ + "Real codegraph_explore queries agents asked about this repository, harvested by harvest-queries.mjs and reviewed by hand.", + "Replay them against bompus/codegraph checked out at commit, indexed with either build: node scripts/agent-eval/replay-queries.mjs --tasks scripts/agent-eval/replay/codegraph.json --base --cand --out ." + ], + "repository": "bompus/codegraph", + "commit": "2a4af7c3", + "tasks": [ + { + "id": "q-89489f4611", + "prompt": "is_lexically_reachable is_bare_js_call js_store_binding js_unit_to_byte match_by_exact_name" + }, + { + "id": "q-5a5324c072", + "prompt": "call_shape_target kotlin_call_receiver_type call_site kotlin_unknown_receiver_allowed kotlin_infix_site retain_lang_scope_tracked" + }, + { + "id": "q-58b626100e", + "prompt": "KernelResolver read_file source cache sorted_files kotlin_companion_receiver_memo" + }, + { + "id": "q-0bf6fd8205", + "prompt": "getExploreBudget getExploreOutputBudget src/mcp/tools.ts" + }, + { + "id": "q-4ce3a49f75", + "prompt": "writer lock CODEGRAPH_NO_DAEMON readonly no watch serve mcp auto sync environment" + }, + { + "id": "q-5329a0730f", + "prompt": "server-instructions.ts SERVER_INSTRUCTIONS" + }, + { + "id": "q-2c1bab2282", + "prompt": "instructions-template.ts INSTRUCTIONS_BODY upsertInstructionsEntry CODEGRAPH_START" + }, + { + "id": "q-bb5c2f3458", + "prompt": "CHANGELOG.md Unreleased New Features Fixes" + }, + { + "id": "q-722771cf35", + "prompt": "writer.pid fallback mode daemon start writer lock acquire SIGTERM shutdown clients idle timeout refresh launcher fallback proxy existing MCP session" + }, + { + "id": "q-83904696a8", + "prompt": "handleExplore getExploreOutputBudget pinnedFiles source excerpts exact path literal selection Vue test assertions" + }, + { + "id": "q-3e323a81de", + "prompt": "src/extraction/index.ts extractFileWithParser retryOnCorruption: trace WASM heap recreation retry guards to parseNativeTree" + }, + { + "id": "q-e298f733b6", + "prompt": "TreeSitterExtractor extract isTreeSitterTimeoutError parseAst src/extraction/parse-worker.ts: how are native parser errors and memory messages propagated to retries?" + }, + { + "id": "q-f194f1d4e3", + "prompt": "src/extraction/parse-pool.ts ParseWorkerPool recycle recycleAll handleWorkerError onMessage timeout: which live native worker errors trigger restart and retry?" + }, + { + "id": "q-79895a605a", + "prompt": "src/mcp explicit project idle writer lock LRU release structuredContent text-only stale explore status; codegraph-kernel/src/resolve/bound.rs lexical siblings closure declaration method_call shadow field" + }, + { + "id": "q-4b9d4fe1e9", + "prompt": "src/mcp/project-lifecycle.ts src/mcp/tools.ts trimProjects setProjectLifecycle src/mcp/handlers/base.ts structuredContent textResult __tests__ idle explicit project release stale text-only structuredContent" + }, + { + "id": "q-82c2bc4abb", + "prompt": "__tests__/mcp-projectpath-lifecycle.test.ts __tests__/mcp-idle-release-order.test.ts __tests__/mcp-stale-refusal.test.ts __tests__/mcp-status-freshness.test.ts" + }, + { + "id": "q-56ad613635", + "prompt": "codegraph-kernel/src/resolve/receivers.rs infer_local_receiver_type codegraph-kernel/src/resolve/names.rs innermost_binding codegraph-kernel/src/resolve/method_call.rs receiver_field_type __tests__/bindings-receiver.test.ts" + }, + { + "id": "q-5bee81ff55", + "prompt": "codegraph-kernel/src/resolve/prefilter.rs innermost_binding codegraph-kernel/src/resolve/method_call.rs match_method_call java_field_receiver __tests__/bindings-receiver.test.ts __tests__/receiver-owners.test.ts" + }, + { + "id": "q-84a7939e6c", + "prompt": "__tests__/bindings-receiver.test.ts __tests__/bound-owner-walk.test.ts codegraph-kernel/src/tsjs/bindings.rs local_scope codegraph-kernel/src/go/bindings.rs" + }, + { + "id": "q-e3cbe24999", + "prompt": "infer_java_field_receiver_type infer_declared_member_receiver_type jvm_field_receiver" + }, + { + "id": "q-2195039d4e", + "prompt": "SQLite database driver node:sqlite read-only worker pool query-worker database connection" + }, + { + "id": "q-1795f3a486", + "prompt": "src/mcp/tools.ts trimProjects DEFAULT_PROJECT_IDLE getProject defaultProject src/mcp/project-lifecycle.ts writer lock fallback idle release" + }, + { + "id": "q-9c8c79ed8a", + "prompt": "__tests__/mcp-project-idle.test.ts idleRelease engine default project CODEGRAPH_PROJECT_IDLE_TIMEOUT_MS" + }, + { + "id": "q-d3903019db", + "prompt": "__tests__/mcp-projectpath-lifecycle.test.ts __tests__/mcp-idle-release-order.test.ts __tests__/mcp-writer-lock.test.ts" + }, + { + "id": "q-4bfc6b3a39", + "prompt": "src/mcp/proxy.ts ensureEngine retireEngine scheduleDaemonRetry CODEGRAPH_DAEMON_RETRY_MS src/mcp/engine.ts ensureInitialized stop defaultLease" + }, + { + "id": "q-31969f6470", + "prompt": "src/mcp/proxy.ts LocalHandshakeDeps runLocalHandshakeProxy stdin onData getDaemonSocket makeEngine __tests__/proxy-shutdown.test.ts __tests__/mcp-daemon.test.ts fallback retries hand writer" + }, + { + "id": "q-66cb0e759e", + "prompt": "src/codegraph.ts initSync src/kernel/loader.ts require codegraph-kernel src/mcp/engine.ts catchUpSync startWatching" + }, + { + "id": "q-ee1a00ea8d", + "prompt": "__tests__/proxy-shutdown.test.ts startProxy env src/mcp/liveness-watchdog.ts installMainThreadWatchdog env" + }, + { + "id": "q-df62321741", + "prompt": "kernel build stamp untracked source invalidation build-kernel scripts build stamp hash git ls-files" + }, + { + "id": "q-1f48df9dba", + "prompt": "scripts/kernel-stamp.mjs scripts/ensure-kernel.mjs __tests__/ensure-kernel-stamp.test.ts" + }, + { + "id": "q-f88ad05305", + "prompt": "src/mcp/tools.ts callersTool calleesTool codegraph_callers codegraph_callees" + }, + { + "id": "q-875f4f64e3", + "prompt": "handleCallers handleCallees collectWiringNotes nodeHeader" + }, + { + "id": "q-3ae4233c4b", + "prompt": "formatNodeList edgeLabel GraphTraverser getCallers getCallees" + }, + { + "id": "q-104036dbc2", + "prompt": "scripts/build-kernel.mjs scripts/ensure-kernel.mjs runtime-control.ts checkRuntimeReady protocol version runtime identity build stamp" + }, + { + "id": "q-f9910072c8", + "prompt": "src/mcp/daemon-registry.ts getRegistryDir src/directory.ts getCodeGraphStateDir registry dir global config" + }, + { + "id": "q-60d91393ac", + "prompt": "src/mcp/version.ts resolveManagedBuildVersion managed source stamp file" + }, + { + "id": "q-9c3899c519", + "prompt": "src/codegraph.ts codegraph-kernel/src/lib.rs src/db/sqlite-adapter.ts src/mcp/server.ts runtime architecture Rust kernel Node transport public library API ui browser" + }, + { + "id": "q-f795406e75", + "prompt": "codegraph-kernel/src/lib.rs codegraph-kernel/src/resolve/name_matcher_upstream.rs native Rust #[cfg(test)] tests: identify a pure helper with direct Rust unit tests usable by cargo test; contrast __tests__/kernel-scaffold.test.ts and scripts/ensure-kernel.mjs NAPI prebuild test wiring" + }, + { + "id": "q-da10d1df74", + "prompt": "codegraph-kernel/src/ids.rs codegraph-kernel/src/docstring.rs codegraph-kernel/src/lib.rs cfg(test) NAPI test stubs" + }, + { + "id": "q-a49ca91c4e", + "prompt": "codegraph-kernel/src/util.rs col16 utf16 tests" + }, + { + "id": "q-ebde6c9ef9", + "prompt": "Cols col utf16_prefix codegraph-kernel/src/textutil.rs callers parse_tree_inner" + }, + { + "id": "q-a3cbcd491c", + "prompt": "__tests__/kernel-parse-tree.test.ts UTF-16 unicode astral" + }, + { + "id": "q-57dc520bed", + "prompt": "src/extraction/kernel/tree.ts NativeNode startIndex endIndex startPosition descendantForIndex descendantForPosition" + }, + { + "id": "q-872326d0b3", + "prompt": "codegraph-kernel/src/tree.rs parse_tree_inner Row start_byte start_idx utf16_prefix src/extraction/parse-tree.ts TreeNode" + }, + { + "id": "q-621688899a", + "prompt": "src/graph/branch-guards.ts innermostAt __tests__/kernel-parse-tree.test.ts firstViolation" + }, + { + "id": "q-c0cacd2a20", + "prompt": "createDatabase SqliteDatabase src/db/schema.sql FTS5" + } + ] +} From 591d574290f0756b333679b802d443dc2134c01b Mon Sep 17 00:00:00 2001 From: bompus Date: Sun, 18 Oct 2026 23:12:27 -0600 Subject: [PATCH 2/3] eval: compare replayed answer text, keep file limits, tighten the public-repository filter Equal-length answers with different text now count as changed. The harvest reads code-mode calls with braces or single quotes in a string, keeps each call's maxFiles, and probe-explore passes it on. A remote counts only when its host is github.com itself. An --ids filter that selects nothing exits 2 instead of passing. --- __tests__/query-replay.test.ts | 19 ++- docs/AGENTS.md | 4 +- scripts/agent-eval/harvest-queries.mjs | 86 +++++++++++--- scripts/agent-eval/probe-explore.mjs | 14 ++- scripts/agent-eval/replay-queries.mjs | 11 +- scripts/agent-eval/replay/codegraph.json | 145 ++++++++++++++--------- 6 files changed, 191 insertions(+), 88 deletions(-) diff --git a/__tests__/query-replay.test.ts b/__tests__/query-replay.test.ts index 150859539a..97ba75fc26 100644 --- a/__tests__/query-replay.test.ts +++ b/__tests__/query-replay.test.ts @@ -29,6 +29,11 @@ describe('replay comparison', () => { expect(compareRow(row([], { mustMatchAt: [5] }), row([], { mustMatchAt: [-1] })).lostMatch).toBe(1); }); + it('counts different text with the same files and length as a change', () => { + const r = compareRow(row(['a.ts']), row(['a.ts']), '', false); + expect(r).toMatchObject({ regressed: false, changed: true }); + }); + it('counts a gained file as a change without loss', () => { const r = compareRow(row(['a.ts']), row(['a.ts', 'c.ts'])); expect(r).toMatchObject({ regressed: false, changed: true, gainedFiles: ['c.ts'] }); @@ -42,14 +47,26 @@ describe('query harvest', () => { expect(callsInLine({ payload: { type: 'function_call', name: 'mcp__codegraph__codegraph_explore', arguments: '{"query":"C"}' } })) .toEqual([{ query: 'C', projectPath: undefined }]); const input = 'text(await tools.mcp__codegraph__codegraph_explore({projectPath:"/y",query:"D \\"e\\"",maxFiles:8}));'; - expect(callsInLine({ payload: { type: 'custom_tool_call', input } })).toEqual([{ query: 'D "e"', projectPath: '/y' }]); + expect(callsInLine({ payload: { type: 'custom_tool_call', input } })).toEqual([{ query: 'D "e"', projectPath: '/y', maxFiles: 8 }]); expect(callsInLine({ message: { content: [{ type: 'tool_use', name: 'Read', input: { query: 'no' } }] } })).toEqual([]); }); + it('reads code-mode calls with braces, single quotes and file limits', () => { + const call = (input: string) => callsInLine({ payload: { type: 'custom_tool_call', input } }); + expect(call('codegraph_explore({query:"make({a: 1}) } brace"})')).toEqual([{ query: 'make({a: 1}) } brace', projectPath: undefined }]); + expect(call("codegraph_explore({ query: 'auth flow', maxFiles: 12 })")).toEqual([{ query: 'auth flow', projectPath: undefined, maxFiles: 12 }]); + expect(call('codegraph_explore({query:`a ${x}`})')).toEqual([]); + expect(callsInLine({ message: { content: [{ type: 'tool_use', name: 'mcp__codegraph__codegraph_explore', input: { query: 'Q', maxFiles: 3 } }] } })) + .toEqual([{ query: 'Q', projectPath: undefined, maxFiles: 3 }]); + }); + it('maps GitHub remotes to owner/repo', () => { expect(repoSlug('git@github.com:Owner/Repo.git\n')).toBe('owner/repo'); expect(repoSlug('https://github.com/owner/repo')).toBe('owner/repo'); + expect(repoSlug('ssh://git@github.com/owner/repo.git')).toBe('owner/repo'); expect(repoSlug('https://gitlab.com/owner/repo.git')).toBeNull(); + expect(repoSlug('https://notgithub.com/owner/repo.git')).toBeNull(); + expect(repoSlug('git@privategithub.com:owner/repo.git')).toBeNull(); }); }); diff --git a/docs/AGENTS.md b/docs/AGENTS.md index efd0e00e1f..f6a93cdfd4 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -26,11 +26,11 @@ Full playbook + per-mechanism design: `docs/design/dynamic-dispatch-coverage-pla A change to explore ranking, budgets, rendering or search replays real agent queries against the base and candidate builds instead of running an agent A/B. The past change-gating A/Bs mostly agreed with the deterministic probes; the defects only agents caught came from query shapes the probes lacked, and replaying those agents' queries reproduced each one. The corpus keeps those shapes. 1. Check out the corpus repository at the corpus's `commit` and index it with either build (`codegraph init -y`). The replay refuses a checkout at another commit. -2. `node scripts/agent-eval/replay-queries.mjs --tasks scripts/agent-eval/replay/.json --base --cand --out ` (each build is a checkout with `dist/` built). It prints a table of changed answers and exits 1 when an answer lost its source section, a file the base rendered (marked when the query names the file) or a `mustMatch` pattern. Answers are saved under `/base` and `/cand`. +2. `node scripts/agent-eval/replay-queries.mjs --tasks scripts/agent-eval/replay/.json --base --cand --out ` (each build is a checkout with `dist/` built). It prints a table of answers whose text differs and exits 1 when an answer lost its source section, a file the base rendered (marked when the query names the file) or a `mustMatch` pattern. Answers are saved under `/base` and `/cand`. 3. Read every changed and regressed answer. A lost file can be a fix (a peripheral file swapped for a better one); say so per query in the PR. Same-build replay is identical, so any change comes from the candidate. 4. Once per release batch, before a build is promoted or released, run one agent A/B (`ab-new-vs-baseline.sh`) on a flow question the corpus covers least, then harvest its queries: `node scripts/agent-eval/harvest-queries.mjs --public ...`. It keeps queries only on repositories named with `--public` and writes no paths; still read every prompt before committing, since a public query can quote a private name. `__tests__/query-replay.test.ts` fails a committed corpus that contains a local path. -Measured on `replay/codegraph.json` (46 queries, this repository at `2a4af7c3`): same build 46 of 46 identical in 9 s; a build capped at two files per answer flagged 39; the root `__tests__/` test-path fix (#500) changed 5 answers and flagged 1, where test files the query did not name were swapped for other test files. +Measured on `replay/codegraph.json` (46 queries, this repository at `2a4af7c3`; 29 replay the agent's own `maxFiles`): same build 46 of 46 identical in 9 s; a build whose default limit is two files flagged 16 of the 17 queries that use the default, 4 of them for a file the query named; the root `__tests__/` test-path fix (#500) changed 6 answers and flagged none, one of them with the same length and different text. ### Worked example — Excalidraw (TS/React, medium, 643 files) diff --git a/scripts/agent-eval/harvest-queries.mjs b/scripts/agent-eval/harvest-queries.mjs index e0fa3189ab..0bc854375b 100644 --- a/scripts/agent-eval/harvest-queries.mjs +++ b/scripts/agent-eval/harvest-queries.mjs @@ -10,8 +10,8 @@ // directory, mapped to a repository through `git remote get-url origin`. Only // queries on a repository named by --public are kept; everything else is // dropped and counted, so a private project never reaches the output. Output -// on stdout: `{ repos: { "": { tasks: [{ id, prompt }] } } }`, -// deduplicated, with no paths. Review every prompt before committing it: a +// on stdout: `{ repos: { "": { tasks: [{ id, prompt, maxFiles? }] } } }`, +// deduplicated on query and file limit, with no paths. Review every prompt before committing it: a // query on a public repository can still quote a private name. import { execFileSync } from 'node:child_process'; import { createReadStream, readdirSync, statSync } from 'node:fs'; @@ -19,14 +19,14 @@ import { createInterface } from 'node:readline'; import { join } from 'node:path'; import { createHash } from 'node:crypto'; -/** Explore calls in one transcript line: [{ query, projectPath? }]. */ +/** Explore calls in one transcript line: [{ query, projectPath?, maxFiles? }]. */ export function callsInLine(entry) { const calls = []; const blocks = entry?.message?.content; if (Array.isArray(blocks)) { for (const b of blocks) { if (b?.type === 'tool_use' && /codegraph_explore$/.test(b.name ?? '') && typeof b.input?.query === 'string') { - calls.push({ query: b.input.query, projectPath: b.input.projectPath }); + calls.push({ query: b.input.query, projectPath: b.input.projectPath, maxFiles: b.input.maxFiles }); } } } @@ -34,25 +34,72 @@ export function callsInLine(entry) { if (p?.type === 'function_call' && /codegraph_explore$/.test(p.name ?? '')) { try { const args = JSON.parse(p.arguments); - if (typeof args.query === 'string') calls.push({ query: args.query, projectPath: args.projectPath }); + if (typeof args.query === 'string') calls.push({ query: args.query, projectPath: args.projectPath, maxFiles: args.maxFiles }); } catch { /* not JSON arguments */ } } if (p?.type === 'custom_tool_call' && typeof p.input === 'string') { - for (const m of p.input.matchAll(/codegraph_explore\(\{([^}]*)\}\)/g)) { - const field = (name) => { - const f = new RegExp(`\\b${name}\\s*:\\s*("(?:[^"\\\\]|\\\\.)*")`).exec(m[1]); - try { return f ? JSON.parse(f[1]) : undefined; } catch { return undefined; } - }; - const query = field('query'); - if (typeof query === 'string') calls.push({ query, projectPath: field('projectPath') }); + for (const m of p.input.matchAll(/codegraph_explore\(\s*\{/g)) { + const fields = literalFields(p.input, m.index + m[0].length - 1); + if (typeof fields.query === 'string') calls.push({ query: fields.query, projectPath: fields.projectPath, maxFiles: fields.maxFiles }); } } - return calls; + return calls.map((c) => (typeof c.maxFiles === 'number' ? c : { query: c.query, projectPath: c.projectPath })); } -/** `owner/repo` from a GitHub remote URL, or null. */ +const ESCAPES = { n: '\n', t: '\t', r: '\r', b: '\b', f: '\f', v: '\v', 0: '\0' }; + +/** + * The top-level string and number fields of the JavaScript object literal that + * opens at `src[open]` ('{'). Code-mode calls are JavaScript, not JSON: keys may + * be bare, strings may use any quote and may contain braces. Values that are + * not plain literals (expressions, template strings with `${`) are skipped. + */ +export function literalFields(src, open) { + const fields = {}; + let depth = 0, key, expectKey = false, expectValue = false; + for (let i = open; i < src.length; i++) { + const ch = src[i]; + if (ch === '"' || ch === "'" || ch === '`') { + let s = '', literal = true, j = i + 1; + for (; j < src.length && src[j] !== ch; j++) { + if (src[j] === '\\') { j++; s += ESCAPES[src[j]] ?? src[j]; } + else if (ch === '`' && src[j] === '$' && src[j + 1] === '{') { literal = false; s += src[j]; } + else s += src[j]; + } + i = j; + if (depth === 1 && expectKey) { key = s; expectKey = false; } + else if (depth === 1 && expectValue && key !== undefined && literal) fields[key] = s; + expectValue = false; + } else if (ch === '{' || ch === '[' || ch === '(') { + depth++; + if (depth === 1) expectKey = true; + expectValue = false; + } else if (ch === '}' || ch === ']' || ch === ')') { + if (--depth === 0) break; + } else if (depth === 1 && ch === ',') { + expectKey = true; expectValue = false; key = undefined; + } else if (depth === 1 && ch === ':') { + expectValue = true; + } else if (depth === 1 && /[A-Za-z_$0-9]/.test(ch)) { + const word = /^[A-Za-z_$0-9.]+/.exec(src.slice(i))[0]; + i += word.length - 1; + if (expectKey) { key = word; expectKey = false; } + else if (expectValue && key !== undefined && /^\d+$/.test(word)) fields[key] = Number(word); + expectValue = false; + } + } + return fields; +} + +/** `owner/repo` from a remote on github.com itself (HTTPS, SSH URL or scp form), or null. */ export function repoSlug(url) { - const m = /github\.com[:/]([^/\s]+)\/([^/\s]+?)(?:\.git)?\/?$/.exec(url.trim()); + const text = url.trim(); + const scp = /^git@github\.com:([^/\s]+)\/([^/\s]+?)(?:\.git)?\/?$/.exec(text); + if (scp) return `${scp[1]}/${scp[2]}`.toLowerCase(); + let parsed; + try { parsed = new URL(text); } catch { return null; } + if (parsed.hostname.toLowerCase() !== 'github.com') return null; + const m = /^\/([^/]+)\/([^/]+?)(?:\.git)?\/?$/.exec(parsed.pathname); return m ? `${m[1]}/${m[2]}`.toLowerCase() : null; } @@ -105,11 +152,14 @@ async function main(argv) { const slug = dir ? slugFor(dir) : null; if (!slug || !pub.has(slug)) { dropped++; continue; } const prompt = call.query.trim().replace(/\s+/g, ' '); - const key = `${slug}\0${prompt.toLowerCase()}`; + const limit = call.maxFiles === undefined ? '' : `\0${call.maxFiles}`; + const key = `${slug}\0${prompt.toLowerCase()}${limit}`; if (seen.has(key)) { duplicate++; continue; } seen.add(key); - const id = createHash('sha256').update(prompt).digest('hex').slice(0, 10); - (repos[slug] ??= { tasks: [] }).tasks.push({ id: `q-${id}`, prompt }); + const id = createHash('sha256').update(prompt + limit).digest('hex').slice(0, 10); + const task = { id: `q-${id}`, prompt }; + if (call.maxFiles !== undefined) task.maxFiles = call.maxFiles; + (repos[slug] ??= { tasks: [] }).tasks.push(task); kept++; } } diff --git a/scripts/agent-eval/probe-explore.mjs b/scripts/agent-eval/probe-explore.mjs index 11e6b28faf..c757b22636 100644 --- a/scripts/agent-eval/probe-explore.mjs +++ b/scripts/agent-eval/probe-explore.mjs @@ -14,7 +14,8 @@ // files the answer renders) and mustMatchAt (first offset of each task's // mustMatch pattern, -1 when missing). A final row with id "memory" reports // resident memory idle and after the run. tasks.json is `{ "tasks": [...] }` -// or a bare array of `{ id, prompt, mustMatch? }`. +// or a bare array of `{ id, prompt, mustMatch?, maxFiles? }`; maxFiles is +// passed to codegraph_explore as the caller's file limit. // // Options: // --build codegraph checkout whose dist/ answers (default: this one) @@ -103,8 +104,8 @@ async function inProcess(build, repo) { const handler = new ToolHandler(cg); return { pid: process.pid, - async explore(query) { - const res = await handler.execute('codegraph_explore', { query }); + async explore(query, maxFiles) { + const res = await handler.execute('codegraph_explore', maxFiles === undefined ? { query } : { query, maxFiles }); if (res.isError) throw new Error(`codegraph_explore: ${JSON.stringify(res.content)}`); return (res.content ?? []).map((c) => c.text ?? '').join('\n'); }, @@ -171,8 +172,9 @@ async function overMcp(build, repo, runtime, timeoutMs) { proc.stdin.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized', params: {} }) + '\n'); return { pid: proc.pid, - async explore(query) { - const res = await rpc('tools/call', { name: 'codegraph_explore', arguments: { query, projectPath: repo } }); + async explore(query, maxFiles) { + const args = maxFiles === undefined ? { query, projectPath: repo } : { query, projectPath: repo, maxFiles }; + const res = await rpc('tools/call', { name: 'codegraph_explore', arguments: args }); return (res.result?.content ?? []).map((c) => c.text ?? '').join('\n'); }, close, @@ -206,7 +208,7 @@ async function main() { const idle = memMB(session.pid); for (const task of tasks) { const t0 = Date.now(); - const text = await session.explore(task.prompt); + const text = await session.explore(task.prompt, task.maxFiles); const row = probeRow(task.id, text, Date.now() - t0, task.mustMatch); missing += row.mustMatchAt.filter((at) => at < 0).length; if (opts.out) writeFileSync(join(opts.out, `${task.id}.md`), text); diff --git a/scripts/agent-eval/replay-queries.mjs b/scripts/agent-eval/replay-queries.mjs index 2ae3b4f632..ad7944c20c 100644 --- a/scripts/agent-eval/replay-queries.mjs +++ b/scripts/agent-eval/replay-queries.mjs @@ -17,14 +17,14 @@ import { fileURLToPath } from 'node:url'; import { parseArgs } from 'node:util'; /** How one query's answer moved from base to candidate. */ -export function compareRow(base, cand, prompt = '') { +export function compareRow(base, cand, prompt = '', sameText = base.chars === cand.chars) { const lostFiles = base.files.filter((f) => !cand.files.includes(f)); const lostNamed = lostFiles.filter((f) => prompt.includes(f)); const gainedFiles = cand.files.filter((f) => !base.files.includes(f)); const lostSource = base.sourceAt >= 0 && cand.sourceAt < 0; const lostMatch = base.mustMatchAt.filter((at, i) => at >= 0 && (cand.mustMatchAt[i] ?? -1) < 0).length; const regressed = lostSource || lostFiles.length > 0 || lostMatch > 0; - const changed = regressed || gainedFiles.length > 0 || base.chars !== cand.chars; + const changed = regressed || gainedFiles.length > 0 || !sameText; return { id: base.id, regressed, changed, lostSource, lostFiles, lostNamed, gainedFiles, lostMatch, baseChars: base.chars, candChars: cand.chars }; } @@ -65,11 +65,16 @@ function main() { } const base = probe(resolve(values.base), repo, values.tasks, values.ids, join(out, 'base')); const cand = probe(resolve(values.cand), repo, values.tasks, values.ids, join(out, 'cand')); + if (base.size === 0) { + console.error('no queries selected; check --ids and the corpus'); + return 2; + } const prompts = new Map(corpus.tasks.map((t) => [t.id, t.prompt])); + const answer = (side, id) => readFileSync(join(out, side, `${id}.md`), 'utf8'); const rows = [...base.values()].map((b) => { const c = cand.get(b.id); if (!c) throw new Error(`candidate has no answer for ${b.id}`); - return compareRow(b, c, prompts.get(b.id)); + return compareRow(b, c, prompts.get(b.id), answer('base', b.id) === answer('cand', b.id)); }); const changed = rows.filter((r) => r.changed); const regressed = rows.filter((r) => r.regressed); diff --git a/scripts/agent-eval/replay/codegraph.json b/scripts/agent-eval/replay/codegraph.json index cf2066f16e..b7f7d951d7 100644 --- a/scripts/agent-eval/replay/codegraph.json +++ b/scripts/agent-eval/replay/codegraph.json @@ -39,24 +39,28 @@ "prompt": "CHANGELOG.md Unreleased New Features Fixes" }, { - "id": "q-722771cf35", - "prompt": "writer.pid fallback mode daemon start writer lock acquire SIGTERM shutdown clients idle timeout refresh launcher fallback proxy existing MCP session" + "id": "q-ce29b1b841", + "prompt": "writer.pid fallback mode daemon start writer lock acquire SIGTERM shutdown clients idle timeout refresh launcher fallback proxy existing MCP session", + "maxFiles": 10 }, { "id": "q-83904696a8", "prompt": "handleExplore getExploreOutputBudget pinnedFiles source excerpts exact path literal selection Vue test assertions" }, { - "id": "q-3e323a81de", - "prompt": "src/extraction/index.ts extractFileWithParser retryOnCorruption: trace WASM heap recreation retry guards to parseNativeTree" + "id": "q-0a123254bf", + "prompt": "src/extraction/index.ts extractFileWithParser retryOnCorruption: trace WASM heap recreation retry guards to parseNativeTree", + "maxFiles": 2 }, { - "id": "q-e298f733b6", - "prompt": "TreeSitterExtractor extract isTreeSitterTimeoutError parseAst src/extraction/parse-worker.ts: how are native parser errors and memory messages propagated to retries?" + "id": "q-08aa47e98e", + "prompt": "TreeSitterExtractor extract isTreeSitterTimeoutError parseAst src/extraction/parse-worker.ts: how are native parser errors and memory messages propagated to retries?", + "maxFiles": 3 }, { - "id": "q-f194f1d4e3", - "prompt": "src/extraction/parse-pool.ts ParseWorkerPool recycle recycleAll handleWorkerError onMessage timeout: which live native worker errors trigger restart and retry?" + "id": "q-75f1ef5562", + "prompt": "src/extraction/parse-pool.ts ParseWorkerPool recycle recycleAll handleWorkerError onMessage timeout: which live native worker errors trigger restart and retry?", + "maxFiles": 1 }, { "id": "q-79895a605a", @@ -87,108 +91,133 @@ "prompt": "infer_java_field_receiver_type infer_declared_member_receiver_type jvm_field_receiver" }, { - "id": "q-2195039d4e", - "prompt": "SQLite database driver node:sqlite read-only worker pool query-worker database connection" + "id": "q-1d3452b493", + "prompt": "SQLite database driver node:sqlite read-only worker pool query-worker database connection", + "maxFiles": 4 }, { - "id": "q-1795f3a486", - "prompt": "src/mcp/tools.ts trimProjects DEFAULT_PROJECT_IDLE getProject defaultProject src/mcp/project-lifecycle.ts writer lock fallback idle release" + "id": "q-512a0429cd", + "prompt": "src/mcp/tools.ts trimProjects DEFAULT_PROJECT_IDLE getProject defaultProject src/mcp/project-lifecycle.ts writer lock fallback idle release", + "maxFiles": 5 }, { - "id": "q-9c8c79ed8a", - "prompt": "__tests__/mcp-project-idle.test.ts idleRelease engine default project CODEGRAPH_PROJECT_IDLE_TIMEOUT_MS" + "id": "q-41b6397dee", + "prompt": "__tests__/mcp-project-idle.test.ts idleRelease engine default project CODEGRAPH_PROJECT_IDLE_TIMEOUT_MS", + "maxFiles": 3 }, { - "id": "q-d3903019db", - "prompt": "__tests__/mcp-projectpath-lifecycle.test.ts __tests__/mcp-idle-release-order.test.ts __tests__/mcp-writer-lock.test.ts" + "id": "q-48aa64ac59", + "prompt": "__tests__/mcp-projectpath-lifecycle.test.ts __tests__/mcp-idle-release-order.test.ts __tests__/mcp-writer-lock.test.ts", + "maxFiles": 3 }, { - "id": "q-4bfc6b3a39", - "prompt": "src/mcp/proxy.ts ensureEngine retireEngine scheduleDaemonRetry CODEGRAPH_DAEMON_RETRY_MS src/mcp/engine.ts ensureInitialized stop defaultLease" + "id": "q-4184a8ada8", + "prompt": "src/mcp/proxy.ts ensureEngine retireEngine scheduleDaemonRetry CODEGRAPH_DAEMON_RETRY_MS src/mcp/engine.ts ensureInitialized stop defaultLease", + "maxFiles": 2 }, { - "id": "q-31969f6470", - "prompt": "src/mcp/proxy.ts LocalHandshakeDeps runLocalHandshakeProxy stdin onData getDaemonSocket makeEngine __tests__/proxy-shutdown.test.ts __tests__/mcp-daemon.test.ts fallback retries hand writer" + "id": "q-01c72f6fb7", + "prompt": "src/mcp/proxy.ts LocalHandshakeDeps runLocalHandshakeProxy stdin onData getDaemonSocket makeEngine __tests__/proxy-shutdown.test.ts __tests__/mcp-daemon.test.ts fallback retries hand writer", + "maxFiles": 3 }, { - "id": "q-66cb0e759e", - "prompt": "src/codegraph.ts initSync src/kernel/loader.ts require codegraph-kernel src/mcp/engine.ts catchUpSync startWatching" + "id": "q-982311adc5", + "prompt": "src/codegraph.ts initSync src/kernel/loader.ts require codegraph-kernel src/mcp/engine.ts catchUpSync startWatching", + "maxFiles": 3 }, { - "id": "q-ee1a00ea8d", - "prompt": "__tests__/proxy-shutdown.test.ts startProxy env src/mcp/liveness-watchdog.ts installMainThreadWatchdog env" + "id": "q-17fd054490", + "prompt": "__tests__/proxy-shutdown.test.ts startProxy env src/mcp/liveness-watchdog.ts installMainThreadWatchdog env", + "maxFiles": 2 }, { - "id": "q-df62321741", - "prompt": "kernel build stamp untracked source invalidation build-kernel scripts build stamp hash git ls-files" + "id": "q-6132e3251f", + "prompt": "kernel build stamp untracked source invalidation build-kernel scripts build stamp hash git ls-files", + "maxFiles": 4 }, { - "id": "q-1f48df9dba", - "prompt": "scripts/kernel-stamp.mjs scripts/ensure-kernel.mjs __tests__/ensure-kernel-stamp.test.ts" + "id": "q-91ba466d1c", + "prompt": "scripts/kernel-stamp.mjs scripts/ensure-kernel.mjs __tests__/ensure-kernel-stamp.test.ts", + "maxFiles": 3 }, { - "id": "q-f88ad05305", - "prompt": "src/mcp/tools.ts callersTool calleesTool codegraph_callers codegraph_callees" + "id": "q-65ca5f2c30", + "prompt": "src/mcp/tools.ts callersTool calleesTool codegraph_callers codegraph_callees", + "maxFiles": 1 }, { - "id": "q-875f4f64e3", - "prompt": "handleCallers handleCallees collectWiringNotes nodeHeader" + "id": "q-7759bc1b75", + "prompt": "handleCallers handleCallees collectWiringNotes nodeHeader", + "maxFiles": 2 }, { - "id": "q-3ae4233c4b", - "prompt": "formatNodeList edgeLabel GraphTraverser getCallers getCallees" + "id": "q-46f2f7edeb", + "prompt": "formatNodeList edgeLabel GraphTraverser getCallers getCallees", + "maxFiles": 2 }, { "id": "q-104036dbc2", "prompt": "scripts/build-kernel.mjs scripts/ensure-kernel.mjs runtime-control.ts checkRuntimeReady protocol version runtime identity build stamp" }, { - "id": "q-f9910072c8", - "prompt": "src/mcp/daemon-registry.ts getRegistryDir src/directory.ts getCodeGraphStateDir registry dir global config" + "id": "q-170426f071", + "prompt": "src/mcp/daemon-registry.ts getRegistryDir src/directory.ts getCodeGraphStateDir registry dir global config", + "maxFiles": 3 }, { - "id": "q-60d91393ac", - "prompt": "src/mcp/version.ts resolveManagedBuildVersion managed source stamp file" + "id": "q-7abd239dc0", + "prompt": "src/mcp/version.ts resolveManagedBuildVersion managed source stamp file", + "maxFiles": 1 }, { - "id": "q-9c3899c519", - "prompt": "src/codegraph.ts codegraph-kernel/src/lib.rs src/db/sqlite-adapter.ts src/mcp/server.ts runtime architecture Rust kernel Node transport public library API ui browser" + "id": "q-01ba61171b", + "prompt": "src/codegraph.ts codegraph-kernel/src/lib.rs src/db/sqlite-adapter.ts src/mcp/server.ts runtime architecture Rust kernel Node transport public library API ui browser", + "maxFiles": 5 }, { - "id": "q-f795406e75", - "prompt": "codegraph-kernel/src/lib.rs codegraph-kernel/src/resolve/name_matcher_upstream.rs native Rust #[cfg(test)] tests: identify a pure helper with direct Rust unit tests usable by cargo test; contrast __tests__/kernel-scaffold.test.ts and scripts/ensure-kernel.mjs NAPI prebuild test wiring" + "id": "q-5d83a6b73c", + "prompt": "codegraph-kernel/src/lib.rs codegraph-kernel/src/resolve/name_matcher_upstream.rs native Rust #[cfg(test)] tests: identify a pure helper with direct Rust unit tests usable by cargo test; contrast __tests__/kernel-scaffold.test.ts and scripts/ensure-kernel.mjs NAPI prebuild test wiring", + "maxFiles": 4 }, { - "id": "q-da10d1df74", - "prompt": "codegraph-kernel/src/ids.rs codegraph-kernel/src/docstring.rs codegraph-kernel/src/lib.rs cfg(test) NAPI test stubs" + "id": "q-0338db736f", + "prompt": "codegraph-kernel/src/ids.rs codegraph-kernel/src/docstring.rs codegraph-kernel/src/lib.rs cfg(test) NAPI test stubs", + "maxFiles": 3 }, { - "id": "q-a49ca91c4e", - "prompt": "codegraph-kernel/src/util.rs col16 utf16 tests" + "id": "q-d19a67e53a", + "prompt": "codegraph-kernel/src/util.rs col16 utf16 tests", + "maxFiles": 3 }, { - "id": "q-ebde6c9ef9", - "prompt": "Cols col utf16_prefix codegraph-kernel/src/textutil.rs callers parse_tree_inner" + "id": "q-5ed5e923e9", + "prompt": "Cols col utf16_prefix codegraph-kernel/src/textutil.rs callers parse_tree_inner", + "maxFiles": 2 }, { - "id": "q-a3cbcd491c", - "prompt": "__tests__/kernel-parse-tree.test.ts UTF-16 unicode astral" + "id": "q-9a8cf9697a", + "prompt": "__tests__/kernel-parse-tree.test.ts UTF-16 unicode astral", + "maxFiles": 1 }, { - "id": "q-57dc520bed", - "prompt": "src/extraction/kernel/tree.ts NativeNode startIndex endIndex startPosition descendantForIndex descendantForPosition" + "id": "q-c3e864cffb", + "prompt": "src/extraction/kernel/tree.ts NativeNode startIndex endIndex startPosition descendantForIndex descendantForPosition", + "maxFiles": 1 }, { - "id": "q-872326d0b3", - "prompt": "codegraph-kernel/src/tree.rs parse_tree_inner Row start_byte start_idx utf16_prefix src/extraction/parse-tree.ts TreeNode" + "id": "q-c91e32a1b3", + "prompt": "codegraph-kernel/src/tree.rs parse_tree_inner Row start_byte start_idx utf16_prefix src/extraction/parse-tree.ts TreeNode", + "maxFiles": 2 }, { - "id": "q-621688899a", - "prompt": "src/graph/branch-guards.ts innermostAt __tests__/kernel-parse-tree.test.ts firstViolation" + "id": "q-daf8ef7fb3", + "prompt": "src/graph/branch-guards.ts innermostAt __tests__/kernel-parse-tree.test.ts firstViolation", + "maxFiles": 2 }, { - "id": "q-c0cacd2a20", - "prompt": "createDatabase SqliteDatabase src/db/schema.sql FTS5" + "id": "q-368e0f6b10", + "prompt": "createDatabase SqliteDatabase src/db/schema.sql FTS5", + "maxFiles": 2 } ] } From 526acffb9e19a706c4ab8abd50459a82e5059643 Mon Sep 17 00:00:00 2001 From: bompus Date: Sun, 18 Oct 2026 23:13:13 -0600 Subject: [PATCH 3/3] docs: replay time is about 10 s --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 8a783c0d7b..e48418f993 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,7 +94,7 @@ Defined in `src/types.ts`. Both extractors and resolvers must use these exact st **Required** for every new language/framework: validate on small/medium/large real repos with >=3 flow prompts; deterministic `scripts/agent-eval/probe-explore.mjs` probes, then agent A/B (`scripts/agent-eval/run-all.sh` / `ab-new-vs-baseline.sh`). Pass bar: ~0 Read/Grep within the explore-call budget, faster than without-codegraph, no control-repo regression. -**Every other retrieval change** (explore ranking, budgets, rendering, search) is gated by replaying saved agent queries, not by an agent A/B: `node scripts/agent-eval/replay-queries.mjs --tasks scripts/agent-eval/replay/.json --base --cand --out ` (about 20 s). Read every changed or regressed answer and say in the PR why each is acceptable. Run one agent A/B per release batch, before a build is promoted or released, and add its new query shapes to the corpus with `harvest-queries.mjs`. Details: `docs/AGENTS.md` § Query replay. +**Every other retrieval change** (explore ranking, budgets, rendering, search) is gated by replaying saved agent queries, not by an agent A/B: `node scripts/agent-eval/replay-queries.mjs --tasks scripts/agent-eval/replay/.json --base --cand --out ` (about 10 s on 46 queries). Read every changed or regressed answer and say in the PR why each is acceptable. Run one agent A/B per release batch, before a build is promoted or released, and add its new query shapes to the corpus with `harvest-queries.mjs`. Details: `docs/AGENTS.md` § Query replay. Full methodology (feedback metrics, CLI contamination guard, Sonnet/`--effort high` model policy, daemon pre-warm), the Excalidraw worked example, and coverage matrix live in: - `docs/AGENTS.md` (nested; also loaded when cwd is under `docs/`)