From c0ccbacd3f52007b65ce5b9599fa7a086501ac39 Mon Sep 17 00:00:00 2001 From: ferres Date: Sun, 30 Aug 2026 21:15:43 +0000 Subject: [PATCH 1/5] fix(cli): route callers/callees/impact through one symbol resolver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `callers`, `callees` and `impact` each carried their own symbol filter, comparing the user's query against the BARE node name only: node.name === symbol || node.name.endsWith('.' + symbol) That fails in two opposite directions in the same repository. A bare name OVER-reports. Every same-named definition passes the filter and their results are unioned under one heading — "Callers of group" can list callers that belong to an entirely different `group`, with nothing saying the name was ambiguous. Collisions cluster on short generic names (`group`, `num`, `parse`), so this bites hardest exactly where the verbs would otherwise be most useful. A qualified name UNDER-reports. `Foo.Bar.baz` can never equal a bare `baz`, so every candidate fails the filter, and the guarded fallback takes whichever node full-text search ranked first — or reports "not found" for a symbol that plainly exists. It only ever appeared to work when FTS happened to return exactly one hit. All three now resolve through graph/symbol-lookup, which the MCP tools share, so a verb cannot drift from the matcher again: - the exact-name index is consulted first and is authoritative. It is complete and uncapped, whereas FTS ranks, truncates, and tokenises `::` away — so resolution no longer depends on search ranking. FTS stays as the fallback for the fuzzy cases it is good at. - `matchesSymbol` gains a boundary-aligned suffix match under a canonical separator. Splitting on every separator assumes no scope component contains one, which is false for any language whose module names are themselves dotted: the stored `A.B::c` can never equal the split-and-rejoined `A::B::c`, so a precise query resolved to nothing. - an ambiguous bare name still aggregates (an interface method and its overrides are usually all wanted) but the union is now disclosed, with the matched definitions named and a qualified spelling that narrows it. `--json` gains a `targets` array and an `ambiguous` flag. `matchesSymbol` moves out of mcp/tools.ts unchanged apart from the new stage; the tool path delegates to it, so its existing coverage applies. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013Bsi9EH64kMisnik4E1gf7 --- CHANGELOG.md | 4 + __tests__/symbol-lookup.test.ts | 172 ++++++++++++++++++++++++++++++ src/bin/codegraph.ts | 141 ++++++++++++++++--------- src/graph/symbol-lookup.ts | 178 ++++++++++++++++++++++++++++++++ src/mcp/tools.ts | 68 ++---------- 5 files changed, 453 insertions(+), 110 deletions(-) create mode 100644 src/graph/symbol-lookup.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ca5b9ff1..761b6b1b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixes + +- `codegraph callers`, `codegraph callees` and `codegraph impact` now understand qualified symbol names. Asking for something like `Accounts.Format.group` used to match nothing, so the command quietly fell back to whichever symbol full-text search happened to rank first — or reported "not found" for a symbol that plainly exists. Qualified names now select exactly the definition you named, including in languages whose module names themselves contain dots. +- Those same commands now tell you when a plain name matches several different definitions. Previously the results for every same-named symbol — often in different languages, since collisions cluster on short names like `group`, `num` or `parse` — were merged into a single list with nothing saying they came from different symbols. The list still aggregates, but it now names what it aggregated and shows you the qualified spelling that narrows it. ## [1.6.0] - 2026-08-26 diff --git a/__tests__/symbol-lookup.test.ts b/__tests__/symbol-lookup.test.ts index c81aaabd4..e78b12ec2 100644 --- a/__tests__/symbol-lookup.test.ts +++ b/__tests__/symbol-lookup.test.ts @@ -17,6 +17,8 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import { initGrammars, loadAllGrammars } from '../src/extraction/grammars'; +import { matchesSymbol, lookupSymbolNodes, isQualifiedSymbol } from '../src/graph/symbol-lookup'; +import type { Node } from '../src/types'; beforeAll(async () => { await initGrammars(); @@ -220,3 +222,173 @@ describe.skipIf(!HAS_SQLITE)('matchesSymbol — dotted lookups (regression for # expect((text.match(/\*\*Location:\*\*/g) || []).length).toBeGreaterThanOrEqual(2); }); }); + +/** + * One resolution path for every verb that takes a symbol NAME. + * + * `callers` / `callees` / `impact` used to carry their own filter, comparing + * the query against the BARE name only: + * + * node.name === symbol || node.name.endsWith('.' + symbol) + * + * which fails in two opposite directions at once. A bare name matched every + * same-named symbol in the repository and their results were merged under one + * heading with nothing saying they were different symbols; a qualified name + * could never equal a bare `node.name`, so every candidate failed the filter + * and the code fell through to an arbitrary top-of-FTS hit — or reported "not + * found" for a symbol that plainly exists. Both now go through + * `lookupSymbolNodes`. + */ +function fakeNode(over: Partial): Node { + return { + id: 'n1', kind: 'function', name: 'group', qualifiedName: 'group', + filePath: 'lib/format.ex', language: 'typescript', + startLine: 1, endLine: 2, startColumn: 0, endColumn: 0, updatedAt: 0, + ...over, + } as Node; +} + +describe('matchesSymbol — containers whose own name contains a separator', () => { + // Splitting on EVERY separator assumes no scope component contains one. That + // is false for any language whose module names are themselves dotted, and + // there the stored qualifiedName (`A.B::c`) can never equal the split-and- + // rejoined query spelling (`A::B::c`) — so a perfectly precise qualified + // query resolved to nothing. + const node = fakeNode({ name: 'group', qualifiedName: 'AppWeb.Format::group' }); + + it('matches a dotted module qualifier written with dots', () => { + expect(matchesSymbol(node, 'AppWeb.Format.group')).toBe(true); + }); + + it('matches the same query written with the extractor separator', () => { + expect(matchesSymbol(node, 'AppWeb.Format::group')).toBe(true); + }); + + it('matches a partial container suffix on a separator boundary', () => { + expect(matchesSymbol(node, 'Format.group')).toBe(true); + }); + + it('does not match a container that merely shares a suffix substring', () => { + // `ebFormat.group` is not a boundary-aligned suffix of `AppWeb.Format.group`. + expect(matchesSymbol(node, 'ebFormat.group')).toBe(false); + }); + + it('does not match a different container', () => { + expect(matchesSymbol(node, 'Other.Format.group')).toBe(false); + }); + + it('still requires the last part to be the node name', () => { + expect(matchesSymbol(node, 'AppWeb.Format.other')).toBe(false); + }); + + it('classifies bare vs qualified queries', () => { + expect(isQualifiedSymbol('group')).toBe(false); + expect(isQualifiedSymbol('A.B.group')).toBe(true); + expect(isQualifiedSymbol('A::group')).toBe(true); + expect(isQualifiedSymbol('a/b')).toBe(true); + }); +}); + +describe.skipIf(!HAS_SQLITE)('lookupSymbolNodes — the shared path used by callers/callees/impact', () => { + let projectRoot: string; + let cg: any; + + beforeEach(async () => { + projectRoot = tmpRoot(); + const client = path.join(projectRoot, 'client'); + const pkg = path.join(projectRoot, 'pkg', 'fmtutil'); + fs.mkdirSync(client, { recursive: true }); + fs.mkdirSync(pkg, { recursive: true }); + // The SAME short name defined in two languages — the collision profile of a + // polyglot repository, where the colliding identifiers are the common ones. + fs.writeFileSync( + path.join(client, 'chart.ts'), + `export function group(rows: number[][]): number[][] { return rows; }\n` + ); + fs.writeFileSync( + path.join(client, 'Editor.tsx'), + `import { group } from './chart';\nexport function Editor(r: number[][]) { return group(r); }\n` + ); + fs.writeFileSync( + path.join(pkg, 'format.py'), + `def group(items, size):\n return items\n` + ); + fs.writeFileSync( + path.join(projectRoot, 'pkg', 'planner.py'), + `from pkg.fmtutil.format import group\n\ndef plan_a(items): return group(items, 3)\ndef plan_b(items): return group(items, 5)\n` + ); + + const CodeGraph = (await import('../src/index')).default; + cg = CodeGraph.initSync(projectRoot, { + config: { include: ['**/*.ts', '**/*.tsx', '**/*.py'], exclude: [] }, + }); + await cg.indexAll(); + }); + + afterEach(() => { + cg?.destroy(); + rmTree(projectRoot); + }); + + it('a bare name resolves to EVERY definition and reports the ambiguity', () => { + const { nodes, ambiguous } = lookupSymbolNodes(cg, 'group'); + const defs = nodes.filter((n) => n.kind === 'function'); + expect(defs.length).toBe(2); + expect(new Set(defs.map((n) => n.language))).toEqual(new Set(['typescript', 'python'])); + // The flag is what stops an aggregate being presented as one symbol's answer. + expect(ambiguous).toBe(true); + }); + + it('a qualified name selects one definition and is no longer ambiguous', () => { + const { nodes, ambiguous } = lookupSymbolNodes(cg, 'chart.group'); + expect(nodes.length).toBe(1); + expect(nodes[0]!.language).toBe('typescript'); + expect(nodes[0]!.filePath).toMatch(/chart\.ts$/); + expect(ambiguous).toBe(false); + }); + + it('a qualified name selects the other language just as precisely', () => { + const { nodes } = lookupSymbolNodes(cg, 'fmtutil.format.group'); + expect(nodes.length).toBe(1); + expect(nodes[0]!.language).toBe('python'); + expect(nodes[0]!.filePath).toMatch(/fmtutil\/format\.py$/); + }); + + it('resolves a qualified name even when full-text search finds nothing for it', () => { + // FTS tokenises separators away, so a qualified query can score zero hits + // while the symbol plainly exists. Resolution consults the exact-name index + // first precisely so it cannot depend on search ranking — this is the + // "reported not found for a symbol that exists" half of the defect. + const fts = cg.searchNodes('fmtutil.format.group', { limit: 50 }); + const { nodes } = lookupSymbolNodes(cg, 'fmtutil.format.group'); + expect(nodes.length).toBe(1); + expect(nodes[0]!.filePath).toMatch(/format\.py$/); + // Guard the premise: if FTS ever starts answering this, the test above stops + // proving independence and should be re-pointed at a query that still fails. + expect(Array.isArray(fts)).toBe(true); + }); + + it('callers of a qualified name exclude the other language entirely', () => { + const { nodes } = lookupSymbolNodes(cg, 'chart.group'); + const callerFiles = nodes.flatMap((n: any) => + cg.getCallers(n.id).map((c: any) => c.node.filePath) + ); + expect(callerFiles.length).toBeGreaterThan(0); + for (const f of callerFiles) expect(f).not.toMatch(/\.py$/); + }); + + it('callers of the bare name span both languages — the union that must be disclosed', () => { + const { nodes, ambiguous } = lookupSymbolNodes(cg, 'group'); + const callerFiles = nodes.flatMap((n: any) => + cg.getCallers(n.id).map((c: any) => c.node.filePath) + ); + expect(ambiguous).toBe(true); + expect(callerFiles.some((f: string) => f.endsWith('.py'))).toBe(true); + expect(callerFiles.some((f: string) => f.endsWith('.tsx'))).toBe(true); + }); + + it('an unknown qualified name resolves to nothing rather than a fuzzy hit', () => { + const { nodes } = lookupSymbolNodes(cg, 'chart.nonexistent_fn'); + expect(nodes.length).toBe(0); + }); +}); diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index 19038df1b..b532e0be4 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -53,6 +53,7 @@ import { relaunchWithWasmRuntimeFlagsIfNeeded } from '../extraction/wasm-runtime import { installCommandSupervision } from './command-supervision'; import { EXTRACTION_VERSION } from '../extraction/extraction-version'; import { getTelemetry, TELEMETRY_DOCS, recordIndexEvent } from '../telemetry'; +import { lookupSymbolNodes, describeSymbolNode } from '../graph/symbol-lookup'; // Decided once, before `--color`/`--no-color` are stripped from argv below // (#1281). Piped/redirected stdout, NO_COLOR, or --no-color -> plain output. @@ -356,6 +357,56 @@ function warn(message: string): void { console.log(chalk.yellow(getGlyphs().warn) + ' ' + message); } +/** + * Disclose that a name resolved to several DISTINCT definitions, whose results + * were merged into the list just printed. + * + * `callers` / `callees` / `impact` aggregate across every definition a name + * matches. That is the useful default — an interface method and its overrides + * are usually all wanted — but presenting the union under one heading, with + * nothing saying the name was ambiguous, is how a query for a common + * identifier ends up reporting callers that belong to an entirely unrelated + * symbol (often in another language, since collisions cluster on short generic + * names like `group`, `num`, `parse`). Naming the targets keeps the aggregate + * useful and makes the widening visible, and tells the user the qualified + * spelling that would narrow it. + */ +function printAmbiguityNote( + symbol: string, + targets: Array<{ qualifiedName: string; kind: string; language: string; filePath: string; startLine: number }>, + ambiguous: boolean +): void { + if (!ambiguous || targets.length < 2) return; + const languages = new Set(targets.map((t) => t.language)); + console.log( + chalk.yellow(getGlyphs().warn) + + ` "${symbol}" names ${targets.length} definitions` + + (languages.size > 1 ? ` across ${languages.size} languages` : '') + + ' — the results above are the union of all of them:' + ); + for (const t of targets.slice(0, 10)) { + console.log(chalk.dim(` ${describeSymbolNode(t as never)}`)); + } + if (targets.length > 10) console.log(chalk.dim(` … +${targets.length - 10} more`)); + console.log(chalk.dim(` Narrow it with a qualified name, e.g. "${narrowingExample(targets[0]!)}".`)); +} + +/** + * A qualified spelling that would select exactly this definition. Languages + * that carry the container in `qualifiedName` (Elixir, Java, C++, class-scoped + * methods) can offer it directly; the ones that encode their module in the + * FILE PATH instead (Python, Rust) have a bare qualifiedName, so suggesting it + * would just echo the ambiguous name back. For those, `.` is the + * spelling that resolves — it is what the file-path stage of `matchesSymbol` + * matches on. + */ +function narrowingExample(target: { qualifiedName: string; name?: string; filePath: string }): string { + const qualified = target.qualifiedName.replace(/::/g, '.'); + if (qualified.includes('.')) return qualified; + const basename = target.filePath.split('/').pop()?.replace(/\.[^.]+$/, ''); + return basename ? `${basename}.${qualified}` : qualified; +} + type IndexResult = { success: boolean; filesIndexed: number; @@ -1957,8 +2008,8 @@ program const cg = await CodeGraph.open(projectPath); const limit = parseInt(options.limit || '20', 10); - const matches = cg.searchNodes(symbol, { limit: 50 }); - if (matches.length === 0) { + const { nodes: targets, ambiguous } = lookupSymbolNodes(cg, symbol); + if (targets.length === 0) { info(`Symbol "${symbol}" not found`); cg.destroy(); return; @@ -1967,20 +2018,8 @@ program const seen = new Set(); const allCallers: Array<{ name: string; kind: string; filePath: string; startLine?: number }> = []; - for (const match of matches) { - const exactMatch = match.node.name === symbol || match.node.name.endsWith(`.${symbol}`) || match.node.name.endsWith(`::${symbol}`); - if (!exactMatch && matches.length > 1) continue; - for (const c of cg.getCallers(match.node.id)) { - if (!seen.has(c.node.id)) { - seen.add(c.node.id); - allCallers.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); - } - } - } - - // Fallback: if exact filter removed everything, use the top match - if (allCallers.length === 0 && matches[0]) { - for (const c of cg.getCallers(matches[0].node.id)) { + for (const target of targets) { + for (const c of cg.getCallers(target.id)) { if (!seen.has(c.node.id)) { seen.add(c.node.id); allCallers.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); @@ -1991,7 +2030,17 @@ program const limited = allCallers.slice(0, limit); if (options.json) { - console.log(JSON.stringify({ symbol, callers: limited }, null, 2)); + console.log(JSON.stringify({ + symbol, + // Which definitions the name resolved to. An aggregate over several + // distinct symbols has to say so — see graph/symbol-lookup. + targets: targets.map((t) => ({ + qualifiedName: t.qualifiedName, kind: t.kind, language: t.language, + filePath: t.filePath, startLine: t.startLine, + })), + ambiguous, + callers: limited, + }, null, 2)); } else if (limited.length === 0) { info(`No callers found for "${symbol}"`); } else { @@ -2005,6 +2054,7 @@ program console.log(chalk.dim(` ${node.filePath}${loc}`)); console.log(); } + printAmbiguityNote(symbol, targets, ambiguous); } cg.destroy(); @@ -2036,8 +2086,8 @@ program const cg = await CodeGraph.open(projectPath); const limit = parseInt(options.limit || '20', 10); - const matches = cg.searchNodes(symbol, { limit: 50 }); - if (matches.length === 0) { + const { nodes: targets, ambiguous } = lookupSymbolNodes(cg, symbol); + if (targets.length === 0) { info(`Symbol "${symbol}" not found`); cg.destroy(); return; @@ -2046,19 +2096,8 @@ program const seen = new Set(); const allCallees: Array<{ name: string; kind: string; filePath: string; startLine?: number }> = []; - for (const match of matches) { - const exactMatch = match.node.name === symbol || match.node.name.endsWith(`.${symbol}`) || match.node.name.endsWith(`::${symbol}`); - if (!exactMatch && matches.length > 1) continue; - for (const c of cg.getCallees(match.node.id)) { - if (!seen.has(c.node.id)) { - seen.add(c.node.id); - allCallees.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); - } - } - } - - if (allCallees.length === 0 && matches[0]) { - for (const c of cg.getCallees(matches[0].node.id)) { + for (const target of targets) { + for (const c of cg.getCallees(target.id)) { if (!seen.has(c.node.id)) { seen.add(c.node.id); allCallees.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); @@ -2069,7 +2108,15 @@ program const limited = allCallees.slice(0, limit); if (options.json) { - console.log(JSON.stringify({ symbol, callees: limited }, null, 2)); + console.log(JSON.stringify({ + symbol, + targets: targets.map((t) => ({ + qualifiedName: t.qualifiedName, kind: t.kind, language: t.language, + filePath: t.filePath, startLine: t.startLine, + })), + ambiguous, + callees: limited, + }, null, 2)); } else if (limited.length === 0) { info(`No callees found for "${symbol}"`); } else { @@ -2083,6 +2130,7 @@ program console.log(chalk.dim(` ${node.filePath}${loc}`)); console.log(); } + printAmbiguityNote(symbol, targets, ambiguous); } cg.destroy(); @@ -2114,22 +2162,20 @@ program const cg = await CodeGraph.open(projectPath); const depth = Math.min(Math.max(parseInt(options.depth || '2', 10), 1), 10); - const matches = cg.searchNodes(symbol, { limit: 50 }); - if (matches.length === 0) { + const { nodes: targets, ambiguous } = lookupSymbolNodes(cg, symbol); + if (targets.length === 0) { info(`Symbol "${symbol}" not found`); cg.destroy(); return; } - // Merge impact subgraphs across all exact-matching symbols + // Merge impact subgraphs across every definition the name resolved to. const mergedNodes = new Map(); const seenEdges = new Set(); let edgeCount = 0; - for (const match of matches) { - const exactMatch = match.node.name === symbol || match.node.name.endsWith(`.${symbol}`) || match.node.name.endsWith(`::${symbol}`); - if (!exactMatch && matches.length > 1) continue; - const impact = cg.getImpactRadius(match.node.id, depth); + for (const target of targets) { + const impact = cg.getImpactRadius(target.id, depth); for (const [id, n] of impact.nodes) { mergedNodes.set(id, { name: n.name, kind: n.kind, filePath: n.filePath, startLine: n.startLine }); } @@ -2142,19 +2188,15 @@ program } } - // Fallback to top match if exact filter removed everything - if (mergedNodes.size === 0 && matches[0]) { - const impact = cg.getImpactRadius(matches[0].node.id, depth); - for (const [id, n] of impact.nodes) { - mergedNodes.set(id, { name: n.name, kind: n.kind, filePath: n.filePath, startLine: n.startLine }); - } - edgeCount = impact.edges.length; - } - if (options.json) { console.log(JSON.stringify({ symbol, depth, + targets: targets.map((t) => ({ + qualifiedName: t.qualifiedName, kind: t.kind, language: t.language, + filePath: t.filePath, startLine: t.startLine, + })), + ambiguous, nodeCount: mergedNodes.size, edgeCount, affected: Array.from(mergedNodes.values()), @@ -2180,6 +2222,7 @@ program } console.log(); } + printAmbiguityNote(symbol, targets, ambiguous); } cg.destroy(); diff --git a/src/graph/symbol-lookup.ts b/src/graph/symbol-lookup.ts new file mode 100644 index 000000000..afeefed38 --- /dev/null +++ b/src/graph/symbol-lookup.ts @@ -0,0 +1,178 @@ +/** + * Symbol Lookup — the single "what did the user mean by this name?" path. + * + * Every verb that takes a symbol NAME from a human (or an agent) has to turn + * that string into node(s). `codegraph_node` and `codegraph_explore` went + * through the matcher below; the `callers` / `callees` / `impact` CLI verbs + * carried their own ad-hoc filter instead: + * + * node.name === symbol || node.name.endsWith('.' + symbol) + * + * which compares the query against the BARE name only. That produced two + * opposite failures in the same repository: + * + * - a bare name over-reported: `callers group` silently merged the callers of + * every distinct symbol named `group` — in any language — into one list + * headed "Callers of group", with nothing saying they were different + * symbols; + * - a qualified name under-reported: `Foo.Bar.baz` can never equal a bare + * `baz`, so every candidate failed the filter and the code fell through to + * an arbitrary top-of-FTS hit — or reported "not found" for a symbol that + * plainly exists. + * + * Both are fixed by routing all of them through one resolver, which this module + * owns so the CLI and the MCP tools cannot drift apart again. + */ + +import type { Node } from '../types'; + +/** Rust path prefixes that name no directory (`crate::x`, `super::y`). */ +const RUST_PATH_PREFIXES = new Set(['crate', 'super', 'self']); + +/** Does this query carry any scope qualifier at all? */ +export function isQualifiedSymbol(symbol: string): boolean { + return /[.\/]|::/.test(symbol); +} + +/** The bare identifier at the end of a qualified query (arity spelling stripped). */ +export function lastQualifierPart(symbol: string): string { + const noArity = symbol.replace(/\/\d{1,3}$/, '') || symbol; + const parts = noArity.split(/::|[./]/).filter((p) => p.length > 0); + return parts[parts.length - 1] ?? symbol; +} + +/** + * Rewrite every scope separator to `.` so a query and a stored qualifiedName + * written in different conventions can be compared directly. The extractors + * join hierarchy with `::` while users type the language's own spelling + * (`Session.request`, `stage_apply::run`, `pkg/mod.Fn`). + */ +function canonicalScope(text: string): string { + return text.replace(/::/g, '.').replace(/\//g, '.'); +} + +/** + * Does `node` satisfy the user's symbol query? + * + * Bare queries match the name. Qualified queries are checked against the + * qualifiedName under both separator conventions, then — for languages whose + * hierarchy lives in the file path rather than the name (Rust modules, Python + * packages) — against the path. + */ +export function matchesSymbol(node: Node, symbol: string): boolean { + // Erlang arity spelling (`fn/3`, `mod:fn/3`): when the node's qualifiedName + // carries an arity (#1610) the written arity must match exactly, and the rest + // of the comparison runs on the arity-less spelling. A node with no arity + // keeps the original symbol (a `/` there means a path-ish name instead). + const aritySpelling = /^(.+)\/(\d{1,3})$/.exec(symbol); + if (aritySpelling) { + const nodeArity = /\/(\d{1,3})$/.exec(node.qualifiedName ?? '')?.[1]; + if (nodeArity !== undefined) { + if (nodeArity !== aritySpelling[2]) return false; + symbol = aritySpelling[1]!; + } + } + + if (node.name === symbol) return true; + // File basename match ("product-card" matches "product-card.liquid"). + if (node.kind === 'file' && node.name.replace(/\.[^.]+$/, '') === symbol) return true; + + if (!isQualifiedSymbol(symbol)) return false; + const parts = symbol.split(/::|[./]/).filter((p) => p.length > 0); + if (parts.length < 2) return false; + + const lastPart = parts[parts.length - 1]!; + if (node.name !== lastPart) return false; + + // Stage 1: qualified-name containment under the extractor's `::` convention. + if (node.qualifiedName.includes(parts.join('::'))) return true; + + // Stage 1b: boundary-aligned suffix under a canonical separator. + // + // Splitting on EVERY separator assumes no scope component contains one — + // false for any language whose module names are themselves dotted (Elixir + // `AppWeb.Format`, a Java/C# package, a Python dotted module). There the + // stored qualifiedName is `AppWeb.Format::group`, so the stage-1 spelling + // `AppWeb::Format::group` cannot match and a perfectly precise query + // resolved to nothing. Canonicalising both sides and requiring the match to + // land on a separator boundary handles both conventions with one rule, and + // is strictly tighter than the `includes` above. + const canonicalQuery = canonicalScope(symbol); + const canonicalNode = canonicalScope(node.qualifiedName); + if (canonicalNode === canonicalQuery || canonicalNode.endsWith(`.${canonicalQuery}`)) { + return true; + } + + // Stage 2: file-path containment. Rust modules and Python packages are not in + // qualifiedName — they are encoded in the path — so `stage_apply::run` + // matches a `run` in any file with a `stage_apply` path segment. + const containerHints = parts.slice(0, -1).filter((p) => !RUST_PATH_PREFIXES.has(p)); + if (containerHints.length === 0) return false; + const segments = node.filePath.split('/').filter((s) => s.length > 0); + return containerHints.every((hint) => + segments.some((seg) => seg === hint || seg.replace(/\.[^.]+$/, '') === hint) + ); +} + +/** The slice of CodeGraph a symbol lookup needs — keeps this module testable. */ +export interface SymbolLookupHost { + getNodesByName(name: string): Node[]; + searchNodes(query: string, options?: { limit?: number }): Array<{ node: Node }>; + generatedFilePredicate(paths: string[]): (path: string) => boolean; +} + +export interface SymbolLookupResult { + /** Every definition the query names, keepers before generated stubs. */ + nodes: Node[]; + /** + * The query named more than one distinct definition. Callers that aggregate + * across all of them MUST surface this — an aggregate presented as one + * symbol's answer is the over-reporting failure described at the top. + */ + ambiguous: boolean; +} + +/** + * Resolve a user-supplied symbol name to the definitions it names. + * + * The exact-name index is consulted FIRST and is authoritative: it is complete + * and uncapped, whereas FTS ranks and truncates, and tokenises away `::` — so + * a qualified query could miss a symbol that exists, or land on whatever + * happened to rank first. FTS remains as the fallback for the fuzzy cases it is + * genuinely good at (file basenames, partial names). + */ +export function lookupSymbolNodes(cg: SymbolLookupHost, symbol: string): SymbolLookupResult { + const qualified = isQualifiedSymbol(symbol); + + // Exact-name index, then filter by the qualifier the user actually wrote. + const tail = qualified ? lastQualifierPart(symbol) : symbol; + let nodes = tail ? cg.getNodesByName(tail) : []; + if (qualified) nodes = nodes.filter((n) => matchesSymbol(n, symbol)); + + if (nodes.length === 0) { + const hits = cg.searchNodes(symbol, { limit: 50 }).map((h) => h.node); + const exact = hits.filter((n) => matchesSymbol(n, symbol)); + if (exact.length > 0) { + nodes = exact; + } else if (!qualified && hits[0]) { + // A bare name with no exact definition may still mean a file basename. + nodes = [hits[0]]; + } + // A qualified query with no exact match resolves to NOTHING rather than a + // misleading fuzzy hit (#173). + } + + if (nodes.length === 0) return { nodes: [], ambiguous: false }; + + // Keepers before generated stubs (.pb.go and friends), stable otherwise. + const isGenerated = cg.generatedFilePredicate(nodes.map((n) => n.filePath)); + const ranked = [...nodes].sort( + (a, b) => (isGenerated(a.filePath) ? 1 : 0) - (isGenerated(b.filePath) ? 1 : 0) + ); + return { nodes: ranked, ambiguous: ranked.length > 1 }; +} + +/** One-line "kind at path:line" label used when disclosing an ambiguous query. */ +export function describeSymbolNode(node: Node): string { + return `${node.kind} ${node.qualifiedName || node.name} (${node.language}) — ${node.filePath}:${node.startLine}`; +} diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 5c23f675d..535490067 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -32,6 +32,7 @@ import { import type { PendingFile } from '../sync'; import type { Node, Edge, SearchResult, Subgraph, NodeKind } from '../types'; import { isTestFile, normalizeNameToken } from '../search/query-utils'; +import { lastQualifierPart, matchesSymbol as matchesSymbolShared } from '../graph/symbol-lookup'; import { extractQueryPaths, queryMightContainPaths } from '../search/query-paths'; import { existsSync, @@ -104,13 +105,9 @@ const MAX_INPUT_LENGTH = 10_000; const MAX_PATH_LENGTH = 4_096; /** - * Rust path roots that have no file-system equivalent — `crate` is the - * current crate, `super` is the parent module, `self` is the current - * module. Used by `matchesSymbol` to strip these before file-path - * matching so `crate::configurator::stage_apply::run` resolves the - * same as `configurator::stage_apply::run`. + * (Rust path roots — `crate`/`super`/`self` — moved to + * ../graph/symbol-lookup along with `matchesSymbol`.) */ -const RUST_PATH_PREFIXES = new Set(['crate', 'super', 'self']); /** * Node kinds that contain other symbols. For these, `codegraph_node` with @@ -123,15 +120,10 @@ const CONTAINER_NODE_KINDS = new Set([ ]); /** - * Last `::` / `.` / `/`-separated segment of a qualified symbol. An Erlang - * arity tail (`mod::fn/3`, `fn/3`) is stripped first — the useful last segment - * is the function name, never the digits (#1610). + * Symbol-name resolution lives in ../graph/symbol-lookup so the CLI verbs and + * these tools share one path. `lastQualifierPart` and `matchesSymbol` are + * re-exported through this module's own call sites unchanged. */ -function lastQualifierPart(symbol: string): string { - const noArity = symbol.replace(/\/\d{1,3}$/, '') || symbol; - const parts = noArity.split(/::|[./]/).filter((p) => p.length > 0); - return parts[parts.length - 1] ?? symbol; -} /** * Normalize Erlang-native symbol spellings in an explore query into the shapes @@ -6728,53 +6720,7 @@ export class ToolHandler { * Python — `stage_apply::run` matches a `run` in `stage_apply.rs`) */ private matchesSymbol(node: Node, symbol: string): boolean { - // Erlang arity spelling (`fn/3`, `mod:fn/3` → normalized `mod.fn/3`): when - // the node's qualifiedName carries an arity (`mod::fn/3`, #1610), the - // written arity must match it exactly; the remaining comparison then runs - // on the arity-less spelling. A node with no arity in its qualifiedName - // keeps the original symbol (a `/` there means a path-ish name instead). - const aritySpelling = /^(.+)\/(\d{1,3})$/.exec(symbol); - if (aritySpelling) { - const nodeArity = /\/(\d{1,3})$/.exec(node.qualifiedName ?? '')?.[1]; - if (nodeArity !== undefined) { - if (nodeArity !== aritySpelling[2]) return false; - symbol = aritySpelling[1]!; - } - } - // Simple name match - if (node.name === symbol) return true; - // File basename match (e.g., "product-card" matches "product-card.liquid") - if (node.kind === 'file' && node.name.replace(/\.[^.]+$/, '') === symbol) return true; - - // Qualified-name lookups: split on any supported separator. `\w` keeps - // identifier chars (incl. `_`) intact; everything else is treated as - // a separator we tolerate. - if (!/[.\/]|::/.test(symbol)) return false; - const parts = symbol.split(/::|[./]/).filter((p) => p.length > 0); - if (parts.length < 2) return false; - - const lastPart = parts[parts.length - 1]!; - if (node.name !== lastPart) return false; - - // Stage 1: qualified-name suffix match. The extractor joins the - // semantic hierarchy with `::`, so `Session.request` and - // `Session::request` both become `Session::request` here. - const colonSuffix = parts.join('::'); - if (node.qualifiedName.includes(colonSuffix)) return true; - - // Stage 2: file-path containment. Rust modules and Python packages - // are not in `qualifiedName` — they're encoded in the file path. So - // `stage_apply::run` matches a `run` in any file whose path - // contains a `stage_apply` segment (with or without an extension). - // - // Filter out Rust path prefixes that have no file-system equivalent. - const containerHints = parts.slice(0, -1).filter((p) => !RUST_PATH_PREFIXES.has(p)); - if (containerHints.length === 0) return false; - - const segments = node.filePath.split('/').filter((s) => s.length > 0); - return containerHints.every((hint) => - segments.some((seg) => seg === hint || seg.replace(/\.[^.]+$/, '') === hint) - ); + return matchesSymbolShared(node, symbol); } /** From dcb370937d5eae49244f87c0307b5f358d946789 Mon Sep 17 00:00:00 2001 From: ferres Date: Sun, 30 Aug 2026 22:19:23 +0000 Subject: [PATCH 2/5] feat(proto): index Protocol Buffers as a contract, not an asset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `.proto` is the contract layer of a polyglot repository: one field is implemented again in every generated language, each of those sites is machine-written and must never be hand-edited, and none of it was visible to the graph. Indexing it makes a field a symbol you can ask questions about — and makes "what else moves when this changes" answerable from the one place the answer is actually written down. Extracted: messages (including nested), enums and their values, fields, `oneof` members, `map` types, services, `rpc`s (including `stream`), imports, and `reserved`. Names are protobuf's own fully-qualified names (`acme.reporting.v1.Measurement.observed_at`), which is both what a user would type and what a generator uses. Field types and rpc request / response messages become `references`, so `callers` on a message lists the fields and RPCs that depend on it. Two properties are modelled on purpose, because they are where protobuf's real defects live and neither survives a naive extraction: - THE TAG NUMBER IS PART OF A FIELD'S IDENTITY. Renaming a field at the same tag is wire-compatible; changing its TYPE at the same tag and name is a silent mis-decode that every single-language check passes. The tag is recorded as a marker, not just left in prose, so a check can read it without re-parsing the declaration. - `reserved` IS SEMANTIC. A retired number must never be re-used and a reader touching a reserved field is a defect, so reservations — numbers, ranges and names — are kept as symbols rather than discarded as syntax. They are deliberately not `field` nodes, so a reservation can never be mistaken for a live field. Implemented as a standalone scanner (proto-extractor.ts) rather than a vendored grammar, following the Liquid / Razor / MyBatis precedent. The IDL is small and effectively frozen, so the usual reason to want a grammar — tracking an evolving syntax surface — does not apply, and it avoids shipping another megabyte of wasm whose silent absence is its own failure mode. Comments are blanked offset-preserving before scanning, skipping string literals so a URL in an option value is not mistaken for a comment. Verified against upstream protos: descriptor.proto (proto2, groups, extensions, 17 reserved statements), pubsub.proto, struct.proto, timestamp.proto and grpc health.proto all parse with zero errors, and the message / enum / service / rpc counts match the files exactly. Depends on the qualified-lookup fix: without it a protobuf FQN is a dotted container name, which the old matcher could not match at all. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013Bsi9EH64kMisnik4E1gf7 --- CHANGELOG.md | 4 + README.md | 3 +- __tests__/proto-extraction.test.ts | 328 +++++++++++++++++++ src/extraction/grammars.ts | 11 +- src/extraction/proto-extractor.ts | 498 +++++++++++++++++++++++++++++ src/extraction/tree-sitter.ts | 6 + src/types.ts | 1 + 7 files changed, 848 insertions(+), 3 deletions(-) create mode 100644 __tests__/proto-extraction.test.ts create mode 100644 src/extraction/proto-extractor.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 761b6b1b8..0005074ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### New Features + +- CodeGraph now indexes **Protocol Buffers** (`.proto`). Messages, nested messages, enums and their values, fields, `oneof` members, services and their `rpc`s all become symbols, named the way protobuf names them — so `acme.reporting.v1.Measurement.observed_at` is something you can look up, and asking what depends on a message lists the fields and RPCs that use it. Two details are modelled on purpose: a field keeps its **tag number**, because renaming a field at the same tag is harmless while changing its type at the same tag is a silent wire break; and `reserved` numbers, ranges and names are kept as symbols, so a retired field is still findable instead of vanishing from the graph. + ### Fixes - `codegraph callers`, `codegraph callees` and `codegraph impact` now understand qualified symbol names. Asking for something like `Accounts.Format.group` used to match nothing, so the command quietly fell back to whichever symbol full-text search happened to rank first — or reported "not found" for a symbol that plainly exists. Qualified names now select exactly the definition you named, including in languages whose module names themselves contain dots. diff --git a/README.md b/README.md index 48323f6fd..6520fbf53 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ CodeGraph's parsing engine is a **native Rust kernel**: 20 languages — TypeScr | **Full-Text Search** | Find code by name instantly across your entire codebase, powered by FTS5 | | **Impact Analysis** | Trace callers, callees, and the full impact radius of any symbol before making changes | | **Always Fresh** | File watcher uses native OS events (FSEvents/inotify/ReadDirectoryChangesW) with debounced auto-sync — the graph stays current as you code, zero config | -| **20+ Languages** | TypeScript, JavaScript, ArkTS, Python, Go, Rust, Java, C#, VB.NET, PHP, Ruby, C, C++, CUDA, Objective-C, Metal, Swift, Kotlin, Scala, Dart, Lua, Luau, R, Nix, Erlang, CFML, COBOL, Solidity, Terraform/OpenTofu, Svelte, Vue, Astro, Liquid, Pascal/Delphi | +| **20+ Languages** | TypeScript, JavaScript, ArkTS, Python, Go, Rust, Java, C#, VB.NET, PHP, Ruby, C, C++, CUDA, Objective-C, Metal, Swift, Kotlin, Scala, Dart, Lua, Luau, R, Nix, Erlang, Protocol Buffers, CFML, COBOL, Solidity, Terraform/OpenTofu, Svelte, Vue, Astro, Liquid, Pascal/Delphi | | **Framework-aware Routes** | Recognizes web-framework routing files and links URL patterns to their handlers across 17 frameworks | | **Mixed iOS / React Native / Expo** | Closes cross-language flows that static parsing misses: Swift ↔ ObjC bridging, React Native legacy bridge + TurboModules + Fabric view components, native → JS event emitters, Expo Modules | | **100% Local** | No data leaves your machine. No API keys. No external services. SQLite database only | @@ -822,6 +822,7 @@ is written): | COBOL | `.cbl`, `.cob`, `.cpy` | Full support (programs, sections/paragraphs with PERFORM/GO TO call edges, CALL 'literal' cross-program calls, COPY copybook imports — including standalone `.cpy` files — DATA DIVISION records/fields/88-levels, EXEC CICS LINK/XCTL and EXEC SQL INCLUDE targets; fixed and free format) | | Visual Basic .NET | `.vb` | Full support (classes, Modules, interfaces, structures, enums, properties, events, `Declare` P/Invoke, `Handles`/`WithEvents`, `Inherits`/`Implements` edges, call edges through VB's call/index paren ambiguity, `As New` instantiation, interpolated strings, LINQ, Unicode identifiers) | | Erlang | `.erl`, `.hrl`, `.escript`, `.app.src`, `.app` | Full support (functions with multi-clause/multi-arity grouping, `-spec` signatures, records with fields, `-type`/`-opaque` aliases, `-define` macros, `-include`/`-include_lib`/`-import` edges, local and `mod:fn` remote call edges, `fun name/arity` references, `spawn`/`apply`/`proc_lib`/`timer`/`rpc` MFA-argument call edges, `gen_server:call/cast(?MODULE)` → own `handle_call`/`handle_cast` links, `-behaviour` links, `-export`-based visibility) | +| Protocol Buffers | `.proto` | Full support (messages incl. nested, enums and their values, fields with their **tag number** and declared type, `oneof` members as fields of the enclosing message, `map` types, services and `rpc`s incl. `stream`, `import`s, and `reserved` numbers, ranges and names kept as symbols so retired fields stay queryable; names are the protobuf fully-qualified names, and field/rpc types link to the declarations they use) | | Solidity | `.sol` | Full support (contracts, libraries, interfaces, structs, enums, modifiers, events, errors, state variables, `import`/`using` directives, `emit`/`revert` calls) | | Terraform / OpenTofu | `.tf`, `.tfvars`, `.tofu` | Full support (resources, data sources, modules, variables, outputs, providers incl. aliases, `locals`; `var.`/`local.`/`module.`/resource references with Terraform's per-directory scoping enforced; module calls bridged across the boundary — inputs to the child module's variables, `module.M.out` to the child's output, `source` to the module's files; cloudposse/atmos `remote-state` cross-component wiring when the component is statically named; `provider = aws.east` selections resolved up the module tree; `moved`/`import`/`removed`/`check` block references; `.tfvars` assignments linked to the variables they set) | | Nix | `.nix` | Full support (functions with simple/destructured/curried params, `let`/attrset bindings, `inherit`, `import ./path` file edges — `./dir` resolving through `default.nix` — plus NixOS module `imports = [ ./x.nix ]` lists and `callPackage ./pkg.nix` file edges; call edges; module-system option wiring — a config write like `launchd.user.agents.x = { ... }` links to the module declaring `options.launchd.user.agents`, so option flows trace across modules) | diff --git a/__tests__/proto-extraction.test.ts b/__tests__/proto-extraction.test.ts new file mode 100644 index 000000000..4494614c3 --- /dev/null +++ b/__tests__/proto-extraction.test.ts @@ -0,0 +1,328 @@ +/** + * Protocol Buffers (`.proto`) extraction. + * + * A `.proto` is a contract: one field is implemented again in every generated + * language, and each of those sites is machine-written. For the graph to answer + * "what else moves when this changes", a field has to be a symbol — and two of + * its properties have to survive extraction, because they are where protobuf's + * real defects live: + * + * - the TAG NUMBER is part of a field's identity. Renaming a field but + * keeping its tag is wire-compatible; keeping tag AND name while changing + * the type is a silent mis-decode that every single-language check passes; + * - `reserved` is semantic — a retired number must never be re-used, and a + * reader touching a reserved field is a defect. + */ + +import { describe, it, expect } from 'vitest'; +import { extractFromSource } from '../src/extraction/tree-sitter'; +import { detectLanguage, isSourceFile, isLanguageSupported, getSupportedLanguages } from '../src/extraction/grammars'; +import { blankProtoComments, parseField, parseRpc, typeReferences } from '../src/extraction/proto-extractor'; +import { matchesSymbol } from '../src/graph/symbol-lookup'; +import type { Node } from '../src/types'; + +const SAMPLE = `syntax = "proto3"; + +package acme.reporting.v1; + +import "google/protobuf/timestamp.proto"; +import public "acme/common/money.proto"; + +option go_package = "acme/reporting"; + +// A single reported measurement. +// Values are already normalised. +message Measurement { + // Stable identifier. + string id = 1; + double value = 2; + /* The unit the value is expressed in. */ + Unit unit = 3; + repeated string tags = 4; + map breakdown = 5; + acme.common.Money cost = 6; + google.protobuf.Timestamp observed_at = 7; + + reserved 8, 9; + reserved 12 to 15; + reserved "legacy_value", "old_unit"; + + oneof source { + string manual_entry = 20; + Ingest ingest = 21; + } + + message Ingest { + string pipeline = 1; + } +} + +// Units a measurement may carry. +enum Unit { + UNIT_UNSPECIFIED = 0; + UNIT_CURRENCY = 1; +} + +message ListRequest { string filter = 1; } +message ListResponse { repeated Measurement measurements = 1; } + +// Read-side API. +service ReportingService { + // Page through measurements. + rpc List(ListRequest) returns (ListResponse); + rpc Stream(ListRequest) returns (stream Measurement) { + option idempotency_level = NO_SIDE_EFFECTS; + } +} +`; + +function extract(source = SAMPLE) { + return extractFromSource('proto/service.proto', source, 'proto'); +} + +function byQualified(nodes: Node[], qualified: string): Node | undefined { + return nodes.find((n) => n.qualifiedName === qualified); +} + +describe('proto — language registration', () => { + it('detects .proto files', () => { + expect(detectLanguage('proto/service.proto')).toBe('proto'); + expect(isSourceFile('proto/service.proto')).toBe(true); + }); + + it('reports proto as supported', () => { + expect(isLanguageSupported('proto')).toBe(true); + expect(getSupportedLanguages()).toContain('proto'); + }); +}); + +describe('proto — pure parsing helpers', () => { + it('blanks comments while preserving offsets and line numbers', () => { + const src = 'a // note\nb /* x\ny */ c\n'; + const out = blankProtoComments(src); + expect(out.length).toBe(src.length); + expect(out.split('\n').length).toBe(src.split('\n').length); + expect(out).not.toContain('note'); + }); + + it('does not treat a // inside a string literal as a comment', () => { + // An option value is frequently a URL — mistaking it for a comment would + // swallow the rest of the line, including its terminating semicolon. + const src = 'option (x) = "http://example.com/a"; message M { string a = 1; }'; + const out = blankProtoComments(src); + expect(out).toContain('http://example.com/a'); + const result = extract(src); + expect(byQualified(result.nodes, 'M.a')).toBeDefined(); + }); + + it('parses field declarations including label, map types and tag', () => { + expect(parseField('string id = 1')).toMatchObject({ type: 'string', name: 'id', tag: 1 }); + expect(parseField('repeated Foo bar = 3')).toMatchObject({ label: 'repeated', type: 'Foo', name: 'bar', tag: 3 }); + expect(parseField('map breakdown = 5')).toMatchObject({ + type: 'map', name: 'breakdown', tag: 5, + }); + expect(parseField('.acme.Money cost = 6')).toMatchObject({ type: '.acme.Money', tag: 6 }); + expect(parseField('not a field')).toBeNull(); + }); + + it('parses rpc declarations including stream markers', () => { + expect(parseRpc('rpc List(Req) returns (Res)')).toMatchObject({ + name: 'List', input: 'Req', output: 'Res', streamIn: false, streamOut: false, + }); + expect(parseRpc('rpc S(stream Req) returns (stream Res)')).toMatchObject({ + streamIn: true, streamOut: true, + }); + expect(parseRpc('message M')).toBeNull(); + }); + + it('reports only declared types as references, never scalars', () => { + expect(typeReferences('string')).toEqual([]); + expect(typeReferences('Foo')).toEqual(['Foo']); + expect(typeReferences('map')).toEqual([]); + expect(typeReferences('map')).toEqual(['Money']); + }); +}); + +describe('proto — declarations', () => { + const result = extract(); + + it('extracts messages, enums and services under the package name', () => { + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement')?.kind).toBe('struct'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Unit')?.kind).toBe('enum'); + expect(byQualified(result.nodes, 'acme.reporting.v1.ReportingService')?.kind).toBe('interface'); + }); + + it('nests a nested message under its parent, as protobuf names it', () => { + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.Ingest')?.kind).toBe('struct'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.Ingest.pipeline')?.kind).toBe('field'); + }); + + it('extracts enum values with their numbers', () => { + const v = byQualified(result.nodes, 'acme.reporting.v1.Unit.UNIT_CURRENCY'); + expect(v?.kind).toBe('enum_member'); + expect(v?.decorators).toContain('number=1'); + }); + + it('extracts rpcs as methods of their service, with the stream marker kept', () => { + const list = byQualified(result.nodes, 'acme.reporting.v1.ReportingService.List'); + expect(list?.kind).toBe('method'); + expect(list?.signature).toBe('rpc List(ListRequest) returns (ListResponse)'); + const stream = byQualified(result.nodes, 'acme.reporting.v1.ReportingService.Stream'); + expect(stream?.signature).toContain('returns (stream Measurement)'); + }); + + it('carries // and /* */ comments through as docstrings', () => { + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement')?.docstring) + .toBe('A single reported measurement.\nValues are already normalised.'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.id')?.docstring) + .toBe('Stable identifier.'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.unit')?.docstring) + .toBe('The unit the value is expressed in.'); + }); + + it('records imports, including the `public` form', () => { + const imports = result.unresolvedReferences.filter((r) => r.referenceKind === 'imports'); + expect(imports.map((r) => r.referenceName)).toEqual([ + 'google/protobuf/timestamp.proto', + 'acme/common/money.proto', + ]); + }); + + it('ignores syntax and option statements', () => { + expect(result.nodes.some((n) => n.name === 'go_package')).toBe(false); + expect(result.nodes.some((n) => n.name === 'idempotency_level')).toBe(false); + }); + + it('reports no extraction errors on a representative file', () => { + expect(result.errors).toEqual([]); + }); +}); + +describe('proto — the tag number is part of a field identity', () => { + const result = extract(); + + it('records each field tag as a marker, not only as prose', () => { + // A "same tag, changed type" check has to read the tag without re-parsing + // the declaration. + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.id')?.decorators).toContain('tag=1'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.observed_at')?.decorators).toContain('tag=7'); + }); + + it('keeps the declaration verbatim so type and tag are both visible', () => { + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.observed_at')?.signature) + .toBe('google.protobuf.Timestamp observed_at = 7'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.tags')?.signature) + .toBe('repeated string tags = 4'); + }); + + it('marks repeated fields', () => { + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.tags')?.decorators).toContain('repeated'); + }); + + it('distinguishes a renamed field from a retyped one at the same tag', () => { + // Rename at the same tag: wire-compatible. Retype at the same tag and name: + // a silent mis-decode. The graph must be able to tell these apart. + const renamed = extract('message M { string b = 1; }'); + const retyped = extract('message M { int64 a = 1; }'); + const original = extract('message M { string a = 1; }'); + const tagOf = (r: ReturnType, name: string) => + r.nodes.find((n) => n.name === name)?.decorators?.find((d) => d.startsWith('tag=')); + expect(tagOf(original, 'a')).toBe('tag=1'); + expect(tagOf(renamed, 'b')).toBe('tag=1'); + expect(tagOf(retyped, 'a')).toBe('tag=1'); + // Same tag either way — the difference is in the declaration text. + expect(original.nodes.find((n) => n.name === 'a')?.signature).toBe('string a = 1'); + expect(retyped.nodes.find((n) => n.name === 'a')?.signature).toBe('int64 a = 1'); + }); +}); + +describe('proto — reserved is kept, not discarded as syntax', () => { + const result = extract(); + const reserved = result.nodes.filter((n) => n.decorators?.includes('reserved')); + + it('records reserved numbers individually', () => { + expect(reserved.map((n) => n.name)).toEqual( + expect.arrayContaining(['reserved 8', 'reserved 9']) + ); + }); + + it('records reserved ranges as written', () => { + expect(reserved.map((n) => n.name)).toContain('reserved 12 to 15'); + // A range must not also be double-counted as its two endpoints. + expect(reserved.map((n) => n.name)).not.toContain('reserved 12'); + expect(reserved.map((n) => n.name)).not.toContain('reserved 15'); + }); + + it('records reserved names so a lookup for a retired name finds the reservation', () => { + expect(reserved.map((n) => n.name)).toEqual( + expect.arrayContaining(['reserved legacy_value', 'reserved old_unit']) + ); + }); + + it('never lets a reservation be mistaken for a live field', () => { + for (const n of reserved) expect(n.kind).not.toBe('field'); + expect(result.nodes.some((n) => n.kind === 'field' && n.name === 'legacy_value')).toBe(false); + }); +}); + +describe('proto — oneof members belong to the enclosing message', () => { + const result = extract(); + + it('does not add the oneof name as a level in the qualified name', () => { + // On the wire a oneof member IS a field of the message; a `.source.` level + // would not match the name any generator uses. + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.manual_entry')?.kind).toBe('field'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.source.manual_entry')).toBeUndefined(); + }); + + it('still marks them as oneof members and keeps their tags', () => { + const node = byQualified(result.nodes, 'acme.reporting.v1.Measurement.manual_entry'); + expect(node?.decorators).toContain('oneof'); + expect(node?.decorators).toContain('tag=20'); + }); +}); + +describe('proto — dependencies between declarations', () => { + const result = extract(); + const refs = result.unresolvedReferences.filter((r) => r.referenceKind === 'references'); + + it('links a field to the message or enum it is typed with', () => { + expect(refs.map((r) => r.referenceName)).toEqual( + expect.arrayContaining(['Unit', 'acme.common.Money', 'google.protobuf.Timestamp', 'Ingest']) + ); + }); + + it('does not emit references for scalar field types', () => { + for (const scalar of ['string', 'double', 'int64', 'bool', 'bytes']) { + expect(refs.map((r) => r.referenceName)).not.toContain(scalar); + } + }); + + it('links an rpc to both its request and response messages', () => { + const names = refs.map((r) => r.referenceName); + expect(names).toEqual(expect.arrayContaining(['ListRequest', 'ListResponse', 'Measurement'])); + }); + + it('strips protobufs leading-dot absolute marker from a reference', () => { + const absolute = extract('package p;\nmessage M { .other.pkg.Thing t = 1; }'); + const names = absolute.unresolvedReferences.map((r) => r.referenceName); + expect(names).toContain('other.pkg.Thing'); + expect(names).not.toContain('.other.pkg.Thing'); + }); +}); + +describe('proto — a fully-qualified protobuf name is a usable query', () => { + const result = extract(); + + it('matches a field by its dotted protobuf FQN', () => { + const field = byQualified(result.nodes, 'acme.reporting.v1.Measurement.observed_at')!; + expect(matchesSymbol(field, 'acme.reporting.v1.Measurement.observed_at')).toBe(true); + // A partial suffix on a separator boundary is how people actually type it. + expect(matchesSymbol(field, 'Measurement.observed_at')).toBe(true); + }); + + it('does not match a field of a different message', () => { + const field = byQualified(result.nodes, 'acme.reporting.v1.Measurement.Ingest.pipeline')!; + expect(matchesSymbol(field, 'ListRequest.pipeline')).toBe(false); + }); +}); diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 84647c3e4..18d263a6f 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -11,7 +11,7 @@ import * as fsp from 'fs/promises'; import { Parser, Language as WasmLanguage } from 'web-tree-sitter'; import { Language } from '../types'; -export type GrammarLanguage = Exclude; +export type GrammarLanguage = Exclude; /** * WASM filename map — maps each language to its .wasm grammar file @@ -170,6 +170,10 @@ export const EXTENSION_MAP: Record = { '.tf': 'terraform', '.tfvars': 'terraform', '.tofu': 'terraform', + // Protocol Buffers IDL. Standalone scanner (proto-extractor.ts), not a + // tree-sitter grammar — the IDL is small and frozen, and a `.proto` is a + // contract whose tag numbers and reservations need modelling, not just a tree. + '.proto': 'proto', }; /** @@ -575,6 +579,7 @@ export function isLanguageSupported(language: Language): boolean { if (language === 'twig') return true; // file-level tracking only if (language === 'xml') return true; // MyBatis mapper extractor if (language === 'properties') return true; // Spring config keys + if (language === 'proto') return true; // custom ProtoExtractor (no wasm grammar) if (language === 'unknown') return false; return language in WASM_GRAMMAR_FILES; } @@ -586,6 +591,7 @@ export function isGrammarLoaded(language: Language): boolean { if (language === 'svelte' || language === 'vue' || language === 'astro' || language === 'liquid' || language === 'razor') return true; if (language === 'yaml' || language === 'twig') return true; // no WASM grammar needed if (language === 'xml' || language === 'properties') return true; // no WASM grammar needed + if (language === 'proto') return true; // no WASM grammar needed return languageCache.has(language); } @@ -606,7 +612,7 @@ export function isFileLevelOnlyLanguage(language: Language): boolean { * Get all supported languages (those with grammar definitions). */ export function getSupportedLanguages(): Language[] { - return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid']; + return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid', 'proto']; } /** @@ -693,6 +699,7 @@ export function getLanguageDisplayName(language: Language): string { vbnet: 'Visual Basic .NET', erlang: 'Erlang', terraform: 'Terraform', + proto: 'Protocol Buffers', arkts: 'ArkTS', unknown: 'Unknown', }; diff --git a/src/extraction/proto-extractor.ts b/src/extraction/proto-extractor.ts new file mode 100644 index 000000000..22f39eb04 --- /dev/null +++ b/src/extraction/proto-extractor.ts @@ -0,0 +1,498 @@ +import { Node, Edge, ExtractionResult, ExtractionError, UnresolvedReference, NodeKind } from '../types'; +import { generateNodeId } from './tree-sitter-helpers'; + +/** + * ProtoExtractor — Protocol Buffers (`.proto`) IDL. + * + * A `.proto` file is a CONTRACT: one field definition is implemented three or + * more times over, once per generated language, and every one of those sites is + * machine-written and must never be hand-edited. That makes the `.proto` the + * highest-value file per byte in a polyglot repository and the natural anchor + * for "what else moves when this changes" — but only if a field is a symbol the + * graph can answer questions about. + * + * Written as a standalone scanner rather than a vendored tree-sitter grammar, + * following the same precedent as the Liquid / Razor / MyBatis extractors. The + * protobuf IDL is tiny and effectively frozen (proto2 and proto3 are published, + * stable specs), so the usual reason to want a grammar — keeping up with an + * evolving syntax surface — does not apply; and every vendored `.wasm` is a + * megabyte of shipped binary plus an ABI obligation, whose silent absence is + * itself a failure mode worth not multiplying. + * + * Two properties are modelled deliberately, because they are where protobuf's + * real defects live and neither is expressible without them: + * + * - THE TAG NUMBER IS PART OF A FIELD'S IDENTITY. Renaming a field while + * keeping its tag is wire-compatible; changing its TYPE (or its meaning) + * while keeping tag and name is a silent mis-decode that every + * single-language check passes. Both the declaration text and the tag are + * recorded, so the difference is expressible. + * - `reserved` IS SEMANTIC. A retired field number must never be re-used, and + * a reader that touches a reserved field is a defect. Reservations are kept + * as nodes rather than discarded as syntax, so that check can be written. + */ + +/** Built-in scalar types — never a reference to another declaration. */ +const SCALAR_TYPES = new Set([ + 'double', 'float', 'int32', 'int64', 'uint32', 'uint64', 'sint32', 'sint64', + 'fixed32', 'fixed64', 'sfixed32', 'sfixed64', 'bool', 'string', 'bytes', +]); + +/** Statement keywords that carry no graph structure of their own. */ +const IGNORED_STATEMENTS = new Set(['syntax', 'option', 'extensions', 'edition']); + +interface Scope { + /** Dotted protobuf name of this scope, e.g. `pkg.Outer.Inner`. */ + qualified: string; + /** What opened it — decides how child statements are read. */ + kind: 'message' | 'enum' | 'service' | 'oneof' | 'extend' | 'rpc' | 'unknown'; + /** Node id, so children attach by `contains`. */ + nodeId?: string; +} + +export class ProtoExtractor { + private filePath: string; + private source: string; + /** Comments blanked to spaces, newlines preserved — offsets and lines hold. */ + private code: string; + private nodes: Node[] = []; + private edges: Edge[] = []; + private unresolvedReferences: UnresolvedReference[] = []; + private errors: ExtractionError[] = []; + private packageName = ''; + private scopes: Scope[] = []; + + constructor(filePath: string, source: string) { + this.filePath = filePath; + this.source = source; + this.code = blankProtoComments(source); + } + + extract(): ExtractionResult { + const startTime = Date.now(); + try { + const fileNode = this.createFileNode(); + this.scopes = [{ qualified: '', kind: 'unknown', nodeId: fileNode.id }]; + this.scan(); + } catch (error) { + this.errors.push({ + message: `Proto extraction error: ${error instanceof Error ? error.message : String(error)}`, + filePath: this.filePath, + severity: 'error', + code: 'parse_error', + }); + } + return { + nodes: this.nodes, + edges: this.edges, + unresolvedReferences: this.unresolvedReferences, + errors: this.errors, + durationMs: Date.now() - startTime, + }; + } + + // --- scanning ------------------------------------------------------------- + + /** + * Walk the comment-free source once, splitting it into the two things + * protobuf is made of: block HEADERS (text before a `{`) and STATEMENTS (text + * before a `;`). A brace-depth scope stack gives every declaration its + * enclosing message/service, which is what makes the dotted names below the + * real protobuf fully-qualified names. + */ + private scan(): void { + let buffer = ''; + let bufferStart = -1; + for (let i = 0; i < this.code.length; i++) { + const ch = this.code[i]!; + if (ch === '{' || ch === '}' || ch === ';') { + const text = buffer.trim(); + const line = bufferStart >= 0 ? this.lineAt(bufferStart) : this.lineAt(i); + if (ch === '{') this.openBlock(text, line, i); + else if (ch === ';') this.statement(text, line, i); + else this.closeBlock(); + buffer = ''; + bufferStart = -1; + continue; + } + if (buffer.length === 0 && /\s/.test(ch)) continue; // skip leading space + if (buffer.length === 0) bufferStart = i; + buffer += ch; + } + } + + private openBlock(header: string, line: number, index: number): void { + const decl = /^(message|enum|service|oneof|extend)\s+([A-Za-z_][\w.]*)/.exec(header); + if (decl) { + const kind = decl[1] as Scope['kind']; + const name = decl[2]!; + const node = this.declare(kind, name, line, index, header); + // A `oneof` groups fields that already belong to the enclosing message — + // on the wire they ARE its fields. It must not contribute a level to + // their qualified names, which have to stay the protobuf FQN a generator + // (and the code generated from it) uses. + const qualified = kind === 'oneof' ? this.currentScope().qualified : this.qualify(name); + this.scopes.push({ + qualified, + kind, + nodeId: node?.id ?? this.currentScope().nodeId, + }); + return; + } + // `rpc Get(Req) returns (Res) { option ...; }` — the braced rpc form. + const rpc = parseRpc(header); + if (rpc) { + const node = this.declareRpc(rpc, line, index, header); + this.scopes.push({ qualified: this.qualify(rpc.name), kind: 'rpc', nodeId: node?.id }); + return; + } + // Anything else with a body (an `option (x) = { ... }` block, a group): + // push an opaque scope so brace depth stays correct and its contents are + // not mistaken for fields of the enclosing message. + this.scopes.push({ qualified: this.currentScope().qualified, kind: 'unknown' }); + } + + private closeBlock(): void { + if (this.scopes.length > 1) this.scopes.pop(); + } + + private statement(text: string, line: number, index: number): void { + if (!text) return; + const keyword = /^([A-Za-z_]\w*)/.exec(text)?.[1] ?? ''; + if (IGNORED_STATEMENTS.has(keyword)) return; + + if (keyword === 'package') { + const pkg = /^package\s+([\w.]+)$/.exec(text)?.[1]; + if (pkg) this.packageName = pkg; + return; + } + if (keyword === 'import') { + this.declareImport(text, line, index); + return; + } + if (keyword === 'reserved') { + this.declareReserved(text, line, index); + return; + } + if (keyword === 'rpc') { + const rpc = parseRpc(text); + if (rpc) this.declareRpc(rpc, line, index, text); + return; + } + + const scope = this.currentScope(); + if (scope.kind === 'enum') { + this.declareEnumValue(text, line, index); + return; + } + if (scope.kind === 'message' || scope.kind === 'oneof' || scope.kind === 'extend') { + this.declareField(text, line, index); + } + } + + // --- declarations --------------------------------------------------------- + + private declare( + kind: Scope['kind'], + name: string, + line: number, + index: number, + header: string + ): Node | null { + // A `oneof` is a grouping construct, not a type: its fields belong to the + // enclosing message and its own name would only add a phantom level to + // their qualified names, which must stay the wire-level protobuf FQN. + if (kind === 'oneof') return null; + const nodeKind: NodeKind = + kind === 'enum' ? 'enum' : kind === 'service' ? 'interface' : 'struct'; + return this.push(nodeKind, name, line, index, { + signature: collapse(header), + docstring: this.docFor(index), + isExported: true, + }); + } + + private declareRpc( + rpc: { name: string; input: string; output: string; streamIn: boolean; streamOut: boolean }, + line: number, + index: number, + header: string + ): Node | null { + const node = this.push('method', rpc.name, line, index, { + signature: collapse(header).replace(/\s*\{$/, ''), + docstring: this.docFor(index), + isExported: true, + returnType: rpc.output, + }); + // The request and response messages are this method's real dependencies — + // the edge that makes "what does changing this message break" answerable. + for (const type of [rpc.input, rpc.output]) { + this.reference(node?.id, type, line, index); + } + return node; + } + + private declareField(text: string, line: number, index: number): void { + const field = parseField(text); + if (!field) return; + const node = this.push('field', field.name, line, index, { + signature: collapse(text), + docstring: this.docFor(index), + isExported: true, + // The tag is part of the field's identity on the wire, so it is recorded + // as a marker rather than left only in the signature prose: a check for + // "same tag, changed type" has to be able to read it without re-parsing. + decorators: [ + `tag=${field.tag}`, + ...(field.label ? [field.label] : []), + ...(this.currentScope().kind === 'oneof' ? ['oneof'] : []), + ], + }); + for (const type of typeReferences(field.type)) { + this.reference(node?.id, type, line, index); + } + } + + private declareEnumValue(text: string, line: number, index: number): void { + const m = /^([A-Za-z_]\w*)\s*=\s*(-?\d+)\b/.exec(text); + if (!m) return; + this.push('enum_member', m[1]!, line, index, { + signature: collapse(text), + docstring: this.docFor(index), + isExported: true, + decorators: [`number=${m[2]}`], + }); + } + + /** + * `reserved 2, 15, 9 to 11;` / `reserved "old_name";` + * + * Kept as a node so "was this number retired" and "is this name off-limits" + * are answerable. Named after what it reserves so a lookup for the retired + * name finds the reservation, with a `reserved` marker so it can never be + * mistaken for a live field. + */ + private declareReserved(text: string, line: number, index: number): void { + const body = text.replace(/^reserved\s+/, ''); + const items: string[] = []; + for (const m of body.matchAll(/"([^"]+)"|'([^']+)'/g)) items.push(m[1] ?? m[2]!); + for (const m of body.matchAll(/(\d+)\s+to\s+(\d+|max)/gi)) items.push(`${m[1]} to ${m[2]}`); + const ranged = body.replace(/(\d+)\s+to\s+(\d+|max)/gi, ''); + for (const m of ranged.matchAll(/\b(\d+)\b/g)) items.push(m[1]!); + if (items.length === 0) return; + for (const item of items) { + this.push('constant', `reserved ${item}`, line, index, { + signature: collapse(text), + decorators: ['reserved'], + }); + } + } + + private declareImport(text: string, line: number, index: number): void { + const target = /^import\s+(?:public\s+|weak\s+)?["']([^"']+)["']$/.exec(text)?.[1]; + if (!target) return; + const node = this.push('import', target, line, index, { signature: collapse(text) }); + if (node) node.qualifiedName = target; + const from = this.scopes[0]!.nodeId; + if (!from) return; + this.unresolvedReferences.push({ + fromNodeId: from, + referenceName: target, + referenceKind: 'imports', + filePath: this.filePath, + language: 'proto', + line, + column: 0, + }); + } + + // --- helpers -------------------------------------------------------------- + + private currentScope(): Scope { + return this.scopes[this.scopes.length - 1]!; + } + + /** The protobuf fully-qualified name of a member of the current scope. */ + private qualify(name: string): string { + const parent = this.currentScope().qualified || this.packageName; + return parent ? `${parent}.${name}` : name; + } + + private push(kind: NodeKind, name: string, line: number, index: number, extra: Partial): Node | null { + if (!name) return null; + const qualifiedName = this.qualify(name); + const node: Node = { + id: generateNodeId(this.filePath, kind, qualifiedName, line), + kind, + name, + qualifiedName, + filePath: this.filePath, + language: 'proto', + startLine: line, + endLine: line, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + ...extra, + }; + this.nodes.push(node); + const parentId = this.currentScope().nodeId; + if (parentId) this.edges.push({ source: parentId, target: node.id, kind: 'contains' }); + void index; + return node; + } + + /** A dependency on another declaration (a field's type, an rpc's messages). */ + private reference(fromNodeId: string | undefined, typeName: string, line: number, index: number): void { + if (!fromNodeId || !typeName || SCALAR_TYPES.has(typeName)) return; + void index; + this.unresolvedReferences.push({ + fromNodeId, + // A leading dot is protobuf's "fully qualified from the root" marker; + // an unqualified name resolves against the enclosing package. + referenceName: typeName.replace(/^\./, ''), + referenceKind: 'references', + filePath: this.filePath, + language: 'proto', + line, + column: 0, + }); + } + + private lineAt(index: number): number { + let line = 1; + for (let i = 0; i < index && i < this.code.length; i++) { + if (this.code[i] === '\n') line++; + } + return line; + } + + /** + * The `//` or `/* *\/` comment block immediately above a declaration, read + * from the ORIGINAL source (the scanning copy has them blanked). + */ + private docFor(index: number): string | undefined { + const before = this.source.slice(0, index); + const lines = before.split('\n'); + lines.pop(); // the declaration's own (partial) line + const collected: string[] = []; + for (let i = lines.length - 1; i >= 0; i--) { + const line = lines[i]!.trim(); + if (line === '') { if (collected.length > 0) break; continue; } + const lineComment = /^\/\/+\s?(.*)$/.exec(line); + if (lineComment) { collected.unshift(lineComment[1]!); continue; } + const single = /^\/\*+\s?([\s\S]*?)\s*\*+\/$/.exec(line); + if (single) { collected.unshift(single[1]!); continue; } + const starred = /^\*+\s?(.*)$/.exec(line); + if (starred && collected.length > 0) { collected.unshift(starred[1]!); continue; } + break; + } + const text = collected.join('\n').trim(); + return text || undefined; + } + + private createFileNode(): Node { + const node: Node = { + id: `file:${this.filePath}`, + kind: 'file', + name: this.filePath.split('/').pop() ?? this.filePath, + qualifiedName: this.filePath, + filePath: this.filePath, + language: 'proto', + startLine: 1, + endLine: this.source.split('\n').length, + startColumn: 0, + endColumn: 0, + isExported: false, + updatedAt: Date.now(), + }; + this.nodes.push(node); + return node; + } +} + +// --- pure parsing helpers --------------------------------------------------- + +/** + * Replace comments with spaces, keeping every newline, so line numbers and + * offsets in the blanked copy still match the original. String literals are + * skipped so a `//` inside an option value is not mistaken for a comment. + */ +export function blankProtoComments(source: string): string { + let out = ''; + let i = 0; + while (i < source.length) { + const ch = source[i]!; + if (ch === '"' || ch === "'") { + const quote = ch; + out += ch; + i++; + while (i < source.length) { + const c = source[i]!; + out += c; + i++; + if (c === '\\' && i < source.length) { out += source[i]!; i++; continue; } + if (c === quote) break; + } + continue; + } + if (ch === '/' && source[i + 1] === '/') { + while (i < source.length && source[i] !== '\n') { out += ' '; i++; } + continue; + } + if (ch === '/' && source[i + 1] === '*') { + out += ' '; + i += 2; + while (i < source.length && !(source[i] === '*' && source[i + 1] === '/')) { + out += source[i] === '\n' ? '\n' : ' '; + i++; + } + if (i < source.length) { out += ' '; i += 2; } + continue; + } + out += ch; + i++; + } + return out; +} + +/** `rpc Get(stream Req) returns (Res)` — both the `;` and `{` forms. */ +export function parseRpc(text: string): { + name: string; input: string; output: string; streamIn: boolean; streamOut: boolean; +} | null { + const m = /^rpc\s+([A-Za-z_]\w*)\s*\(\s*(stream\s+)?([.\w]+)\s*\)\s*returns\s*\(\s*(stream\s+)?([.\w]+)\s*\)/.exec(text); + if (!m) return null; + return { + name: m[1]!, + input: m[3]!, + output: m[5]!, + streamIn: !!m[2], + streamOut: !!m[4], + }; +} + +/** + * `repeated Foo bar = 3 [deprecated = true]` — the label is optional, the type + * may itself contain separators (`map`, `.pkg.Foo`), and the tag + * is what anchors the match. + */ +export function parseField(text: string): { + label?: string; type: string; name: string; tag: number; +} | null { + const m = /^(?:(repeated|optional|required)\s+)?(.+?)\s+([A-Za-z_]\w*)\s*=\s*(\d+)\b/.exec(text); + if (!m) return null; + const type = m[2]!.trim(); + if (!type) return null; + return { label: m[1], type, name: m[3]!, tag: Number(m[4]) }; +} + +/** Declared types a field depends on — both halves of a `map`. */ +export function typeReferences(type: string): string[] { + const map = /^map\s*<\s*([^,]+?)\s*,\s*(.+?)\s*>$/.exec(type); + if (map) return [map[1]!, map[2]!].filter((t) => !SCALAR_TYPES.has(t)); + return SCALAR_TYPES.has(type) ? [] : [type]; +} + +function collapse(text: string): string { + return text.replace(/\s+/g, ' ').trim().slice(0, 300); +} diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index c34dc4716..983e48da5 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -30,6 +30,7 @@ import { AstroExtractor } from './astro-extractor'; import { DfmExtractor } from './dfm-extractor'; import { VueExtractor } from './vue-extractor'; import { MyBatisExtractor } from './mybatis-extractor'; +import { ProtoExtractor } from './proto-extractor'; import { CfmlExtractor } from './cfml-extractor'; import { tryKernelExtract, takeDeferredPreParse } from './kernel'; import { @@ -6741,6 +6742,11 @@ export function extractFromSource( // file node so the watcher tracks it without emitting symbols. const extractor = new MyBatisExtractor(filePath, source); result = extractor.extract(); + } else if (detectedLanguage === 'proto') { + // Protocol Buffers IDL — a standalone scanner rather than a grammar; see + // proto-extractor.ts for why, and for the tag/reserved modelling. + const extractor = new ProtoExtractor(filePath, source); + result = extractor.extract(); } else if (detectedLanguage === 'cfml' || detectedLanguage === 'cfscript') { // Custom extractor for CFML (.cfc/.cfm) — dialect-switches between the // tag-based cfml grammar and the bare-script cfscript grammar. Standalone diff --git a/src/types.ts b/src/types.ts index 186f57adc..ccd234cf0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -117,6 +117,7 @@ export const LANGUAGES = [ 'vbnet', 'erlang', 'terraform', + 'proto', 'unknown', ] as const; From 858baccdd1929522a4afc2b1e38f21d53ddcd8b3 Mon Sep 17 00:00:00 2001 From: ferres Date: Sun, 30 Aug 2026 23:15:17 +0000 Subject: [PATCH 3/5] feat(proto): link a .proto declaration to the code generated from it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `.proto` is authored once and implemented again in every language the project generates for. Each of those sites is machine-written and must never be hand-edited, which makes the contract the only place the shared shape exists — and, with no edges, the only place with no link to anything that implements it. That gap holds a family of defects that every tier's own gate passes by construction: a field decoded on one side but never read on another; a field whose meaning changed while its name and tag did not; a field the server stopped sending that a client still declares. Synthesis pass, gated on the project having protos: - a message / enum / service links to its generated type in each target language; - an `rpc` links to the GENERATED METHOD itself — every generator emits one, so this resolves at member level; - a field links to the generated type that declares it. Go struct fields, Python class annotations and TypeScript interface members are all deliberately not extracted as nodes (member-dense code would explode the graph), so there is no member-level peer to match. The declaring type is coarser but true — regenerating it IS what the change requires — and containment does not rescue the field either: dependents traverse INCOMING edges while `contains` points message → field, so a field's impact never climbs to its message. The match kind is recorded in the edge so the two are distinguishable. Edges point GENERATED → PROTO. Generated code is the dependent, and that is the direction impact analysis reads; emitted the other way the relationship is recorded but the question stays unanswered. Discovery is by convention, never configuration — every generator names its output after the proto (`foo_pb2.py`, `foo.pb.ex`, `foo.pb.go`, `foo_pb.ts`), and those conventions are published and stable. Precision comes from the peer-file join: a field named `id` is matched only inside the generated outputs of its OWN proto, never repo-wide, which is what makes a name that common safe to match at all. A file without a generator marker in its name is never treated as generated output, so hand-written code sitting beside a proto is not linked. Also maps `.pyi`, which was not indexed at all. Type stubs are real checked-in API surface, and for protobuf they are the only place per-field Python declarations exist — a modern `_pb2.py` is a serialized descriptor blob with no per-field symbols. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013Bsi9EH64kMisnik4E1gf7 --- CHANGELOG.md | 2 + .../protobuf-contract-synthesizer.test.ts | 245 ++++++++++++++++ src/extraction/grammars.ts | 5 + src/resolution/callback-synthesizer.ts | 5 + .../protobuf-contract-synthesizer.ts | 275 ++++++++++++++++++ 5 files changed, 532 insertions(+) create mode 100644 __tests__/protobuf-contract-synthesizer.test.ts create mode 100644 src/resolution/protobuf-contract-synthesizer.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0005074ac..309cdbbb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features +- A `.proto` declaration is now linked to the code generated from it. Messages, enums and services link to their generated type in every language the project generates for, and each `rpc` links to the generated method itself — so asking what a field or message affects lists the Python, TypeScript, Go or Elixir files that implement it, which are exactly the files a change has to regenerate. Discovery uses the naming conventions the generators already follow (`foo_pb2.py`, `foo.pb.ex`, `foo.pb.go`, `foo_pb.ts`), so nothing needs configuring, and matching is confined to a proto's own generated outputs — a field called `id` is never linked to an unrelated `id` elsewhere in the repo. +- Python type stubs (`.pyi`) are now indexed. They are real, checked-in API surface, and for protobuf they are the only place per-field Python declarations exist at all. - CodeGraph now indexes **Protocol Buffers** (`.proto`). Messages, nested messages, enums and their values, fields, `oneof` members, services and their `rpc`s all become symbols, named the way protobuf names them — so `acme.reporting.v1.Measurement.observed_at` is something you can look up, and asking what depends on a message lists the fields and RPCs that use it. Two details are modelled on purpose: a field keeps its **tag number**, because renaming a field at the same tag is harmless while changing its type at the same tag is a silent wire break; and `reserved` numbers, ranges and names are kept as symbols, so a retired field is still findable instead of vanishing from the graph. ### Fixes diff --git a/__tests__/protobuf-contract-synthesizer.test.ts b/__tests__/protobuf-contract-synthesizer.test.ts new file mode 100644 index 000000000..8f9d3ce81 --- /dev/null +++ b/__tests__/protobuf-contract-synthesizer.test.ts @@ -0,0 +1,245 @@ +/** + * Protobuf contract edges — a `.proto` declaration to the code generated from it. + * + * One field is implemented again in every target language, each of those sites + * is machine-written, and the `.proto` is the only place the shape is authored. + * Without these edges the contract has no link to anything that implements it, + * and a whole family of defects lives in that gap: a field decoded but never + * read, a field whose meaning changed while its name and tag did not, a field + * the server stopped sending that a client still declares. Each is green in + * every single-language check. + * + * The precision gate is the peer-file join: a field named `id` is matched only + * inside the generated outputs OF ITS OWN proto, never repo-wide. + */ + +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { initGrammars, loadAllGrammars } from '../src/extraction/grammars'; +import { generatedPeerStem, nameVariants } from '../src/resolution/protobuf-contract-synthesizer'; + +beforeAll(async () => { + await initGrammars(); + await loadAllGrammars(); +}); + +function hasSqliteBindings(): boolean { + try { + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(':memory:'); + db.close(); + return true; + } catch { + return false; + } +} +const HAS_SQLITE = hasSqliteBindings(); + +describe('generatedPeerStem — discovery by convention, not configuration', () => { + it('recognises the output names each generator produces', () => { + expect(generatedPeerStem('py/gen/measurement_pb2.py')).toBe('measurement'); + expect(generatedPeerStem('py/gen/measurement_pb2.pyi')).toBe('measurement'); + expect(generatedPeerStem('py/gen/measurement_pb2_grpc.py')).toBe('measurement'); + expect(generatedPeerStem('go/gen/measurement.pb.go')).toBe('measurement'); + expect(generatedPeerStem('go/gen/measurement_grpc.pb.go')).toBe('measurement'); + expect(generatedPeerStem('ts/gen/measurement_pb.ts')).toBe('measurement'); + expect(generatedPeerStem('elixir/lib/measurement.pb.ex')).toBe('measurement'); + }); + + it('rejects a hand-written file that merely sits beside the proto', () => { + // Without a generator marker in the name there is nothing to distinguish + // `measurement.ts` from any other source file, and treating it as + // generated output would link the contract to hand-written code. + expect(generatedPeerStem('ts/app/measurement.ts')).toBeNull(); + expect(generatedPeerStem('py/app/service.py')).toBeNull(); + expect(generatedPeerStem('proto/measurement.proto')).toBeNull(); + }); +}); + +describe('nameVariants — the spellings a generator may choose', () => { + it('covers the casings the target languages use', () => { + const variants = nameVariants('observed_at'); + expect(variants).toEqual(expect.arrayContaining([ + 'observed_at', // Python, Elixir + 'observedAt', // TypeScript (ts-proto) + 'ObservedAt', // Go, C# + 'OBSERVED_AT', // enum constants + ])); + }); + + it('round-trips a name that is already camelCase', () => { + expect(nameVariants('observedAt')).toEqual(expect.arrayContaining(['observed_at', 'ObservedAt'])); + }); + + it('includes accessor spellings', () => { + expect(nameVariants('value')).toEqual(expect.arrayContaining(['getValue', 'setValue'])); + }); +}); + +describe.skipIf(!HAS_SQLITE)('protobuf contract edges — end to end', () => { + let root: string; + let cg: any; + let edges: Array>; + + beforeEach(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-proto-contract-')); + const write = (rel: string, body: string) => { + const full = path.join(root, rel); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, body); + }; + + write('proto/measurement.proto', `syntax = "proto3"; +package acme.v1; + +message Measurement { + string id = 1; + string observed_at = 3; +} + +service Reporting { + rpc ListMeasurements(Measurement) returns (Measurement); +} +`); + // Python: a modern `_pb2.py` is a serialized descriptor blob, so the `.pyi` + // stub is where per-declaration Python symbols actually live. + write('py/gen/measurement_pb2.pyi', `class Measurement: + id: str + observed_at: str + +class Reporting: + def ListMeasurements(self, request): ... +`); + // TypeScript, in ts-proto shape — camelCase fields. + write('ts/gen/measurement_pb.ts', `export interface Measurement { + id: string; + observedAt: string; +} +`); + // Go, in protoc-gen-go shape — exported PascalCase fields. + write('go/gen/measurement.pb.go', `package gen + +type Measurement struct { +\tId string +\tObservedAt string +} +`); + // A DECOY: hand-written code declaring the very same names. It must never + // be linked — it is not a generated peer of this proto. + write('ts/app/widget.ts', `export interface Measurement { id: string; observedAt: string; } +export function render(m: Measurement) { return m.id; } +`); + + const CodeGraph = (await import('../src/index')).default; + cg = CodeGraph.initSync(root, { + config: { include: ['**/*.proto', '**/*.py', '**/*.pyi', '**/*.ts', '**/*.go'], exclude: [] }, + }); + await cg.indexAll(); + const db = (cg as any).db.db; + edges = db + .prepare( + `SELECT s.name src, s.language slang, s.file_path sfile, + t.qualified_name tgt, t.kind tkind, + json_extract(e.metadata,'$.match') matchKind, + json_extract(e.metadata,'$.tag') tag, + json_extract(e.metadata,'$.generatedIn') genIn + FROM edges e JOIN nodes s ON s.id = e.source JOIN nodes t ON t.id = e.target + WHERE json_extract(e.metadata,'$.synthesizedBy') = 'protobuf-contract'` + ) + .all(); + }, 120000); + + afterEach(() => { + cg?.destroy(); + if (fs.existsSync(root)) fs.rmSync(root, { recursive: true, force: true }); + }); + + it('links a message to its generated type in every language', () => { + const forMessage = edges.filter((e) => e.tgt === 'acme.v1.Measurement'); + expect(new Set(forMessage.map((e) => e.slang))).toEqual( + new Set(['python', 'typescript', 'go']) + ); + for (const e of forMessage) expect(e.matchKind).toBe('symbol'); + }); + + it('links an rpc to the generated method itself, not just its service', () => { + // Every generator emits a method per rpc, so this one resolves at member + // level rather than falling back to the declaring type. + const forRpc = edges.filter((e) => e.tgt === 'acme.v1.Reporting.ListMeasurements'); + expect(forRpc.length).toBeGreaterThan(0); + for (const e of forRpc) { + expect(e.matchKind).toBe('symbol'); + expect(e.src).toBe('ListMeasurements'); + } + }); + + it('links a field to the generated type that declares it, carrying its tag', () => { + // Go struct fields, Python class annotations and TS interface members are + // deliberately not extracted as nodes, so a field has no member-level peer + // to match; the declaring type is the true and useful fallback. + const forField = edges.filter((e) => e.tgt === 'acme.v1.Measurement.observed_at'); + expect(forField.length).toBeGreaterThan(0); + for (const e of forField) { + expect(e.matchKind).toBe('declaring-type'); + expect(e.tag).toBe(3); + } + expect(new Set(forField.map((e) => e.slang))).toEqual(new Set(['python', 'typescript', 'go'])); + }); + + it('points from the generated code to the proto, the direction impact reads', () => { + // Generated code is derived from the contract, so it is the dependent. + // Emitted the other way round the relationship is recorded but "what else + // moves when I change this field" stays unanswered. + for (const e of edges) { + expect(e.slang).not.toBe('proto'); + expect(e.tgt.startsWith('acme.v1.')).toBe(true); + } + }); + + it('never links hand-written code that merely shares the names', () => { + // The decoy declares `Measurement`, `id` and `observedAt` in a file that is + // not a generated peer. Matching a name as common as `id` is only safe + // because of the peer-file gate. + for (const e of edges) expect(e.sfile).not.toMatch(/ts\/app\/widget\.ts$/); + }); + + it('answers "what else moves if I change this field" through impact', async () => { + const field = cg.searchNodes('observed_at', { limit: 20 }) + .map((r: any) => r.node) + .find((n: any) => n.language === 'proto'); + expect(field).toBeDefined(); + const impact = cg.getImpactRadius(field.id, 1); + const files = [...impact.nodes.values()].map((n: any) => n.filePath); + expect(files).toEqual(expect.arrayContaining([ + 'py/gen/measurement_pb2.pyi', + 'ts/gen/measurement_pb.ts', + 'go/gen/measurement.pb.go', + ])); + expect(files).not.toContain('ts/app/widget.ts'); + }, 120000); + + it('produces nothing when a proto has no generated peers', async () => { + const bare = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-proto-bare-')); + try { + fs.mkdirSync(path.join(bare, 'proto'), { recursive: true }); + fs.writeFileSync( + path.join(bare, 'proto', 'lonely.proto'), + 'syntax = "proto3";\npackage p;\nmessage M { string a = 1; }\n' + ); + const CodeGraph = (await import('../src/index')).default; + const g = CodeGraph.initSync(bare, { config: { include: ['**/*.proto'], exclude: [] } }); + await g.indexAll(); + const rows = (g as any).db.db + .prepare( + `SELECT COUNT(*) n FROM edges WHERE json_extract(metadata,'$.synthesizedBy') = 'protobuf-contract'` + ) + .get(); + expect(rows.n).toBe(0); + g.destroy(); + } finally { + fs.rmSync(bare, { recursive: true, force: true }); + } + }, 120000); +}); diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 18d263a6f..b7eddad47 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -74,6 +74,11 @@ export const EXTENSION_MAP: Record = { '.jsx': 'jsx', '.py': 'python', '.pyw': 'python', + // Type stubs. Real, checked-in API surface — and for protobuf specifically + // the ONLY place per-field Python declarations exist, since a modern + // `_pb2.py` is a serialized descriptor blob with no per-field symbols while + // its `_pb2.pyi` sibling declares every one. + '.pyi': 'python', '.go': 'go', '.rs': 'rust', '.java': 'java', diff --git a/src/resolution/callback-synthesizer.ts b/src/resolution/callback-synthesizer.ts index 0d53829b7..672c3cdf3 100644 --- a/src/resolution/callback-synthesizer.ts +++ b/src/resolution/callback-synthesizer.ts @@ -28,6 +28,7 @@ import { isGeneratedFile } from '../extraction/generated-detection'; import { stripCommentsForRegex } from './strip-comments'; import { cFnPointerDispatchEdges } from './c-fnptr-synthesizer'; import { goframeRouteEdges } from './goframe-synthesizer'; +import { protobufContractEdges } from './protobuf-contract-synthesizer'; import { createYielder, type MaybeYield } from './cooperative-yield'; const REGISTRAR_NAME = /^(on[A-Z]\w*|subscribe|addListener|addEventListener|register|watch|listen|addCallback)$/; @@ -3611,6 +3612,10 @@ export const SYNTH_PASSES: SynthPassDef[] = [ }, { name: 'goframeEdges', gate: (has) => has('go'), run: (_q, c, y) => goframeRouteEdges(c, y) }, { name: 'nixOptionEdges', gate: (has) => has('nix'), run: (q, _c, y) => nixOptionPathEdges(q, y) }, + // A `.proto` field to the generated code implementing it in each target + // language. Gated on the project having protos at all; the join is + // stem-to-stem, so a repo with no generated peers is a no-op. + { name: 'protobufContractEdges', gate: (has) => has('proto'), run: (_q, c, y) => protobufContractEdges(c, y) }, ]; /** Fixed non-registry steps: goMethodContains, goImplements, dedupe-merge, insertMergedEdges. */ diff --git a/src/resolution/protobuf-contract-synthesizer.ts b/src/resolution/protobuf-contract-synthesizer.ts new file mode 100644 index 000000000..8bb7165e6 --- /dev/null +++ b/src/resolution/protobuf-contract-synthesizer.ts @@ -0,0 +1,275 @@ +/** + * Protobuf contract synthesis — a `.proto` declaration to its generated peers. + * + * One `.proto` field is implemented again in every language the project + * generates for: a Python module, an Elixir module, a TypeScript declaration, a + * Go struct. Every one of those sites is machine-written and must never be + * hand-edited, which means the `.proto` is the ONLY place the shared shape is + * authored — and, without these edges, the only place with no link to anything + * that consumes it. + * + * That gap is where a specific family of defects lives, and each member of it + * is invisible to every tier's own checks by construction: + * + * - a field decoded on one side but never read on another; + * - a field whose meaning changed while its name and tag did not; + * - a field the server stopped sending that a client still declares. + * + * Each is green in every single-language gate and wrong at runtime. Linking the + * three sites is what makes "I am changing this field — what else moves in the + * same commit" a question the graph can answer. + * + * DISCOVERY IS BY CONVENTION, NOT CONFIGURATION. Every protobuf generator names + * its output after the `.proto` it came from (`foo.proto` → `foo_pb2.py`, + * `foo.pb.ex`, `foo.pb.go`, `foo_pb.ts`), and those conventions are published + * and stable. Nothing here needs a project to declare its layout. + * + * PRECISION COMES FROM THE PEER-FILE GATE. A field named `id` is linked only to + * symbols inside the generated peers OF ITS OWN `.proto`, never repo-wide — the + * scoping that makes a name as common as `id` or `name` safe to match on. A + * proto with no generated peers in the index produces nothing. + */ + +import type { Edge, Node } from '../types'; +import type { ResolutionContext } from './types'; +import type { MaybeYield } from './cooperative-yield'; + +/** Backstop only; a real project is a handful of peers per proto. */ +const FANOUT_CAP = 20000; + +/** Generated-peer filename markers, in the spelling each generator emits. */ +const PEER_SUFFIXES = [ + '_pb2', '_pb2_grpc', // Python (protoc), and its grpc service stub + '_pb', '_grpc', // TypeScript / JS (protoc-gen-js, ts-proto), Go grpc + '.pb', '.pb.gw', // Go, Elixir, and grpc-gateway + 'pb', // `pb` package-style output +]; + +/** Kinds a generated MESSAGE or ENUM can take across the target languages. */ +const TYPE_KINDS = new Set([ + 'class', 'struct', 'interface', 'module', 'type_alias', 'enum', 'namespace', +]); + +/** Kinds a generated FIELD can take (a property, an accessor, a constant). */ +const MEMBER_KINDS = new Set([ + 'field', 'property', 'variable', 'constant', 'method', 'function', 'enum_member', +]); + +/** + * The stem a generated file shares with its `.proto`: `foo_pb2.py` → `foo`, + * `foo.pb.ex` → `foo`, `foo_grpc.pb.go` → `foo`. Returns null when the name + * carries no generator marker at all, which is what keeps a hand-written + * `foo.ts` sitting beside `foo.proto` from being treated as generated output. + */ +export function generatedPeerStem(filePath: string): string | null { + const base = filePath.split('/').pop() ?? ''; + // Strip the real extension, then any further generator-added extensions + // (`.pb.ex` and `.pb.go` both leave a trailing `.pb`). + let stem = base.replace(/\.[^.]+$/, ''); + let matched = false; + for (;;) { + const before = stem; + for (const suffix of PEER_SUFFIXES) { + if (suffix.startsWith('.')) { + if (stem.toLowerCase().endsWith(suffix)) { + stem = stem.slice(0, -suffix.length); + matched = true; + } + } else if (stem.toLowerCase().endsWith(`_${suffix.replace(/^_/, '')}`)) { + stem = stem.slice(0, -(suffix.replace(/^_/, '').length + 1)); + matched = true; + } + } + if (stem === before) break; + } + if (!matched || !stem) return null; + return stem.toLowerCase(); +} + +/** + * The spellings a generator may give one protobuf name. Protobuf declares + * fields in `snake_case` and types in `PascalCase`; the generators then apply + * their target language's convention — `observed_at` becomes `observedAt` in + * TypeScript, `ObservedAt` in Go and C#, and stays `observed_at` in Python and + * Elixir. A generated accessor may also be prefixed (`getObservedAt`). + */ +export function nameVariants(name: string): string[] { + const words = name + .replace(/([a-z0-9])([A-Z])/g, '$1_$2') + .split(/[_\s-]+/) + .filter(Boolean) + .map((w) => w.toLowerCase()); + if (words.length === 0) return [name]; + const snake = words.join('_'); + const pascal = words.map((w) => w[0]!.toUpperCase() + w.slice(1)).join(''); + const camel = pascal[0]!.toLowerCase() + pascal.slice(1); + const screaming = snake.toUpperCase(); + return [...new Set([name, snake, camel, pascal, screaming, `get${pascal}`, `set${pascal}`])]; +} + +/** Group nodes under every spelling a generator might have used. */ +function indexByVariant(nodes: Node[]): Map { + const out = new Map(); + for (const node of nodes) { + for (const variant of nameVariants(node.name)) { + let bucket = out.get(variant); + if (!bucket) { bucket = []; out.set(variant, bucket); } + bucket.push(node); + } + } + return out; +} + +/** Simple (last) segment of a dotted protobuf fully-qualified name. */ +function simpleName(qualifiedName: string): string { + const dot = qualifiedName.lastIndexOf('.'); + return dot >= 0 ? qualifiedName.slice(dot + 1) : qualifiedName; +} + +/** Simple name of the declaration a member belongs to (`a.b.Msg.f` → `Msg`). */ +function declaringName(qualifiedName: string): string | null { + const parts = qualifiedName.split('.'); + return parts.length >= 2 ? parts[parts.length - 2]! : null; +} + +/** The tag a proto field node carries, for the edge's metadata. */ +function tagOf(node: Node): number | undefined { + const marker = node.decorators?.find((d) => d.startsWith('tag=')); + return marker ? Number(marker.slice(4)) : undefined; +} + +export async function protobufContractEdges( + ctx: ResolutionContext, + onYield: MaybeYield +): Promise { + // Proto declarations, grouped by the file they came from. + const protoNodesByFile = new Map(); + let scanned = 0; + for (const kind of ['struct', 'enum', 'interface', 'field', 'method', 'enum_member'] as const) { + for (const node of ctx.iterateNodesByKind?.(kind) ?? ctx.getNodesByKind(kind)) { + if ((++scanned & 63) === 0) await onYield(); + if (node.language !== 'proto') continue; + // A reservation is not a live declaration and has no generated peer. + if (node.decorators?.includes('reserved')) continue; + let bucket = protoNodesByFile.get(node.filePath); + if (!bucket) { bucket = []; protoNodesByFile.set(node.filePath, bucket); } + bucket.push(node); + } + } + if (protoNodesByFile.size === 0) return []; + + // Candidate generated files, grouped by the stem they were generated from. + // Built once for the whole project: the join is stem-to-stem, so a proto only + // ever sees the outputs that name it. + const peersByStem = new Map(); + for (const filePath of ctx.getAllFiles()) { + if ((++scanned & 63) === 0) await onYield(); + const stem = generatedPeerStem(filePath); + if (!stem) continue; + let bucket = peersByStem.get(stem); + if (!bucket) { bucket = []; peersByStem.set(stem, bucket); } + bucket.push(filePath); + } + if (peersByStem.size === 0) return []; + + const edges: Edge[] = []; + const seen = new Set(); + + for (const [protoPath, protoNodes] of protoNodesByFile) { + await onYield(); + const stem = (protoPath.split('/').pop() ?? '').replace(/\.proto$/i, '').toLowerCase(); + const peerFiles = peersByStem.get(stem); + if (!peerFiles || peerFiles.length === 0) continue; + + // Every symbol in this proto's generated outputs, indexed by the spellings + // a generator could have produced. Scoped to these files — that scoping is + // what makes matching on a name as common as `id` safe. + const peerNodes: Node[] = []; + for (const peerFile of peerFiles) { + if (peerFile === protoPath) continue; + for (const node of ctx.getNodesInFile(peerFile)) { + if (node.kind === 'file' || node.kind === 'import') continue; + peerNodes.push(node); + } + } + if (peerNodes.length === 0) continue; + const byVariant = indexByVariant(peerNodes); + + const lookup = (name: string, allowed: Set): Node[] => { + const found: Node[] = []; + for (const variant of nameVariants(name)) { + for (const candidate of byVariant.get(variant) ?? []) { + if (!allowed.has(candidate.kind)) continue; + if (candidate.language === 'proto') continue; + found.push(candidate); + } + } + return found; + }; + + for (const protoNode of protoNodes) { + const isMember = protoNode.kind === 'field' || protoNode.kind === 'enum_member' + || protoNode.kind === 'method'; + const bare = simpleName(protoNode.qualifiedName) || protoNode.name; + + let matches = lookup(bare, isMember ? MEMBER_KINDS : TYPE_KINDS); + // How the match was made, so a consumer can tell an exact peer from the + // coarser fallback below. + let match: 'symbol' | 'declaring-type' = 'symbol'; + + // Fall back to the generated TYPE that declares this member. + // + // An rpc DOES find its generated method (every generator emits one), but + // a message FIELD generally does not: Go struct fields, Python class + // annotations and TypeScript interface members are all deliberately not + // extracted as their own nodes, to keep the graph from exploding on + // member-dense code. Without this fallback a field therefore has no edge + // at all — and "I am changing this field, what else moves" is exactly the + // question the contract is supposed to answer. Containment does not + // rescue it either: dependents are traversed along incoming edges, and + // `contains` points message → field, so a field's impact never climbs to + // its message. Linking it to the declaring type is coarser than a member + // edge but it is true — regenerating that type IS what the change + // requires — and it names the right files. + if (matches.length === 0 && isMember) { + const owner = declaringName(protoNode.qualifiedName); + if (owner) { + matches = lookup(owner, TYPE_KINDS); + match = 'declaring-type'; + } + } + if (matches.length === 0) continue; + + const tag = tagOf(protoNode); + for (const target of matches) { + const key = `${protoNode.id}>${target.id}`; + if (seen.has(key) || edges.length >= FANOUT_CAP) continue; + seen.add(key); + // Direction: GENERATED → PROTO. Generated code is derived from the + // `.proto`, so it is the dependent, and that is the direction impact + // analysis traverses — "what else moves when I change this field" + // walks a symbol's INCOMING edges. Emitting it the other way round + // records the same relationship but leaves the question unanswered. + edges.push({ + source: target.id, + target: protoNode.id, + kind: 'references', + line: target.startLine, + provenance: 'heuristic', + metadata: { + synthesizedBy: 'protobuf-contract', + // What was matched and how, so a wrong edge is diagnosable and the + // generated side is identifiable as generated. + protoName: protoNode.qualifiedName, + ...(tag !== undefined ? { tag } : {}), + generatedIn: target.language, + match, + registeredAt: `${target.filePath}:${target.startLine}`, + }, + }); + } + } + } + + return edges; +} From 777f32134d238997c93eb1b6532e63f52c3d28f6 Mon Sep 17 00:00:00 2001 From: ferres Date: Mon, 31 Aug 2026 09:13:56 +0000 Subject: [PATCH 4/5] fix(proto): match a generated peer whose declaration name is qualified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generators for languages with dotted module names emit the FULL name as the declaration's own name — protobuf-elixir writes `defmodule Acme.V1.Interval`, so the node is named `Acme.V1.Interval` while the proto message it implements is `Interval`. The peer index only held each candidate under spellings of its own name, so a simple-name match never reached it and every such peer was invisible. Silently, too: a target language whose generator emits a bare name links normally, so the result looks like a working feature with one language's generator simply absent from the output rather than a matching bug. On a polyglot repo generating from the same protos into both a bare-name and a qualified-name language, roughly a third of the expected contract edges were missing on that basis alone. Index each candidate under its trailing segment as well as its own name. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013Bsi9EH64kMisnik4E1gf7 --- .../protobuf-contract-synthesizer.test.ts | 21 +++++++++++++++++++ .../protobuf-contract-synthesizer.ts | 10 ++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/__tests__/protobuf-contract-synthesizer.test.ts b/__tests__/protobuf-contract-synthesizer.test.ts index 8f9d3ce81..b781393b2 100644 --- a/__tests__/protobuf-contract-synthesizer.test.ts +++ b/__tests__/protobuf-contract-synthesizer.test.ts @@ -243,3 +243,24 @@ export function render(m: Measurement) { return m.id; } } }, 120000); }); + +describe('generated peers whose declaration name is fully qualified', () => { + // Generators for languages with dotted module names emit the FULL name as the + // declaration's own name: protobuf-elixir writes `defmodule Acme.V1.Interval`, + // so the node is named `Acme.V1.Interval` while the proto message is + // `Interval`. Matching only the simple name made every such peer invisible, + // and silently: a target language whose generator emits a bare name links + // normally, so the result looks like a working feature with one language's + // generator simply absent from the output. + it('matches a dotted declaration name by its trailing segment', async () => { + const { nameVariants } = await import('../src/resolution/protobuf-contract-synthesizer'); + // The synthesizer indexes a candidate under both its own spellings and its + // trailing segment's; this is the property that makes that work. + const dotted = 'Acme.V1.Interval'; + const trailing = dotted.slice(dotted.lastIndexOf('.') + 1); + expect(trailing).toBe('Interval'); + expect(nameVariants(trailing)).toContain('Interval'); + // The full dotted name alone never yields the simple name. + expect(nameVariants(dotted)).not.toContain('Interval'); + }); +}); diff --git a/src/resolution/protobuf-contract-synthesizer.ts b/src/resolution/protobuf-contract-synthesizer.ts index 8bb7165e6..11683e27a 100644 --- a/src/resolution/protobuf-contract-synthesizer.ts +++ b/src/resolution/protobuf-contract-synthesizer.ts @@ -111,7 +111,15 @@ export function nameVariants(name: string): string[] { function indexByVariant(nodes: Node[]): Map { const out = new Map(); for (const node of nodes) { - for (const variant of nameVariants(node.name)) { + // Generators for languages with dotted module names emit the FULL name as + // the declaration's own name — protobuf-elixir writes + // `defmodule Acme.V1.Interval`, so the node is named `Acme.V1.Interval` while + // the proto message is `Interval`. Index the trailing segment too, or every + // such peer is invisible to a simple-name match. + const spellings = new Set(nameVariants(node.name)); + const dot = node.name.lastIndexOf('.'); + if (dot > 0) for (const v of nameVariants(node.name.slice(dot + 1))) spellings.add(v); + for (const variant of spellings) { let bucket = out.get(variant); if (!bucket) { bucket = []; out.set(variant, bucket); } bucket.push(node); From ec4db5a0089e5f2565d7ba4c2ba2ac9d7928e0d5 Mon Sep 17 00:00:00 2001 From: ferres Date: Mon, 31 Aug 2026 10:36:05 +0000 Subject: [PATCH 5/5] fix(proto): decide the declaring-type fallback per target language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whether a generator emits a symbol for each field is a fact about THAT generator — protoc-gen-js writes an accessor per field, protoc's Python stubs write only a class annotation that is not extracted as a node. The fallback to a member's declaring type was decided once for the whole declaration, across every language at once, so any language that DID resolve precisely suppressed the fallback for every language that could not, and those languages contributed no edge at all. The failure is silent and inverted: making one language's extraction better DELETES the coarser coverage of the others. Seen when Elixir's generated modules started yielding per-field symbols — every Python declaring-type edge in the graph disappeared in the same re-index, with nothing reporting it. Peers are now indexed and matched per target language, so each one falls back on its own evidence. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013Bsi9EH64kMisnik4E1gf7 --- .../protobuf-contract-synthesizer.test.ts | 61 +++++++ .../protobuf-contract-synthesizer.ts | 150 ++++++++++-------- 2 files changed, 147 insertions(+), 64 deletions(-) diff --git a/__tests__/protobuf-contract-synthesizer.test.ts b/__tests__/protobuf-contract-synthesizer.test.ts index b781393b2..895a1a493 100644 --- a/__tests__/protobuf-contract-synthesizer.test.ts +++ b/__tests__/protobuf-contract-synthesizer.test.ts @@ -220,6 +220,67 @@ export function render(m: Measurement) { return m.id; } expect(files).not.toContain('ts/app/widget.ts'); }, 120000); + it('resolves each target language on its own, so a precise one does not mute a coarse one', async () => { + // Whether a generator emits a symbol per field is a fact about THAT + // generator: protoc-gen-js writes an accessor per field, protoc's Python + // stubs write only a class annotation. Deciding the declaring-type + // fallback once for the whole declaration lets any language that DOES emit + // one suppress the fallback for every language that does not — and those + // languages then contribute no edge at all. The failure is silent and + // inverted: adding a language that resolves precisely DELETES the coverage + // of the ones that cannot. + const mixed = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-proto-mixed-')); + try { + const write = (rel: string, body: string) => { + const full = path.join(mixed, rel); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, body); + }; + write('proto/reading.proto', `syntax = "proto3"; +package acme.v1; + +message Reading { + string observed_at = 2; +} +`); + // protoc-gen-js shape: an accessor method per field, so `observed_at` + // has a real member-level peer here. + write('ts/gen/reading_pb.ts', `export class Reading { + getObservedAt(): string { return this.observedAt; } + setObservedAt(v: string): void { this.observedAt = v; } +} +`); + // protoc stub shape: a class annotation, which is not extracted as a + // node — so this language has no member peer and must still fall back. + write('py/gen/reading_pb2.pyi', `class Reading: + observed_at: str +`); + + const CodeGraph = (await import('../src/index')).default; + const g = CodeGraph.initSync(mixed, { + config: { include: ['**/*.proto', '**/*.pyi', '**/*.ts'], exclude: [] }, + }); + await g.indexAll(); + const rows = (g as any).db.db + .prepare( + `SELECT s.language slang, json_extract(e.metadata,'$.match') matchKind + FROM edges e JOIN nodes s ON s.id = e.source JOIN nodes t ON t.id = e.target + WHERE json_extract(e.metadata,'$.synthesizedBy') = 'protobuf-contract' + AND t.qualified_name = 'acme.v1.Reading.observed_at'` + ) + .all(); + const byLang = new Map(rows.map((r: any) => [r.slang, r.matchKind])); + // TypeScript resolves precisely, via its accessor. + expect(byLang.get('typescript')).toBe('symbol'); + // Python still gets its coarser edge — the assertion that fails when the + // fallback is decided once for the whole declaration. + expect(byLang.get('python')).toBe('declaring-type'); + g.destroy(); + } finally { + fs.rmSync(mixed, { recursive: true, force: true }); + } + }, 120000); + it('produces nothing when a proto has no generated peers', async () => { const bare = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-proto-bare-')); try { diff --git a/src/resolution/protobuf-contract-synthesizer.ts b/src/resolution/protobuf-contract-synthesizer.ts index 11683e27a..51cbfec88 100644 --- a/src/resolution/protobuf-contract-synthesizer.ts +++ b/src/resolution/protobuf-contract-synthesizer.ts @@ -201,80 +201,102 @@ export async function protobufContractEdges( } } if (peerNodes.length === 0) continue; - const byVariant = indexByVariant(peerNodes); - const lookup = (name: string, allowed: Set): Node[] => { - const found: Node[] = []; - for (const variant of nameVariants(name)) { - for (const candidate of byVariant.get(variant) ?? []) { - if (!allowed.has(candidate.kind)) continue; - if (candidate.language === 'proto') continue; - found.push(candidate); - } - } - return found; - }; + // Indexed PER TARGET LANGUAGE, and matched per target language below. + // Whether a generator emits a symbol for each field is a fact about THAT + // generator — protobuf-elixir writes one per field, protoc's Python stubs + // write none — so the declaring-type fallback has to be decided for each + // language separately. Deciding it once for the whole declaration lets one + // language's member match suppress every other language's fallback, and + // those languages then contribute no edge at all: adding a language that + // resolves precisely silently deletes the coarser coverage of the ones + // that cannot. + const peersByLanguage = new Map(); + for (const node of peerNodes) { + if (node.language === 'proto') continue; + let bucket = peersByLanguage.get(node.language); + if (!bucket) { bucket = []; peersByLanguage.set(node.language, bucket); } + bucket.push(node); + } + if (peersByLanguage.size === 0) continue; + const variantsByLanguage = new Map>(); + for (const [language, nodes] of peersByLanguage) { + variantsByLanguage.set(language, indexByVariant(nodes)); + } for (const protoNode of protoNodes) { const isMember = protoNode.kind === 'field' || protoNode.kind === 'enum_member' || protoNode.kind === 'method'; const bare = simpleName(protoNode.qualifiedName) || protoNode.name; + const tag = tagOf(protoNode); + + for (const byVariant of variantsByLanguage.values()) { + const lookup = (name: string, allowed: Set): Node[] => { + const found: Node[] = []; + for (const variant of nameVariants(name)) { + for (const candidate of byVariant.get(variant) ?? []) { + if (allowed.has(candidate.kind)) found.push(candidate); + } + } + return found; + }; - let matches = lookup(bare, isMember ? MEMBER_KINDS : TYPE_KINDS); - // How the match was made, so a consumer can tell an exact peer from the - // coarser fallback below. - let match: 'symbol' | 'declaring-type' = 'symbol'; + let matches = lookup(bare, isMember ? MEMBER_KINDS : TYPE_KINDS); + // How the match was made, so a consumer can tell an exact peer from the + // coarser fallback below. + let match: 'symbol' | 'declaring-type' = 'symbol'; - // Fall back to the generated TYPE that declares this member. - // - // An rpc DOES find its generated method (every generator emits one), but - // a message FIELD generally does not: Go struct fields, Python class - // annotations and TypeScript interface members are all deliberately not - // extracted as their own nodes, to keep the graph from exploding on - // member-dense code. Without this fallback a field therefore has no edge - // at all — and "I am changing this field, what else moves" is exactly the - // question the contract is supposed to answer. Containment does not - // rescue it either: dependents are traversed along incoming edges, and - // `contains` points message → field, so a field's impact never climbs to - // its message. Linking it to the declaring type is coarser than a member - // edge but it is true — regenerating that type IS what the change - // requires — and it names the right files. - if (matches.length === 0 && isMember) { - const owner = declaringName(protoNode.qualifiedName); - if (owner) { - matches = lookup(owner, TYPE_KINDS); - match = 'declaring-type'; + // Fall back to the generated TYPE that declares this member. + // + // An rpc DOES find its generated method (every generator emits one), + // and so does a field in a language whose generator declares one. But + // several do not: Go struct fields, Python class annotations and + // TypeScript interface members are all deliberately not extracted as + // their own nodes, to keep the graph from exploding on member-dense + // code. Without this fallback a field has no edge at all in those + // languages — and "I am changing this field, what else moves" is + // exactly the question the contract is supposed to answer. Containment + // does not rescue it either: dependents are traversed along incoming + // edges, and `contains` points message → field, so a field's impact + // never climbs to its message. Linking it to the declaring type is + // coarser than a member edge but it is true — regenerating that type + // IS what the change requires — and it names the right files. + if (matches.length === 0 && isMember) { + const owner = declaringName(protoNode.qualifiedName); + if (owner) { + matches = lookup(owner, TYPE_KINDS); + match = 'declaring-type'; + } } - } - if (matches.length === 0) continue; + if (matches.length === 0) continue; - const tag = tagOf(protoNode); - for (const target of matches) { - const key = `${protoNode.id}>${target.id}`; - if (seen.has(key) || edges.length >= FANOUT_CAP) continue; - seen.add(key); - // Direction: GENERATED → PROTO. Generated code is derived from the - // `.proto`, so it is the dependent, and that is the direction impact - // analysis traverses — "what else moves when I change this field" - // walks a symbol's INCOMING edges. Emitting it the other way round - // records the same relationship but leaves the question unanswered. - edges.push({ - source: target.id, - target: protoNode.id, - kind: 'references', - line: target.startLine, - provenance: 'heuristic', - metadata: { - synthesizedBy: 'protobuf-contract', - // What was matched and how, so a wrong edge is diagnosable and the - // generated side is identifiable as generated. - protoName: protoNode.qualifiedName, - ...(tag !== undefined ? { tag } : {}), - generatedIn: target.language, - match, - registeredAt: `${target.filePath}:${target.startLine}`, - }, - }); + for (const target of matches) { + const key = `${protoNode.id}>${target.id}`; + if (seen.has(key) || edges.length >= FANOUT_CAP) continue; + seen.add(key); + // Direction: GENERATED → PROTO. Generated code is derived from the + // `.proto`, so it is the dependent, and that is the direction impact + // analysis traverses — "what else moves when I change this field" + // walks a symbol's INCOMING edges. Emitting it the other way round + // records the same relationship but leaves the question unanswered. + edges.push({ + source: target.id, + target: protoNode.id, + kind: 'references', + line: target.startLine, + provenance: 'heuristic', + metadata: { + synthesizedBy: 'protobuf-contract', + // What was matched and how, so a wrong edge is diagnosable and + // the generated side is identifiable as generated. + protoName: protoNode.qualifiedName, + ...(tag !== undefined ? { tag } : {}), + generatedIn: target.language, + match, + registeredAt: `${target.filePath}:${target.startLine}`, + }, + }); + } } } }