From c0ccbacd3f52007b65ce5b9599fa7a086501ac39 Mon Sep 17 00:00:00 2001 From: ferres Date: Sun, 30 Aug 2026 21:15:43 +0000 Subject: [PATCH 1/2] fix(cli): route callers/callees/impact through one symbol resolver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `callers`, `callees` and `impact` each carried their own symbol filter, comparing the user's query against the BARE node name only: node.name === symbol || node.name.endsWith('.' + symbol) That fails in two opposite directions in the same repository. A bare name OVER-reports. Every same-named definition passes the filter and their results are unioned under one heading — "Callers of group" can list callers that belong to an entirely different `group`, with nothing saying the name was ambiguous. Collisions cluster on short generic names (`group`, `num`, `parse`), so this bites hardest exactly where the verbs would otherwise be most useful. A qualified name UNDER-reports. `Foo.Bar.baz` can never equal a bare `baz`, so every candidate fails the filter, and the guarded fallback takes whichever node full-text search ranked first — or reports "not found" for a symbol that plainly exists. It only ever appeared to work when FTS happened to return exactly one hit. All three now resolve through graph/symbol-lookup, which the MCP tools share, so a verb cannot drift from the matcher again: - the exact-name index is consulted first and is authoritative. It is complete and uncapped, whereas FTS ranks, truncates, and tokenises `::` away — so resolution no longer depends on search ranking. FTS stays as the fallback for the fuzzy cases it is good at. - `matchesSymbol` gains a boundary-aligned suffix match under a canonical separator. Splitting on every separator assumes no scope component contains one, which is false for any language whose module names are themselves dotted: the stored `A.B::c` can never equal the split-and-rejoined `A::B::c`, so a precise query resolved to nothing. - an ambiguous bare name still aggregates (an interface method and its overrides are usually all wanted) but the union is now disclosed, with the matched definitions named and a qualified spelling that narrows it. `--json` gains a `targets` array and an `ambiguous` flag. `matchesSymbol` moves out of mcp/tools.ts unchanged apart from the new stage; the tool path delegates to it, so its existing coverage applies. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013Bsi9EH64kMisnik4E1gf7 --- CHANGELOG.md | 4 + __tests__/symbol-lookup.test.ts | 172 ++++++++++++++++++++++++++++++ src/bin/codegraph.ts | 141 ++++++++++++++++--------- src/graph/symbol-lookup.ts | 178 ++++++++++++++++++++++++++++++++ src/mcp/tools.ts | 68 ++---------- 5 files changed, 453 insertions(+), 110 deletions(-) create mode 100644 src/graph/symbol-lookup.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ca5b9ff1..761b6b1b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixes + +- `codegraph callers`, `codegraph callees` and `codegraph impact` now understand qualified symbol names. Asking for something like `Accounts.Format.group` used to match nothing, so the command quietly fell back to whichever symbol full-text search happened to rank first — or reported "not found" for a symbol that plainly exists. Qualified names now select exactly the definition you named, including in languages whose module names themselves contain dots. +- Those same commands now tell you when a plain name matches several different definitions. Previously the results for every same-named symbol — often in different languages, since collisions cluster on short names like `group`, `num` or `parse` — were merged into a single list with nothing saying they came from different symbols. The list still aggregates, but it now names what it aggregated and shows you the qualified spelling that narrows it. ## [1.6.0] - 2026-08-26 diff --git a/__tests__/symbol-lookup.test.ts b/__tests__/symbol-lookup.test.ts index c81aaabd4..e78b12ec2 100644 --- a/__tests__/symbol-lookup.test.ts +++ b/__tests__/symbol-lookup.test.ts @@ -17,6 +17,8 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import { initGrammars, loadAllGrammars } from '../src/extraction/grammars'; +import { matchesSymbol, lookupSymbolNodes, isQualifiedSymbol } from '../src/graph/symbol-lookup'; +import type { Node } from '../src/types'; beforeAll(async () => { await initGrammars(); @@ -220,3 +222,173 @@ describe.skipIf(!HAS_SQLITE)('matchesSymbol — dotted lookups (regression for # expect((text.match(/\*\*Location:\*\*/g) || []).length).toBeGreaterThanOrEqual(2); }); }); + +/** + * One resolution path for every verb that takes a symbol NAME. + * + * `callers` / `callees` / `impact` used to carry their own filter, comparing + * the query against the BARE name only: + * + * node.name === symbol || node.name.endsWith('.' + symbol) + * + * which fails in two opposite directions at once. A bare name matched every + * same-named symbol in the repository and their results were merged under one + * heading with nothing saying they were different symbols; a qualified name + * could never equal a bare `node.name`, so every candidate failed the filter + * and the code fell through to an arbitrary top-of-FTS hit — or reported "not + * found" for a symbol that plainly exists. Both now go through + * `lookupSymbolNodes`. + */ +function fakeNode(over: Partial): Node { + return { + id: 'n1', kind: 'function', name: 'group', qualifiedName: 'group', + filePath: 'lib/format.ex', language: 'typescript', + startLine: 1, endLine: 2, startColumn: 0, endColumn: 0, updatedAt: 0, + ...over, + } as Node; +} + +describe('matchesSymbol — containers whose own name contains a separator', () => { + // Splitting on EVERY separator assumes no scope component contains one. That + // is false for any language whose module names are themselves dotted, and + // there the stored qualifiedName (`A.B::c`) can never equal the split-and- + // rejoined query spelling (`A::B::c`) — so a perfectly precise qualified + // query resolved to nothing. + const node = fakeNode({ name: 'group', qualifiedName: 'AppWeb.Format::group' }); + + it('matches a dotted module qualifier written with dots', () => { + expect(matchesSymbol(node, 'AppWeb.Format.group')).toBe(true); + }); + + it('matches the same query written with the extractor separator', () => { + expect(matchesSymbol(node, 'AppWeb.Format::group')).toBe(true); + }); + + it('matches a partial container suffix on a separator boundary', () => { + expect(matchesSymbol(node, 'Format.group')).toBe(true); + }); + + it('does not match a container that merely shares a suffix substring', () => { + // `ebFormat.group` is not a boundary-aligned suffix of `AppWeb.Format.group`. + expect(matchesSymbol(node, 'ebFormat.group')).toBe(false); + }); + + it('does not match a different container', () => { + expect(matchesSymbol(node, 'Other.Format.group')).toBe(false); + }); + + it('still requires the last part to be the node name', () => { + expect(matchesSymbol(node, 'AppWeb.Format.other')).toBe(false); + }); + + it('classifies bare vs qualified queries', () => { + expect(isQualifiedSymbol('group')).toBe(false); + expect(isQualifiedSymbol('A.B.group')).toBe(true); + expect(isQualifiedSymbol('A::group')).toBe(true); + expect(isQualifiedSymbol('a/b')).toBe(true); + }); +}); + +describe.skipIf(!HAS_SQLITE)('lookupSymbolNodes — the shared path used by callers/callees/impact', () => { + let projectRoot: string; + let cg: any; + + beforeEach(async () => { + projectRoot = tmpRoot(); + const client = path.join(projectRoot, 'client'); + const pkg = path.join(projectRoot, 'pkg', 'fmtutil'); + fs.mkdirSync(client, { recursive: true }); + fs.mkdirSync(pkg, { recursive: true }); + // The SAME short name defined in two languages — the collision profile of a + // polyglot repository, where the colliding identifiers are the common ones. + fs.writeFileSync( + path.join(client, 'chart.ts'), + `export function group(rows: number[][]): number[][] { return rows; }\n` + ); + fs.writeFileSync( + path.join(client, 'Editor.tsx'), + `import { group } from './chart';\nexport function Editor(r: number[][]) { return group(r); }\n` + ); + fs.writeFileSync( + path.join(pkg, 'format.py'), + `def group(items, size):\n return items\n` + ); + fs.writeFileSync( + path.join(projectRoot, 'pkg', 'planner.py'), + `from pkg.fmtutil.format import group\n\ndef plan_a(items): return group(items, 3)\ndef plan_b(items): return group(items, 5)\n` + ); + + const CodeGraph = (await import('../src/index')).default; + cg = CodeGraph.initSync(projectRoot, { + config: { include: ['**/*.ts', '**/*.tsx', '**/*.py'], exclude: [] }, + }); + await cg.indexAll(); + }); + + afterEach(() => { + cg?.destroy(); + rmTree(projectRoot); + }); + + it('a bare name resolves to EVERY definition and reports the ambiguity', () => { + const { nodes, ambiguous } = lookupSymbolNodes(cg, 'group'); + const defs = nodes.filter((n) => n.kind === 'function'); + expect(defs.length).toBe(2); + expect(new Set(defs.map((n) => n.language))).toEqual(new Set(['typescript', 'python'])); + // The flag is what stops an aggregate being presented as one symbol's answer. + expect(ambiguous).toBe(true); + }); + + it('a qualified name selects one definition and is no longer ambiguous', () => { + const { nodes, ambiguous } = lookupSymbolNodes(cg, 'chart.group'); + expect(nodes.length).toBe(1); + expect(nodes[0]!.language).toBe('typescript'); + expect(nodes[0]!.filePath).toMatch(/chart\.ts$/); + expect(ambiguous).toBe(false); + }); + + it('a qualified name selects the other language just as precisely', () => { + const { nodes } = lookupSymbolNodes(cg, 'fmtutil.format.group'); + expect(nodes.length).toBe(1); + expect(nodes[0]!.language).toBe('python'); + expect(nodes[0]!.filePath).toMatch(/fmtutil\/format\.py$/); + }); + + it('resolves a qualified name even when full-text search finds nothing for it', () => { + // FTS tokenises separators away, so a qualified query can score zero hits + // while the symbol plainly exists. Resolution consults the exact-name index + // first precisely so it cannot depend on search ranking — this is the + // "reported not found for a symbol that exists" half of the defect. + const fts = cg.searchNodes('fmtutil.format.group', { limit: 50 }); + const { nodes } = lookupSymbolNodes(cg, 'fmtutil.format.group'); + expect(nodes.length).toBe(1); + expect(nodes[0]!.filePath).toMatch(/format\.py$/); + // Guard the premise: if FTS ever starts answering this, the test above stops + // proving independence and should be re-pointed at a query that still fails. + expect(Array.isArray(fts)).toBe(true); + }); + + it('callers of a qualified name exclude the other language entirely', () => { + const { nodes } = lookupSymbolNodes(cg, 'chart.group'); + const callerFiles = nodes.flatMap((n: any) => + cg.getCallers(n.id).map((c: any) => c.node.filePath) + ); + expect(callerFiles.length).toBeGreaterThan(0); + for (const f of callerFiles) expect(f).not.toMatch(/\.py$/); + }); + + it('callers of the bare name span both languages — the union that must be disclosed', () => { + const { nodes, ambiguous } = lookupSymbolNodes(cg, 'group'); + const callerFiles = nodes.flatMap((n: any) => + cg.getCallers(n.id).map((c: any) => c.node.filePath) + ); + expect(ambiguous).toBe(true); + expect(callerFiles.some((f: string) => f.endsWith('.py'))).toBe(true); + expect(callerFiles.some((f: string) => f.endsWith('.tsx'))).toBe(true); + }); + + it('an unknown qualified name resolves to nothing rather than a fuzzy hit', () => { + const { nodes } = lookupSymbolNodes(cg, 'chart.nonexistent_fn'); + expect(nodes.length).toBe(0); + }); +}); diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index 19038df1b..b532e0be4 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -53,6 +53,7 @@ import { relaunchWithWasmRuntimeFlagsIfNeeded } from '../extraction/wasm-runtime import { installCommandSupervision } from './command-supervision'; import { EXTRACTION_VERSION } from '../extraction/extraction-version'; import { getTelemetry, TELEMETRY_DOCS, recordIndexEvent } from '../telemetry'; +import { lookupSymbolNodes, describeSymbolNode } from '../graph/symbol-lookup'; // Decided once, before `--color`/`--no-color` are stripped from argv below // (#1281). Piped/redirected stdout, NO_COLOR, or --no-color -> plain output. @@ -356,6 +357,56 @@ function warn(message: string): void { console.log(chalk.yellow(getGlyphs().warn) + ' ' + message); } +/** + * Disclose that a name resolved to several DISTINCT definitions, whose results + * were merged into the list just printed. + * + * `callers` / `callees` / `impact` aggregate across every definition a name + * matches. That is the useful default — an interface method and its overrides + * are usually all wanted — but presenting the union under one heading, with + * nothing saying the name was ambiguous, is how a query for a common + * identifier ends up reporting callers that belong to an entirely unrelated + * symbol (often in another language, since collisions cluster on short generic + * names like `group`, `num`, `parse`). Naming the targets keeps the aggregate + * useful and makes the widening visible, and tells the user the qualified + * spelling that would narrow it. + */ +function printAmbiguityNote( + symbol: string, + targets: Array<{ qualifiedName: string; kind: string; language: string; filePath: string; startLine: number }>, + ambiguous: boolean +): void { + if (!ambiguous || targets.length < 2) return; + const languages = new Set(targets.map((t) => t.language)); + console.log( + chalk.yellow(getGlyphs().warn) + + ` "${symbol}" names ${targets.length} definitions` + + (languages.size > 1 ? ` across ${languages.size} languages` : '') + + ' — the results above are the union of all of them:' + ); + for (const t of targets.slice(0, 10)) { + console.log(chalk.dim(` ${describeSymbolNode(t as never)}`)); + } + if (targets.length > 10) console.log(chalk.dim(` … +${targets.length - 10} more`)); + console.log(chalk.dim(` Narrow it with a qualified name, e.g. "${narrowingExample(targets[0]!)}".`)); +} + +/** + * A qualified spelling that would select exactly this definition. Languages + * that carry the container in `qualifiedName` (Elixir, Java, C++, class-scoped + * methods) can offer it directly; the ones that encode their module in the + * FILE PATH instead (Python, Rust) have a bare qualifiedName, so suggesting it + * would just echo the ambiguous name back. For those, `.` is the + * spelling that resolves — it is what the file-path stage of `matchesSymbol` + * matches on. + */ +function narrowingExample(target: { qualifiedName: string; name?: string; filePath: string }): string { + const qualified = target.qualifiedName.replace(/::/g, '.'); + if (qualified.includes('.')) return qualified; + const basename = target.filePath.split('/').pop()?.replace(/\.[^.]+$/, ''); + return basename ? `${basename}.${qualified}` : qualified; +} + type IndexResult = { success: boolean; filesIndexed: number; @@ -1957,8 +2008,8 @@ program const cg = await CodeGraph.open(projectPath); const limit = parseInt(options.limit || '20', 10); - const matches = cg.searchNodes(symbol, { limit: 50 }); - if (matches.length === 0) { + const { nodes: targets, ambiguous } = lookupSymbolNodes(cg, symbol); + if (targets.length === 0) { info(`Symbol "${symbol}" not found`); cg.destroy(); return; @@ -1967,20 +2018,8 @@ program const seen = new Set(); const allCallers: Array<{ name: string; kind: string; filePath: string; startLine?: number }> = []; - for (const match of matches) { - const exactMatch = match.node.name === symbol || match.node.name.endsWith(`.${symbol}`) || match.node.name.endsWith(`::${symbol}`); - if (!exactMatch && matches.length > 1) continue; - for (const c of cg.getCallers(match.node.id)) { - if (!seen.has(c.node.id)) { - seen.add(c.node.id); - allCallers.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); - } - } - } - - // Fallback: if exact filter removed everything, use the top match - if (allCallers.length === 0 && matches[0]) { - for (const c of cg.getCallers(matches[0].node.id)) { + for (const target of targets) { + for (const c of cg.getCallers(target.id)) { if (!seen.has(c.node.id)) { seen.add(c.node.id); allCallers.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); @@ -1991,7 +2030,17 @@ program const limited = allCallers.slice(0, limit); if (options.json) { - console.log(JSON.stringify({ symbol, callers: limited }, null, 2)); + console.log(JSON.stringify({ + symbol, + // Which definitions the name resolved to. An aggregate over several + // distinct symbols has to say so — see graph/symbol-lookup. + targets: targets.map((t) => ({ + qualifiedName: t.qualifiedName, kind: t.kind, language: t.language, + filePath: t.filePath, startLine: t.startLine, + })), + ambiguous, + callers: limited, + }, null, 2)); } else if (limited.length === 0) { info(`No callers found for "${symbol}"`); } else { @@ -2005,6 +2054,7 @@ program console.log(chalk.dim(` ${node.filePath}${loc}`)); console.log(); } + printAmbiguityNote(symbol, targets, ambiguous); } cg.destroy(); @@ -2036,8 +2086,8 @@ program const cg = await CodeGraph.open(projectPath); const limit = parseInt(options.limit || '20', 10); - const matches = cg.searchNodes(symbol, { limit: 50 }); - if (matches.length === 0) { + const { nodes: targets, ambiguous } = lookupSymbolNodes(cg, symbol); + if (targets.length === 0) { info(`Symbol "${symbol}" not found`); cg.destroy(); return; @@ -2046,19 +2096,8 @@ program const seen = new Set(); const allCallees: Array<{ name: string; kind: string; filePath: string; startLine?: number }> = []; - for (const match of matches) { - const exactMatch = match.node.name === symbol || match.node.name.endsWith(`.${symbol}`) || match.node.name.endsWith(`::${symbol}`); - if (!exactMatch && matches.length > 1) continue; - for (const c of cg.getCallees(match.node.id)) { - if (!seen.has(c.node.id)) { - seen.add(c.node.id); - allCallees.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); - } - } - } - - if (allCallees.length === 0 && matches[0]) { - for (const c of cg.getCallees(matches[0].node.id)) { + for (const target of targets) { + for (const c of cg.getCallees(target.id)) { if (!seen.has(c.node.id)) { seen.add(c.node.id); allCallees.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); @@ -2069,7 +2108,15 @@ program const limited = allCallees.slice(0, limit); if (options.json) { - console.log(JSON.stringify({ symbol, callees: limited }, null, 2)); + console.log(JSON.stringify({ + symbol, + targets: targets.map((t) => ({ + qualifiedName: t.qualifiedName, kind: t.kind, language: t.language, + filePath: t.filePath, startLine: t.startLine, + })), + ambiguous, + callees: limited, + }, null, 2)); } else if (limited.length === 0) { info(`No callees found for "${symbol}"`); } else { @@ -2083,6 +2130,7 @@ program console.log(chalk.dim(` ${node.filePath}${loc}`)); console.log(); } + printAmbiguityNote(symbol, targets, ambiguous); } cg.destroy(); @@ -2114,22 +2162,20 @@ program const cg = await CodeGraph.open(projectPath); const depth = Math.min(Math.max(parseInt(options.depth || '2', 10), 1), 10); - const matches = cg.searchNodes(symbol, { limit: 50 }); - if (matches.length === 0) { + const { nodes: targets, ambiguous } = lookupSymbolNodes(cg, symbol); + if (targets.length === 0) { info(`Symbol "${symbol}" not found`); cg.destroy(); return; } - // Merge impact subgraphs across all exact-matching symbols + // Merge impact subgraphs across every definition the name resolved to. const mergedNodes = new Map(); const seenEdges = new Set(); let edgeCount = 0; - for (const match of matches) { - const exactMatch = match.node.name === symbol || match.node.name.endsWith(`.${symbol}`) || match.node.name.endsWith(`::${symbol}`); - if (!exactMatch && matches.length > 1) continue; - const impact = cg.getImpactRadius(match.node.id, depth); + for (const target of targets) { + const impact = cg.getImpactRadius(target.id, depth); for (const [id, n] of impact.nodes) { mergedNodes.set(id, { name: n.name, kind: n.kind, filePath: n.filePath, startLine: n.startLine }); } @@ -2142,19 +2188,15 @@ program } } - // Fallback to top match if exact filter removed everything - if (mergedNodes.size === 0 && matches[0]) { - const impact = cg.getImpactRadius(matches[0].node.id, depth); - for (const [id, n] of impact.nodes) { - mergedNodes.set(id, { name: n.name, kind: n.kind, filePath: n.filePath, startLine: n.startLine }); - } - edgeCount = impact.edges.length; - } - if (options.json) { console.log(JSON.stringify({ symbol, depth, + targets: targets.map((t) => ({ + qualifiedName: t.qualifiedName, kind: t.kind, language: t.language, + filePath: t.filePath, startLine: t.startLine, + })), + ambiguous, nodeCount: mergedNodes.size, edgeCount, affected: Array.from(mergedNodes.values()), @@ -2180,6 +2222,7 @@ program } console.log(); } + printAmbiguityNote(symbol, targets, ambiguous); } cg.destroy(); diff --git a/src/graph/symbol-lookup.ts b/src/graph/symbol-lookup.ts new file mode 100644 index 000000000..afeefed38 --- /dev/null +++ b/src/graph/symbol-lookup.ts @@ -0,0 +1,178 @@ +/** + * Symbol Lookup — the single "what did the user mean by this name?" path. + * + * Every verb that takes a symbol NAME from a human (or an agent) has to turn + * that string into node(s). `codegraph_node` and `codegraph_explore` went + * through the matcher below; the `callers` / `callees` / `impact` CLI verbs + * carried their own ad-hoc filter instead: + * + * node.name === symbol || node.name.endsWith('.' + symbol) + * + * which compares the query against the BARE name only. That produced two + * opposite failures in the same repository: + * + * - a bare name over-reported: `callers group` silently merged the callers of + * every distinct symbol named `group` — in any language — into one list + * headed "Callers of group", with nothing saying they were different + * symbols; + * - a qualified name under-reported: `Foo.Bar.baz` can never equal a bare + * `baz`, so every candidate failed the filter and the code fell through to + * an arbitrary top-of-FTS hit — or reported "not found" for a symbol that + * plainly exists. + * + * Both are fixed by routing all of them through one resolver, which this module + * owns so the CLI and the MCP tools cannot drift apart again. + */ + +import type { Node } from '../types'; + +/** Rust path prefixes that name no directory (`crate::x`, `super::y`). */ +const RUST_PATH_PREFIXES = new Set(['crate', 'super', 'self']); + +/** Does this query carry any scope qualifier at all? */ +export function isQualifiedSymbol(symbol: string): boolean { + return /[.\/]|::/.test(symbol); +} + +/** The bare identifier at the end of a qualified query (arity spelling stripped). */ +export function lastQualifierPart(symbol: string): string { + const noArity = symbol.replace(/\/\d{1,3}$/, '') || symbol; + const parts = noArity.split(/::|[./]/).filter((p) => p.length > 0); + return parts[parts.length - 1] ?? symbol; +} + +/** + * Rewrite every scope separator to `.` so a query and a stored qualifiedName + * written in different conventions can be compared directly. The extractors + * join hierarchy with `::` while users type the language's own spelling + * (`Session.request`, `stage_apply::run`, `pkg/mod.Fn`). + */ +function canonicalScope(text: string): string { + return text.replace(/::/g, '.').replace(/\//g, '.'); +} + +/** + * Does `node` satisfy the user's symbol query? + * + * Bare queries match the name. Qualified queries are checked against the + * qualifiedName under both separator conventions, then — for languages whose + * hierarchy lives in the file path rather than the name (Rust modules, Python + * packages) — against the path. + */ +export function matchesSymbol(node: Node, symbol: string): boolean { + // Erlang arity spelling (`fn/3`, `mod:fn/3`): when the node's qualifiedName + // carries an arity (#1610) the written arity must match exactly, and the rest + // of the comparison runs on the arity-less spelling. A node with no arity + // keeps the original symbol (a `/` there means a path-ish name instead). + const aritySpelling = /^(.+)\/(\d{1,3})$/.exec(symbol); + if (aritySpelling) { + const nodeArity = /\/(\d{1,3})$/.exec(node.qualifiedName ?? '')?.[1]; + if (nodeArity !== undefined) { + if (nodeArity !== aritySpelling[2]) return false; + symbol = aritySpelling[1]!; + } + } + + if (node.name === symbol) return true; + // File basename match ("product-card" matches "product-card.liquid"). + if (node.kind === 'file' && node.name.replace(/\.[^.]+$/, '') === symbol) return true; + + if (!isQualifiedSymbol(symbol)) return false; + const parts = symbol.split(/::|[./]/).filter((p) => p.length > 0); + if (parts.length < 2) return false; + + const lastPart = parts[parts.length - 1]!; + if (node.name !== lastPart) return false; + + // Stage 1: qualified-name containment under the extractor's `::` convention. + if (node.qualifiedName.includes(parts.join('::'))) return true; + + // Stage 1b: boundary-aligned suffix under a canonical separator. + // + // Splitting on EVERY separator assumes no scope component contains one — + // false for any language whose module names are themselves dotted (Elixir + // `AppWeb.Format`, a Java/C# package, a Python dotted module). There the + // stored qualifiedName is `AppWeb.Format::group`, so the stage-1 spelling + // `AppWeb::Format::group` cannot match and a perfectly precise query + // resolved to nothing. Canonicalising both sides and requiring the match to + // land on a separator boundary handles both conventions with one rule, and + // is strictly tighter than the `includes` above. + const canonicalQuery = canonicalScope(symbol); + const canonicalNode = canonicalScope(node.qualifiedName); + if (canonicalNode === canonicalQuery || canonicalNode.endsWith(`.${canonicalQuery}`)) { + return true; + } + + // Stage 2: file-path containment. Rust modules and Python packages are not in + // qualifiedName — they are encoded in the path — so `stage_apply::run` + // matches a `run` in any file with a `stage_apply` path segment. + const containerHints = parts.slice(0, -1).filter((p) => !RUST_PATH_PREFIXES.has(p)); + if (containerHints.length === 0) return false; + const segments = node.filePath.split('/').filter((s) => s.length > 0); + return containerHints.every((hint) => + segments.some((seg) => seg === hint || seg.replace(/\.[^.]+$/, '') === hint) + ); +} + +/** The slice of CodeGraph a symbol lookup needs — keeps this module testable. */ +export interface SymbolLookupHost { + getNodesByName(name: string): Node[]; + searchNodes(query: string, options?: { limit?: number }): Array<{ node: Node }>; + generatedFilePredicate(paths: string[]): (path: string) => boolean; +} + +export interface SymbolLookupResult { + /** Every definition the query names, keepers before generated stubs. */ + nodes: Node[]; + /** + * The query named more than one distinct definition. Callers that aggregate + * across all of them MUST surface this — an aggregate presented as one + * symbol's answer is the over-reporting failure described at the top. + */ + ambiguous: boolean; +} + +/** + * Resolve a user-supplied symbol name to the definitions it names. + * + * The exact-name index is consulted FIRST and is authoritative: it is complete + * and uncapped, whereas FTS ranks and truncates, and tokenises away `::` — so + * a qualified query could miss a symbol that exists, or land on whatever + * happened to rank first. FTS remains as the fallback for the fuzzy cases it is + * genuinely good at (file basenames, partial names). + */ +export function lookupSymbolNodes(cg: SymbolLookupHost, symbol: string): SymbolLookupResult { + const qualified = isQualifiedSymbol(symbol); + + // Exact-name index, then filter by the qualifier the user actually wrote. + const tail = qualified ? lastQualifierPart(symbol) : symbol; + let nodes = tail ? cg.getNodesByName(tail) : []; + if (qualified) nodes = nodes.filter((n) => matchesSymbol(n, symbol)); + + if (nodes.length === 0) { + const hits = cg.searchNodes(symbol, { limit: 50 }).map((h) => h.node); + const exact = hits.filter((n) => matchesSymbol(n, symbol)); + if (exact.length > 0) { + nodes = exact; + } else if (!qualified && hits[0]) { + // A bare name with no exact definition may still mean a file basename. + nodes = [hits[0]]; + } + // A qualified query with no exact match resolves to NOTHING rather than a + // misleading fuzzy hit (#173). + } + + if (nodes.length === 0) return { nodes: [], ambiguous: false }; + + // Keepers before generated stubs (.pb.go and friends), stable otherwise. + const isGenerated = cg.generatedFilePredicate(nodes.map((n) => n.filePath)); + const ranked = [...nodes].sort( + (a, b) => (isGenerated(a.filePath) ? 1 : 0) - (isGenerated(b.filePath) ? 1 : 0) + ); + return { nodes: ranked, ambiguous: ranked.length > 1 }; +} + +/** One-line "kind at path:line" label used when disclosing an ambiguous query. */ +export function describeSymbolNode(node: Node): string { + return `${node.kind} ${node.qualifiedName || node.name} (${node.language}) — ${node.filePath}:${node.startLine}`; +} diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 5c23f675d..535490067 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -32,6 +32,7 @@ import { import type { PendingFile } from '../sync'; import type { Node, Edge, SearchResult, Subgraph, NodeKind } from '../types'; import { isTestFile, normalizeNameToken } from '../search/query-utils'; +import { lastQualifierPart, matchesSymbol as matchesSymbolShared } from '../graph/symbol-lookup'; import { extractQueryPaths, queryMightContainPaths } from '../search/query-paths'; import { existsSync, @@ -104,13 +105,9 @@ const MAX_INPUT_LENGTH = 10_000; const MAX_PATH_LENGTH = 4_096; /** - * Rust path roots that have no file-system equivalent — `crate` is the - * current crate, `super` is the parent module, `self` is the current - * module. Used by `matchesSymbol` to strip these before file-path - * matching so `crate::configurator::stage_apply::run` resolves the - * same as `configurator::stage_apply::run`. + * (Rust path roots — `crate`/`super`/`self` — moved to + * ../graph/symbol-lookup along with `matchesSymbol`.) */ -const RUST_PATH_PREFIXES = new Set(['crate', 'super', 'self']); /** * Node kinds that contain other symbols. For these, `codegraph_node` with @@ -123,15 +120,10 @@ const CONTAINER_NODE_KINDS = new Set([ ]); /** - * Last `::` / `.` / `/`-separated segment of a qualified symbol. An Erlang - * arity tail (`mod::fn/3`, `fn/3`) is stripped first — the useful last segment - * is the function name, never the digits (#1610). + * Symbol-name resolution lives in ../graph/symbol-lookup so the CLI verbs and + * these tools share one path. `lastQualifierPart` and `matchesSymbol` are + * re-exported through this module's own call sites unchanged. */ -function lastQualifierPart(symbol: string): string { - const noArity = symbol.replace(/\/\d{1,3}$/, '') || symbol; - const parts = noArity.split(/::|[./]/).filter((p) => p.length > 0); - return parts[parts.length - 1] ?? symbol; -} /** * Normalize Erlang-native symbol spellings in an explore query into the shapes @@ -6728,53 +6720,7 @@ export class ToolHandler { * Python — `stage_apply::run` matches a `run` in `stage_apply.rs`) */ private matchesSymbol(node: Node, symbol: string): boolean { - // Erlang arity spelling (`fn/3`, `mod:fn/3` → normalized `mod.fn/3`): when - // the node's qualifiedName carries an arity (`mod::fn/3`, #1610), the - // written arity must match it exactly; the remaining comparison then runs - // on the arity-less spelling. A node with no arity in its qualifiedName - // keeps the original symbol (a `/` there means a path-ish name instead). - const aritySpelling = /^(.+)\/(\d{1,3})$/.exec(symbol); - if (aritySpelling) { - const nodeArity = /\/(\d{1,3})$/.exec(node.qualifiedName ?? '')?.[1]; - if (nodeArity !== undefined) { - if (nodeArity !== aritySpelling[2]) return false; - symbol = aritySpelling[1]!; - } - } - // Simple name match - if (node.name === symbol) return true; - // File basename match (e.g., "product-card" matches "product-card.liquid") - if (node.kind === 'file' && node.name.replace(/\.[^.]+$/, '') === symbol) return true; - - // Qualified-name lookups: split on any supported separator. `\w` keeps - // identifier chars (incl. `_`) intact; everything else is treated as - // a separator we tolerate. - if (!/[.\/]|::/.test(symbol)) return false; - const parts = symbol.split(/::|[./]/).filter((p) => p.length > 0); - if (parts.length < 2) return false; - - const lastPart = parts[parts.length - 1]!; - if (node.name !== lastPart) return false; - - // Stage 1: qualified-name suffix match. The extractor joins the - // semantic hierarchy with `::`, so `Session.request` and - // `Session::request` both become `Session::request` here. - const colonSuffix = parts.join('::'); - if (node.qualifiedName.includes(colonSuffix)) return true; - - // Stage 2: file-path containment. Rust modules and Python packages - // are not in `qualifiedName` — they're encoded in the file path. So - // `stage_apply::run` matches a `run` in any file whose path - // contains a `stage_apply` segment (with or without an extension). - // - // Filter out Rust path prefixes that have no file-system equivalent. - const containerHints = parts.slice(0, -1).filter((p) => !RUST_PATH_PREFIXES.has(p)); - if (containerHints.length === 0) return false; - - const segments = node.filePath.split('/').filter((s) => s.length > 0); - return containerHints.every((hint) => - segments.some((seg) => seg === hint || seg.replace(/\.[^.]+$/, '') === hint) - ); + return matchesSymbolShared(node, symbol); } /** From dcb370937d5eae49244f87c0307b5f358d946789 Mon Sep 17 00:00:00 2001 From: ferres Date: Sun, 30 Aug 2026 22:19:23 +0000 Subject: [PATCH 2/2] feat(proto): index Protocol Buffers as a contract, not an asset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `.proto` is the contract layer of a polyglot repository: one field is implemented again in every generated language, each of those sites is machine-written and must never be hand-edited, and none of it was visible to the graph. Indexing it makes a field a symbol you can ask questions about — and makes "what else moves when this changes" answerable from the one place the answer is actually written down. Extracted: messages (including nested), enums and their values, fields, `oneof` members, `map` types, services, `rpc`s (including `stream`), imports, and `reserved`. Names are protobuf's own fully-qualified names (`acme.reporting.v1.Measurement.observed_at`), which is both what a user would type and what a generator uses. Field types and rpc request / response messages become `references`, so `callers` on a message lists the fields and RPCs that depend on it. Two properties are modelled on purpose, because they are where protobuf's real defects live and neither survives a naive extraction: - THE TAG NUMBER IS PART OF A FIELD'S IDENTITY. Renaming a field at the same tag is wire-compatible; changing its TYPE at the same tag and name is a silent mis-decode that every single-language check passes. The tag is recorded as a marker, not just left in prose, so a check can read it without re-parsing the declaration. - `reserved` IS SEMANTIC. A retired number must never be re-used and a reader touching a reserved field is a defect, so reservations — numbers, ranges and names — are kept as symbols rather than discarded as syntax. They are deliberately not `field` nodes, so a reservation can never be mistaken for a live field. Implemented as a standalone scanner (proto-extractor.ts) rather than a vendored grammar, following the Liquid / Razor / MyBatis precedent. The IDL is small and effectively frozen, so the usual reason to want a grammar — tracking an evolving syntax surface — does not apply, and it avoids shipping another megabyte of wasm whose silent absence is its own failure mode. Comments are blanked offset-preserving before scanning, skipping string literals so a URL in an option value is not mistaken for a comment. Verified against upstream protos: descriptor.proto (proto2, groups, extensions, 17 reserved statements), pubsub.proto, struct.proto, timestamp.proto and grpc health.proto all parse with zero errors, and the message / enum / service / rpc counts match the files exactly. Depends on the qualified-lookup fix: without it a protobuf FQN is a dotted container name, which the old matcher could not match at all. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013Bsi9EH64kMisnik4E1gf7 --- CHANGELOG.md | 4 + README.md | 3 +- __tests__/proto-extraction.test.ts | 328 +++++++++++++++++++ src/extraction/grammars.ts | 11 +- src/extraction/proto-extractor.ts | 498 +++++++++++++++++++++++++++++ src/extraction/tree-sitter.ts | 6 + src/types.ts | 1 + 7 files changed, 848 insertions(+), 3 deletions(-) create mode 100644 __tests__/proto-extraction.test.ts create mode 100644 src/extraction/proto-extractor.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 761b6b1b8..0005074ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### New Features + +- CodeGraph now indexes **Protocol Buffers** (`.proto`). Messages, nested messages, enums and their values, fields, `oneof` members, services and their `rpc`s all become symbols, named the way protobuf names them — so `acme.reporting.v1.Measurement.observed_at` is something you can look up, and asking what depends on a message lists the fields and RPCs that use it. Two details are modelled on purpose: a field keeps its **tag number**, because renaming a field at the same tag is harmless while changing its type at the same tag is a silent wire break; and `reserved` numbers, ranges and names are kept as symbols, so a retired field is still findable instead of vanishing from the graph. + ### Fixes - `codegraph callers`, `codegraph callees` and `codegraph impact` now understand qualified symbol names. Asking for something like `Accounts.Format.group` used to match nothing, so the command quietly fell back to whichever symbol full-text search happened to rank first — or reported "not found" for a symbol that plainly exists. Qualified names now select exactly the definition you named, including in languages whose module names themselves contain dots. diff --git a/README.md b/README.md index 48323f6fd..6520fbf53 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ CodeGraph's parsing engine is a **native Rust kernel**: 20 languages — TypeScr | **Full-Text Search** | Find code by name instantly across your entire codebase, powered by FTS5 | | **Impact Analysis** | Trace callers, callees, and the full impact radius of any symbol before making changes | | **Always Fresh** | File watcher uses native OS events (FSEvents/inotify/ReadDirectoryChangesW) with debounced auto-sync — the graph stays current as you code, zero config | -| **20+ Languages** | TypeScript, JavaScript, ArkTS, Python, Go, Rust, Java, C#, VB.NET, PHP, Ruby, C, C++, CUDA, Objective-C, Metal, Swift, Kotlin, Scala, Dart, Lua, Luau, R, Nix, Erlang, CFML, COBOL, Solidity, Terraform/OpenTofu, Svelte, Vue, Astro, Liquid, Pascal/Delphi | +| **20+ Languages** | TypeScript, JavaScript, ArkTS, Python, Go, Rust, Java, C#, VB.NET, PHP, Ruby, C, C++, CUDA, Objective-C, Metal, Swift, Kotlin, Scala, Dart, Lua, Luau, R, Nix, Erlang, Protocol Buffers, CFML, COBOL, Solidity, Terraform/OpenTofu, Svelte, Vue, Astro, Liquid, Pascal/Delphi | | **Framework-aware Routes** | Recognizes web-framework routing files and links URL patterns to their handlers across 17 frameworks | | **Mixed iOS / React Native / Expo** | Closes cross-language flows that static parsing misses: Swift ↔ ObjC bridging, React Native legacy bridge + TurboModules + Fabric view components, native → JS event emitters, Expo Modules | | **100% Local** | No data leaves your machine. No API keys. No external services. SQLite database only | @@ -822,6 +822,7 @@ is written): | COBOL | `.cbl`, `.cob`, `.cpy` | Full support (programs, sections/paragraphs with PERFORM/GO TO call edges, CALL 'literal' cross-program calls, COPY copybook imports — including standalone `.cpy` files — DATA DIVISION records/fields/88-levels, EXEC CICS LINK/XCTL and EXEC SQL INCLUDE targets; fixed and free format) | | Visual Basic .NET | `.vb` | Full support (classes, Modules, interfaces, structures, enums, properties, events, `Declare` P/Invoke, `Handles`/`WithEvents`, `Inherits`/`Implements` edges, call edges through VB's call/index paren ambiguity, `As New` instantiation, interpolated strings, LINQ, Unicode identifiers) | | Erlang | `.erl`, `.hrl`, `.escript`, `.app.src`, `.app` | Full support (functions with multi-clause/multi-arity grouping, `-spec` signatures, records with fields, `-type`/`-opaque` aliases, `-define` macros, `-include`/`-include_lib`/`-import` edges, local and `mod:fn` remote call edges, `fun name/arity` references, `spawn`/`apply`/`proc_lib`/`timer`/`rpc` MFA-argument call edges, `gen_server:call/cast(?MODULE)` → own `handle_call`/`handle_cast` links, `-behaviour` links, `-export`-based visibility) | +| Protocol Buffers | `.proto` | Full support (messages incl. nested, enums and their values, fields with their **tag number** and declared type, `oneof` members as fields of the enclosing message, `map` types, services and `rpc`s incl. `stream`, `import`s, and `reserved` numbers, ranges and names kept as symbols so retired fields stay queryable; names are the protobuf fully-qualified names, and field/rpc types link to the declarations they use) | | Solidity | `.sol` | Full support (contracts, libraries, interfaces, structs, enums, modifiers, events, errors, state variables, `import`/`using` directives, `emit`/`revert` calls) | | Terraform / OpenTofu | `.tf`, `.tfvars`, `.tofu` | Full support (resources, data sources, modules, variables, outputs, providers incl. aliases, `locals`; `var.`/`local.`/`module.`/resource references with Terraform's per-directory scoping enforced; module calls bridged across the boundary — inputs to the child module's variables, `module.M.out` to the child's output, `source` to the module's files; cloudposse/atmos `remote-state` cross-component wiring when the component is statically named; `provider = aws.east` selections resolved up the module tree; `moved`/`import`/`removed`/`check` block references; `.tfvars` assignments linked to the variables they set) | | Nix | `.nix` | Full support (functions with simple/destructured/curried params, `let`/attrset bindings, `inherit`, `import ./path` file edges — `./dir` resolving through `default.nix` — plus NixOS module `imports = [ ./x.nix ]` lists and `callPackage ./pkg.nix` file edges; call edges; module-system option wiring — a config write like `launchd.user.agents.x = { ... }` links to the module declaring `options.launchd.user.agents`, so option flows trace across modules) | diff --git a/__tests__/proto-extraction.test.ts b/__tests__/proto-extraction.test.ts new file mode 100644 index 000000000..4494614c3 --- /dev/null +++ b/__tests__/proto-extraction.test.ts @@ -0,0 +1,328 @@ +/** + * Protocol Buffers (`.proto`) extraction. + * + * A `.proto` is a contract: one field is implemented again in every generated + * language, and each of those sites is machine-written. For the graph to answer + * "what else moves when this changes", a field has to be a symbol — and two of + * its properties have to survive extraction, because they are where protobuf's + * real defects live: + * + * - the TAG NUMBER is part of a field's identity. Renaming a field but + * keeping its tag is wire-compatible; keeping tag AND name while changing + * the type is a silent mis-decode that every single-language check passes; + * - `reserved` is semantic — a retired number must never be re-used, and a + * reader touching a reserved field is a defect. + */ + +import { describe, it, expect } from 'vitest'; +import { extractFromSource } from '../src/extraction/tree-sitter'; +import { detectLanguage, isSourceFile, isLanguageSupported, getSupportedLanguages } from '../src/extraction/grammars'; +import { blankProtoComments, parseField, parseRpc, typeReferences } from '../src/extraction/proto-extractor'; +import { matchesSymbol } from '../src/graph/symbol-lookup'; +import type { Node } from '../src/types'; + +const SAMPLE = `syntax = "proto3"; + +package acme.reporting.v1; + +import "google/protobuf/timestamp.proto"; +import public "acme/common/money.proto"; + +option go_package = "acme/reporting"; + +// A single reported measurement. +// Values are already normalised. +message Measurement { + // Stable identifier. + string id = 1; + double value = 2; + /* The unit the value is expressed in. */ + Unit unit = 3; + repeated string tags = 4; + map breakdown = 5; + acme.common.Money cost = 6; + google.protobuf.Timestamp observed_at = 7; + + reserved 8, 9; + reserved 12 to 15; + reserved "legacy_value", "old_unit"; + + oneof source { + string manual_entry = 20; + Ingest ingest = 21; + } + + message Ingest { + string pipeline = 1; + } +} + +// Units a measurement may carry. +enum Unit { + UNIT_UNSPECIFIED = 0; + UNIT_CURRENCY = 1; +} + +message ListRequest { string filter = 1; } +message ListResponse { repeated Measurement measurements = 1; } + +// Read-side API. +service ReportingService { + // Page through measurements. + rpc List(ListRequest) returns (ListResponse); + rpc Stream(ListRequest) returns (stream Measurement) { + option idempotency_level = NO_SIDE_EFFECTS; + } +} +`; + +function extract(source = SAMPLE) { + return extractFromSource('proto/service.proto', source, 'proto'); +} + +function byQualified(nodes: Node[], qualified: string): Node | undefined { + return nodes.find((n) => n.qualifiedName === qualified); +} + +describe('proto — language registration', () => { + it('detects .proto files', () => { + expect(detectLanguage('proto/service.proto')).toBe('proto'); + expect(isSourceFile('proto/service.proto')).toBe(true); + }); + + it('reports proto as supported', () => { + expect(isLanguageSupported('proto')).toBe(true); + expect(getSupportedLanguages()).toContain('proto'); + }); +}); + +describe('proto — pure parsing helpers', () => { + it('blanks comments while preserving offsets and line numbers', () => { + const src = 'a // note\nb /* x\ny */ c\n'; + const out = blankProtoComments(src); + expect(out.length).toBe(src.length); + expect(out.split('\n').length).toBe(src.split('\n').length); + expect(out).not.toContain('note'); + }); + + it('does not treat a // inside a string literal as a comment', () => { + // An option value is frequently a URL — mistaking it for a comment would + // swallow the rest of the line, including its terminating semicolon. + const src = 'option (x) = "http://example.com/a"; message M { string a = 1; }'; + const out = blankProtoComments(src); + expect(out).toContain('http://example.com/a'); + const result = extract(src); + expect(byQualified(result.nodes, 'M.a')).toBeDefined(); + }); + + it('parses field declarations including label, map types and tag', () => { + expect(parseField('string id = 1')).toMatchObject({ type: 'string', name: 'id', tag: 1 }); + expect(parseField('repeated Foo bar = 3')).toMatchObject({ label: 'repeated', type: 'Foo', name: 'bar', tag: 3 }); + expect(parseField('map breakdown = 5')).toMatchObject({ + type: 'map', name: 'breakdown', tag: 5, + }); + expect(parseField('.acme.Money cost = 6')).toMatchObject({ type: '.acme.Money', tag: 6 }); + expect(parseField('not a field')).toBeNull(); + }); + + it('parses rpc declarations including stream markers', () => { + expect(parseRpc('rpc List(Req) returns (Res)')).toMatchObject({ + name: 'List', input: 'Req', output: 'Res', streamIn: false, streamOut: false, + }); + expect(parseRpc('rpc S(stream Req) returns (stream Res)')).toMatchObject({ + streamIn: true, streamOut: true, + }); + expect(parseRpc('message M')).toBeNull(); + }); + + it('reports only declared types as references, never scalars', () => { + expect(typeReferences('string')).toEqual([]); + expect(typeReferences('Foo')).toEqual(['Foo']); + expect(typeReferences('map')).toEqual([]); + expect(typeReferences('map')).toEqual(['Money']); + }); +}); + +describe('proto — declarations', () => { + const result = extract(); + + it('extracts messages, enums and services under the package name', () => { + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement')?.kind).toBe('struct'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Unit')?.kind).toBe('enum'); + expect(byQualified(result.nodes, 'acme.reporting.v1.ReportingService')?.kind).toBe('interface'); + }); + + it('nests a nested message under its parent, as protobuf names it', () => { + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.Ingest')?.kind).toBe('struct'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.Ingest.pipeline')?.kind).toBe('field'); + }); + + it('extracts enum values with their numbers', () => { + const v = byQualified(result.nodes, 'acme.reporting.v1.Unit.UNIT_CURRENCY'); + expect(v?.kind).toBe('enum_member'); + expect(v?.decorators).toContain('number=1'); + }); + + it('extracts rpcs as methods of their service, with the stream marker kept', () => { + const list = byQualified(result.nodes, 'acme.reporting.v1.ReportingService.List'); + expect(list?.kind).toBe('method'); + expect(list?.signature).toBe('rpc List(ListRequest) returns (ListResponse)'); + const stream = byQualified(result.nodes, 'acme.reporting.v1.ReportingService.Stream'); + expect(stream?.signature).toContain('returns (stream Measurement)'); + }); + + it('carries // and /* */ comments through as docstrings', () => { + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement')?.docstring) + .toBe('A single reported measurement.\nValues are already normalised.'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.id')?.docstring) + .toBe('Stable identifier.'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.unit')?.docstring) + .toBe('The unit the value is expressed in.'); + }); + + it('records imports, including the `public` form', () => { + const imports = result.unresolvedReferences.filter((r) => r.referenceKind === 'imports'); + expect(imports.map((r) => r.referenceName)).toEqual([ + 'google/protobuf/timestamp.proto', + 'acme/common/money.proto', + ]); + }); + + it('ignores syntax and option statements', () => { + expect(result.nodes.some((n) => n.name === 'go_package')).toBe(false); + expect(result.nodes.some((n) => n.name === 'idempotency_level')).toBe(false); + }); + + it('reports no extraction errors on a representative file', () => { + expect(result.errors).toEqual([]); + }); +}); + +describe('proto — the tag number is part of a field identity', () => { + const result = extract(); + + it('records each field tag as a marker, not only as prose', () => { + // A "same tag, changed type" check has to read the tag without re-parsing + // the declaration. + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.id')?.decorators).toContain('tag=1'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.observed_at')?.decorators).toContain('tag=7'); + }); + + it('keeps the declaration verbatim so type and tag are both visible', () => { + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.observed_at')?.signature) + .toBe('google.protobuf.Timestamp observed_at = 7'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.tags')?.signature) + .toBe('repeated string tags = 4'); + }); + + it('marks repeated fields', () => { + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.tags')?.decorators).toContain('repeated'); + }); + + it('distinguishes a renamed field from a retyped one at the same tag', () => { + // Rename at the same tag: wire-compatible. Retype at the same tag and name: + // a silent mis-decode. The graph must be able to tell these apart. + const renamed = extract('message M { string b = 1; }'); + const retyped = extract('message M { int64 a = 1; }'); + const original = extract('message M { string a = 1; }'); + const tagOf = (r: ReturnType, name: string) => + r.nodes.find((n) => n.name === name)?.decorators?.find((d) => d.startsWith('tag=')); + expect(tagOf(original, 'a')).toBe('tag=1'); + expect(tagOf(renamed, 'b')).toBe('tag=1'); + expect(tagOf(retyped, 'a')).toBe('tag=1'); + // Same tag either way — the difference is in the declaration text. + expect(original.nodes.find((n) => n.name === 'a')?.signature).toBe('string a = 1'); + expect(retyped.nodes.find((n) => n.name === 'a')?.signature).toBe('int64 a = 1'); + }); +}); + +describe('proto — reserved is kept, not discarded as syntax', () => { + const result = extract(); + const reserved = result.nodes.filter((n) => n.decorators?.includes('reserved')); + + it('records reserved numbers individually', () => { + expect(reserved.map((n) => n.name)).toEqual( + expect.arrayContaining(['reserved 8', 'reserved 9']) + ); + }); + + it('records reserved ranges as written', () => { + expect(reserved.map((n) => n.name)).toContain('reserved 12 to 15'); + // A range must not also be double-counted as its two endpoints. + expect(reserved.map((n) => n.name)).not.toContain('reserved 12'); + expect(reserved.map((n) => n.name)).not.toContain('reserved 15'); + }); + + it('records reserved names so a lookup for a retired name finds the reservation', () => { + expect(reserved.map((n) => n.name)).toEqual( + expect.arrayContaining(['reserved legacy_value', 'reserved old_unit']) + ); + }); + + it('never lets a reservation be mistaken for a live field', () => { + for (const n of reserved) expect(n.kind).not.toBe('field'); + expect(result.nodes.some((n) => n.kind === 'field' && n.name === 'legacy_value')).toBe(false); + }); +}); + +describe('proto — oneof members belong to the enclosing message', () => { + const result = extract(); + + it('does not add the oneof name as a level in the qualified name', () => { + // On the wire a oneof member IS a field of the message; a `.source.` level + // would not match the name any generator uses. + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.manual_entry')?.kind).toBe('field'); + expect(byQualified(result.nodes, 'acme.reporting.v1.Measurement.source.manual_entry')).toBeUndefined(); + }); + + it('still marks them as oneof members and keeps their tags', () => { + const node = byQualified(result.nodes, 'acme.reporting.v1.Measurement.manual_entry'); + expect(node?.decorators).toContain('oneof'); + expect(node?.decorators).toContain('tag=20'); + }); +}); + +describe('proto — dependencies between declarations', () => { + const result = extract(); + const refs = result.unresolvedReferences.filter((r) => r.referenceKind === 'references'); + + it('links a field to the message or enum it is typed with', () => { + expect(refs.map((r) => r.referenceName)).toEqual( + expect.arrayContaining(['Unit', 'acme.common.Money', 'google.protobuf.Timestamp', 'Ingest']) + ); + }); + + it('does not emit references for scalar field types', () => { + for (const scalar of ['string', 'double', 'int64', 'bool', 'bytes']) { + expect(refs.map((r) => r.referenceName)).not.toContain(scalar); + } + }); + + it('links an rpc to both its request and response messages', () => { + const names = refs.map((r) => r.referenceName); + expect(names).toEqual(expect.arrayContaining(['ListRequest', 'ListResponse', 'Measurement'])); + }); + + it('strips protobufs leading-dot absolute marker from a reference', () => { + const absolute = extract('package p;\nmessage M { .other.pkg.Thing t = 1; }'); + const names = absolute.unresolvedReferences.map((r) => r.referenceName); + expect(names).toContain('other.pkg.Thing'); + expect(names).not.toContain('.other.pkg.Thing'); + }); +}); + +describe('proto — a fully-qualified protobuf name is a usable query', () => { + const result = extract(); + + it('matches a field by its dotted protobuf FQN', () => { + const field = byQualified(result.nodes, 'acme.reporting.v1.Measurement.observed_at')!; + expect(matchesSymbol(field, 'acme.reporting.v1.Measurement.observed_at')).toBe(true); + // A partial suffix on a separator boundary is how people actually type it. + expect(matchesSymbol(field, 'Measurement.observed_at')).toBe(true); + }); + + it('does not match a field of a different message', () => { + const field = byQualified(result.nodes, 'acme.reporting.v1.Measurement.Ingest.pipeline')!; + expect(matchesSymbol(field, 'ListRequest.pipeline')).toBe(false); + }); +}); diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 84647c3e4..18d263a6f 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -11,7 +11,7 @@ import * as fsp from 'fs/promises'; import { Parser, Language as WasmLanguage } from 'web-tree-sitter'; import { Language } from '../types'; -export type GrammarLanguage = Exclude; +export type GrammarLanguage = Exclude; /** * WASM filename map — maps each language to its .wasm grammar file @@ -170,6 +170,10 @@ export const EXTENSION_MAP: Record = { '.tf': 'terraform', '.tfvars': 'terraform', '.tofu': 'terraform', + // Protocol Buffers IDL. Standalone scanner (proto-extractor.ts), not a + // tree-sitter grammar — the IDL is small and frozen, and a `.proto` is a + // contract whose tag numbers and reservations need modelling, not just a tree. + '.proto': 'proto', }; /** @@ -575,6 +579,7 @@ export function isLanguageSupported(language: Language): boolean { if (language === 'twig') return true; // file-level tracking only if (language === 'xml') return true; // MyBatis mapper extractor if (language === 'properties') return true; // Spring config keys + if (language === 'proto') return true; // custom ProtoExtractor (no wasm grammar) if (language === 'unknown') return false; return language in WASM_GRAMMAR_FILES; } @@ -586,6 +591,7 @@ export function isGrammarLoaded(language: Language): boolean { if (language === 'svelte' || language === 'vue' || language === 'astro' || language === 'liquid' || language === 'razor') return true; if (language === 'yaml' || language === 'twig') return true; // no WASM grammar needed if (language === 'xml' || language === 'properties') return true; // no WASM grammar needed + if (language === 'proto') return true; // no WASM grammar needed return languageCache.has(language); } @@ -606,7 +612,7 @@ export function isFileLevelOnlyLanguage(language: Language): boolean { * Get all supported languages (those with grammar definitions). */ export function getSupportedLanguages(): Language[] { - return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid']; + return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid', 'proto']; } /** @@ -693,6 +699,7 @@ export function getLanguageDisplayName(language: Language): string { vbnet: 'Visual Basic .NET', erlang: 'Erlang', terraform: 'Terraform', + proto: 'Protocol Buffers', arkts: 'ArkTS', unknown: 'Unknown', }; diff --git a/src/extraction/proto-extractor.ts b/src/extraction/proto-extractor.ts new file mode 100644 index 000000000..22f39eb04 --- /dev/null +++ b/src/extraction/proto-extractor.ts @@ -0,0 +1,498 @@ +import { Node, Edge, ExtractionResult, ExtractionError, UnresolvedReference, NodeKind } from '../types'; +import { generateNodeId } from './tree-sitter-helpers'; + +/** + * ProtoExtractor — Protocol Buffers (`.proto`) IDL. + * + * A `.proto` file is a CONTRACT: one field definition is implemented three or + * more times over, once per generated language, and every one of those sites is + * machine-written and must never be hand-edited. That makes the `.proto` the + * highest-value file per byte in a polyglot repository and the natural anchor + * for "what else moves when this changes" — but only if a field is a symbol the + * graph can answer questions about. + * + * Written as a standalone scanner rather than a vendored tree-sitter grammar, + * following the same precedent as the Liquid / Razor / MyBatis extractors. The + * protobuf IDL is tiny and effectively frozen (proto2 and proto3 are published, + * stable specs), so the usual reason to want a grammar — keeping up with an + * evolving syntax surface — does not apply; and every vendored `.wasm` is a + * megabyte of shipped binary plus an ABI obligation, whose silent absence is + * itself a failure mode worth not multiplying. + * + * Two properties are modelled deliberately, because they are where protobuf's + * real defects live and neither is expressible without them: + * + * - THE TAG NUMBER IS PART OF A FIELD'S IDENTITY. Renaming a field while + * keeping its tag is wire-compatible; changing its TYPE (or its meaning) + * while keeping tag and name is a silent mis-decode that every + * single-language check passes. Both the declaration text and the tag are + * recorded, so the difference is expressible. + * - `reserved` IS SEMANTIC. A retired field number must never be re-used, and + * a reader that touches a reserved field is a defect. Reservations are kept + * as nodes rather than discarded as syntax, so that check can be written. + */ + +/** Built-in scalar types — never a reference to another declaration. */ +const SCALAR_TYPES = new Set([ + 'double', 'float', 'int32', 'int64', 'uint32', 'uint64', 'sint32', 'sint64', + 'fixed32', 'fixed64', 'sfixed32', 'sfixed64', 'bool', 'string', 'bytes', +]); + +/** Statement keywords that carry no graph structure of their own. */ +const IGNORED_STATEMENTS = new Set(['syntax', 'option', 'extensions', 'edition']); + +interface Scope { + /** Dotted protobuf name of this scope, e.g. `pkg.Outer.Inner`. */ + qualified: string; + /** What opened it — decides how child statements are read. */ + kind: 'message' | 'enum' | 'service' | 'oneof' | 'extend' | 'rpc' | 'unknown'; + /** Node id, so children attach by `contains`. */ + nodeId?: string; +} + +export class ProtoExtractor { + private filePath: string; + private source: string; + /** Comments blanked to spaces, newlines preserved — offsets and lines hold. */ + private code: string; + private nodes: Node[] = []; + private edges: Edge[] = []; + private unresolvedReferences: UnresolvedReference[] = []; + private errors: ExtractionError[] = []; + private packageName = ''; + private scopes: Scope[] = []; + + constructor(filePath: string, source: string) { + this.filePath = filePath; + this.source = source; + this.code = blankProtoComments(source); + } + + extract(): ExtractionResult { + const startTime = Date.now(); + try { + const fileNode = this.createFileNode(); + this.scopes = [{ qualified: '', kind: 'unknown', nodeId: fileNode.id }]; + this.scan(); + } catch (error) { + this.errors.push({ + message: `Proto extraction error: ${error instanceof Error ? error.message : String(error)}`, + filePath: this.filePath, + severity: 'error', + code: 'parse_error', + }); + } + return { + nodes: this.nodes, + edges: this.edges, + unresolvedReferences: this.unresolvedReferences, + errors: this.errors, + durationMs: Date.now() - startTime, + }; + } + + // --- scanning ------------------------------------------------------------- + + /** + * Walk the comment-free source once, splitting it into the two things + * protobuf is made of: block HEADERS (text before a `{`) and STATEMENTS (text + * before a `;`). A brace-depth scope stack gives every declaration its + * enclosing message/service, which is what makes the dotted names below the + * real protobuf fully-qualified names. + */ + private scan(): void { + let buffer = ''; + let bufferStart = -1; + for (let i = 0; i < this.code.length; i++) { + const ch = this.code[i]!; + if (ch === '{' || ch === '}' || ch === ';') { + const text = buffer.trim(); + const line = bufferStart >= 0 ? this.lineAt(bufferStart) : this.lineAt(i); + if (ch === '{') this.openBlock(text, line, i); + else if (ch === ';') this.statement(text, line, i); + else this.closeBlock(); + buffer = ''; + bufferStart = -1; + continue; + } + if (buffer.length === 0 && /\s/.test(ch)) continue; // skip leading space + if (buffer.length === 0) bufferStart = i; + buffer += ch; + } + } + + private openBlock(header: string, line: number, index: number): void { + const decl = /^(message|enum|service|oneof|extend)\s+([A-Za-z_][\w.]*)/.exec(header); + if (decl) { + const kind = decl[1] as Scope['kind']; + const name = decl[2]!; + const node = this.declare(kind, name, line, index, header); + // A `oneof` groups fields that already belong to the enclosing message — + // on the wire they ARE its fields. It must not contribute a level to + // their qualified names, which have to stay the protobuf FQN a generator + // (and the code generated from it) uses. + const qualified = kind === 'oneof' ? this.currentScope().qualified : this.qualify(name); + this.scopes.push({ + qualified, + kind, + nodeId: node?.id ?? this.currentScope().nodeId, + }); + return; + } + // `rpc Get(Req) returns (Res) { option ...; }` — the braced rpc form. + const rpc = parseRpc(header); + if (rpc) { + const node = this.declareRpc(rpc, line, index, header); + this.scopes.push({ qualified: this.qualify(rpc.name), kind: 'rpc', nodeId: node?.id }); + return; + } + // Anything else with a body (an `option (x) = { ... }` block, a group): + // push an opaque scope so brace depth stays correct and its contents are + // not mistaken for fields of the enclosing message. + this.scopes.push({ qualified: this.currentScope().qualified, kind: 'unknown' }); + } + + private closeBlock(): void { + if (this.scopes.length > 1) this.scopes.pop(); + } + + private statement(text: string, line: number, index: number): void { + if (!text) return; + const keyword = /^([A-Za-z_]\w*)/.exec(text)?.[1] ?? ''; + if (IGNORED_STATEMENTS.has(keyword)) return; + + if (keyword === 'package') { + const pkg = /^package\s+([\w.]+)$/.exec(text)?.[1]; + if (pkg) this.packageName = pkg; + return; + } + if (keyword === 'import') { + this.declareImport(text, line, index); + return; + } + if (keyword === 'reserved') { + this.declareReserved(text, line, index); + return; + } + if (keyword === 'rpc') { + const rpc = parseRpc(text); + if (rpc) this.declareRpc(rpc, line, index, text); + return; + } + + const scope = this.currentScope(); + if (scope.kind === 'enum') { + this.declareEnumValue(text, line, index); + return; + } + if (scope.kind === 'message' || scope.kind === 'oneof' || scope.kind === 'extend') { + this.declareField(text, line, index); + } + } + + // --- declarations --------------------------------------------------------- + + private declare( + kind: Scope['kind'], + name: string, + line: number, + index: number, + header: string + ): Node | null { + // A `oneof` is a grouping construct, not a type: its fields belong to the + // enclosing message and its own name would only add a phantom level to + // their qualified names, which must stay the wire-level protobuf FQN. + if (kind === 'oneof') return null; + const nodeKind: NodeKind = + kind === 'enum' ? 'enum' : kind === 'service' ? 'interface' : 'struct'; + return this.push(nodeKind, name, line, index, { + signature: collapse(header), + docstring: this.docFor(index), + isExported: true, + }); + } + + private declareRpc( + rpc: { name: string; input: string; output: string; streamIn: boolean; streamOut: boolean }, + line: number, + index: number, + header: string + ): Node | null { + const node = this.push('method', rpc.name, line, index, { + signature: collapse(header).replace(/\s*\{$/, ''), + docstring: this.docFor(index), + isExported: true, + returnType: rpc.output, + }); + // The request and response messages are this method's real dependencies — + // the edge that makes "what does changing this message break" answerable. + for (const type of [rpc.input, rpc.output]) { + this.reference(node?.id, type, line, index); + } + return node; + } + + private declareField(text: string, line: number, index: number): void { + const field = parseField(text); + if (!field) return; + const node = this.push('field', field.name, line, index, { + signature: collapse(text), + docstring: this.docFor(index), + isExported: true, + // The tag is part of the field's identity on the wire, so it is recorded + // as a marker rather than left only in the signature prose: a check for + // "same tag, changed type" has to be able to read it without re-parsing. + decorators: [ + `tag=${field.tag}`, + ...(field.label ? [field.label] : []), + ...(this.currentScope().kind === 'oneof' ? ['oneof'] : []), + ], + }); + for (const type of typeReferences(field.type)) { + this.reference(node?.id, type, line, index); + } + } + + private declareEnumValue(text: string, line: number, index: number): void { + const m = /^([A-Za-z_]\w*)\s*=\s*(-?\d+)\b/.exec(text); + if (!m) return; + this.push('enum_member', m[1]!, line, index, { + signature: collapse(text), + docstring: this.docFor(index), + isExported: true, + decorators: [`number=${m[2]}`], + }); + } + + /** + * `reserved 2, 15, 9 to 11;` / `reserved "old_name";` + * + * Kept as a node so "was this number retired" and "is this name off-limits" + * are answerable. Named after what it reserves so a lookup for the retired + * name finds the reservation, with a `reserved` marker so it can never be + * mistaken for a live field. + */ + private declareReserved(text: string, line: number, index: number): void { + const body = text.replace(/^reserved\s+/, ''); + const items: string[] = []; + for (const m of body.matchAll(/"([^"]+)"|'([^']+)'/g)) items.push(m[1] ?? m[2]!); + for (const m of body.matchAll(/(\d+)\s+to\s+(\d+|max)/gi)) items.push(`${m[1]} to ${m[2]}`); + const ranged = body.replace(/(\d+)\s+to\s+(\d+|max)/gi, ''); + for (const m of ranged.matchAll(/\b(\d+)\b/g)) items.push(m[1]!); + if (items.length === 0) return; + for (const item of items) { + this.push('constant', `reserved ${item}`, line, index, { + signature: collapse(text), + decorators: ['reserved'], + }); + } + } + + private declareImport(text: string, line: number, index: number): void { + const target = /^import\s+(?:public\s+|weak\s+)?["']([^"']+)["']$/.exec(text)?.[1]; + if (!target) return; + const node = this.push('import', target, line, index, { signature: collapse(text) }); + if (node) node.qualifiedName = target; + const from = this.scopes[0]!.nodeId; + if (!from) return; + this.unresolvedReferences.push({ + fromNodeId: from, + referenceName: target, + referenceKind: 'imports', + filePath: this.filePath, + language: 'proto', + line, + column: 0, + }); + } + + // --- helpers -------------------------------------------------------------- + + private currentScope(): Scope { + return this.scopes[this.scopes.length - 1]!; + } + + /** The protobuf fully-qualified name of a member of the current scope. */ + private qualify(name: string): string { + const parent = this.currentScope().qualified || this.packageName; + return parent ? `${parent}.${name}` : name; + } + + private push(kind: NodeKind, name: string, line: number, index: number, extra: Partial): Node | null { + if (!name) return null; + const qualifiedName = this.qualify(name); + const node: Node = { + id: generateNodeId(this.filePath, kind, qualifiedName, line), + kind, + name, + qualifiedName, + filePath: this.filePath, + language: 'proto', + startLine: line, + endLine: line, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + ...extra, + }; + this.nodes.push(node); + const parentId = this.currentScope().nodeId; + if (parentId) this.edges.push({ source: parentId, target: node.id, kind: 'contains' }); + void index; + return node; + } + + /** A dependency on another declaration (a field's type, an rpc's messages). */ + private reference(fromNodeId: string | undefined, typeName: string, line: number, index: number): void { + if (!fromNodeId || !typeName || SCALAR_TYPES.has(typeName)) return; + void index; + this.unresolvedReferences.push({ + fromNodeId, + // A leading dot is protobuf's "fully qualified from the root" marker; + // an unqualified name resolves against the enclosing package. + referenceName: typeName.replace(/^\./, ''), + referenceKind: 'references', + filePath: this.filePath, + language: 'proto', + line, + column: 0, + }); + } + + private lineAt(index: number): number { + let line = 1; + for (let i = 0; i < index && i < this.code.length; i++) { + if (this.code[i] === '\n') line++; + } + return line; + } + + /** + * The `//` or `/* *\/` comment block immediately above a declaration, read + * from the ORIGINAL source (the scanning copy has them blanked). + */ + private docFor(index: number): string | undefined { + const before = this.source.slice(0, index); + const lines = before.split('\n'); + lines.pop(); // the declaration's own (partial) line + const collected: string[] = []; + for (let i = lines.length - 1; i >= 0; i--) { + const line = lines[i]!.trim(); + if (line === '') { if (collected.length > 0) break; continue; } + const lineComment = /^\/\/+\s?(.*)$/.exec(line); + if (lineComment) { collected.unshift(lineComment[1]!); continue; } + const single = /^\/\*+\s?([\s\S]*?)\s*\*+\/$/.exec(line); + if (single) { collected.unshift(single[1]!); continue; } + const starred = /^\*+\s?(.*)$/.exec(line); + if (starred && collected.length > 0) { collected.unshift(starred[1]!); continue; } + break; + } + const text = collected.join('\n').trim(); + return text || undefined; + } + + private createFileNode(): Node { + const node: Node = { + id: `file:${this.filePath}`, + kind: 'file', + name: this.filePath.split('/').pop() ?? this.filePath, + qualifiedName: this.filePath, + filePath: this.filePath, + language: 'proto', + startLine: 1, + endLine: this.source.split('\n').length, + startColumn: 0, + endColumn: 0, + isExported: false, + updatedAt: Date.now(), + }; + this.nodes.push(node); + return node; + } +} + +// --- pure parsing helpers --------------------------------------------------- + +/** + * Replace comments with spaces, keeping every newline, so line numbers and + * offsets in the blanked copy still match the original. String literals are + * skipped so a `//` inside an option value is not mistaken for a comment. + */ +export function blankProtoComments(source: string): string { + let out = ''; + let i = 0; + while (i < source.length) { + const ch = source[i]!; + if (ch === '"' || ch === "'") { + const quote = ch; + out += ch; + i++; + while (i < source.length) { + const c = source[i]!; + out += c; + i++; + if (c === '\\' && i < source.length) { out += source[i]!; i++; continue; } + if (c === quote) break; + } + continue; + } + if (ch === '/' && source[i + 1] === '/') { + while (i < source.length && source[i] !== '\n') { out += ' '; i++; } + continue; + } + if (ch === '/' && source[i + 1] === '*') { + out += ' '; + i += 2; + while (i < source.length && !(source[i] === '*' && source[i + 1] === '/')) { + out += source[i] === '\n' ? '\n' : ' '; + i++; + } + if (i < source.length) { out += ' '; i += 2; } + continue; + } + out += ch; + i++; + } + return out; +} + +/** `rpc Get(stream Req) returns (Res)` — both the `;` and `{` forms. */ +export function parseRpc(text: string): { + name: string; input: string; output: string; streamIn: boolean; streamOut: boolean; +} | null { + const m = /^rpc\s+([A-Za-z_]\w*)\s*\(\s*(stream\s+)?([.\w]+)\s*\)\s*returns\s*\(\s*(stream\s+)?([.\w]+)\s*\)/.exec(text); + if (!m) return null; + return { + name: m[1]!, + input: m[3]!, + output: m[5]!, + streamIn: !!m[2], + streamOut: !!m[4], + }; +} + +/** + * `repeated Foo bar = 3 [deprecated = true]` — the label is optional, the type + * may itself contain separators (`map`, `.pkg.Foo`), and the tag + * is what anchors the match. + */ +export function parseField(text: string): { + label?: string; type: string; name: string; tag: number; +} | null { + const m = /^(?:(repeated|optional|required)\s+)?(.+?)\s+([A-Za-z_]\w*)\s*=\s*(\d+)\b/.exec(text); + if (!m) return null; + const type = m[2]!.trim(); + if (!type) return null; + return { label: m[1], type, name: m[3]!, tag: Number(m[4]) }; +} + +/** Declared types a field depends on — both halves of a `map`. */ +export function typeReferences(type: string): string[] { + const map = /^map\s*<\s*([^,]+?)\s*,\s*(.+?)\s*>$/.exec(type); + if (map) return [map[1]!, map[2]!].filter((t) => !SCALAR_TYPES.has(t)); + return SCALAR_TYPES.has(type) ? [] : [type]; +} + +function collapse(text: string): string { + return text.replace(/\s+/g, ' ').trim().slice(0, 300); +} diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index c34dc4716..983e48da5 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -30,6 +30,7 @@ import { AstroExtractor } from './astro-extractor'; import { DfmExtractor } from './dfm-extractor'; import { VueExtractor } from './vue-extractor'; import { MyBatisExtractor } from './mybatis-extractor'; +import { ProtoExtractor } from './proto-extractor'; import { CfmlExtractor } from './cfml-extractor'; import { tryKernelExtract, takeDeferredPreParse } from './kernel'; import { @@ -6741,6 +6742,11 @@ export function extractFromSource( // file node so the watcher tracks it without emitting symbols. const extractor = new MyBatisExtractor(filePath, source); result = extractor.extract(); + } else if (detectedLanguage === 'proto') { + // Protocol Buffers IDL — a standalone scanner rather than a grammar; see + // proto-extractor.ts for why, and for the tag/reserved modelling. + const extractor = new ProtoExtractor(filePath, source); + result = extractor.extract(); } else if (detectedLanguage === 'cfml' || detectedLanguage === 'cfscript') { // Custom extractor for CFML (.cfc/.cfm) — dialect-switches between the // tag-based cfml grammar and the bare-script cfscript grammar. Standalone diff --git a/src/types.ts b/src/types.ts index 186f57adc..ccd234cf0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -117,6 +117,7 @@ export const LANGUAGES = [ 'vbnet', 'erlang', 'terraform', + 'proto', 'unknown', ] as const;