From cf3c4b13b2b06cb35c0e0cae4388d2ac70c71c7e Mon Sep 17 00:00:00 2001 From: ferres Date: Sun, 30 Aug 2026 21:38:38 +0000 Subject: [PATCH] feat(status): separate files seen from files parsed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Files by Language` is a GROUP BY over the `files` table, whose `language` column comes from the file extension. It therefore counts a file the parser never understood exactly like one it parsed perfectly, which makes a broken language remarkably cheap to ship: remove a grammar and `init` still exits 0, `status` still lists the language with its full file count, and a symbol lookup still "runs" (its not-found message even contains the symbol name, so grepping for that passes too). Every obvious check stays green while the language contributes nothing. Add `parsedFilesByLanguage` — files with at least one node that is not their own `file` node, which every indexed file gets regardless — and surface it: - `status` prints the parsed count beside the file count whenever they differ, and names any language that parsed nothing at all; - `--strict` exits non-zero on that condition, for packaging and CI. It is opt-in: failing by default would break existing callers; - the MCP status tool reports the same split, so a connected agent is not told a language is available when none of it was extracted. The zero-parsed check is restricted to grammar-backed languages via the new `hasGrammar`. Config formats (yaml, twig, xml) are tracked at file level deliberately and must never raise it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013Bsi9EH64kMisnik4E1gf7 --- CHANGELOG.md | 3 + __tests__/status-parsed-vs-seen.test.ts | 127 ++++++++++++++++++++++++ src/bin/codegraph.ts | 51 +++++++++- src/db/queries.ts | 18 ++++ src/extraction/grammars.ts | 14 +++ src/mcp/tools.ts | 11 +- src/types.ts | 13 ++- 7 files changed, 232 insertions(+), 5 deletions(-) create mode 100644 __tests__/status-parsed-vs-seen.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ca5b9ff1..f80d904d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### New Features + +- `codegraph status` now shows how many files of each language actually produced symbols, not just how many were found. The file count is based on the file extension, so a language whose parser is missing or failing still reported its full count and looked healthy — indexing succeeded, the language was listed, and symbol lookups still ran, while nothing from those files was in the graph. When a language yields no symbols at all, status now says so in plain terms, and `codegraph status --strict` exits non-zero so packaging and CI can gate on it. The same seen-versus-parsed breakdown is shown to connected agents. ## [1.6.0] - 2026-08-26 diff --git a/__tests__/status-parsed-vs-seen.test.ts b/__tests__/status-parsed-vs-seen.test.ts new file mode 100644 index 000000000..9bfdfe189 --- /dev/null +++ b/__tests__/status-parsed-vs-seen.test.ts @@ -0,0 +1,127 @@ +/** + * `status` must distinguish files SEEN from files PARSED. + * + * The per-language file count is derived from the file EXTENSION, so it counts + * a file the parser never understood identically to one it parsed perfectly. + * That makes a broken language very cheap to ship: with a grammar missing, + * `init` still succeeds, `status` still lists the language with its full file + * count, and a symbol lookup still "runs" — every obvious check stays green + * while the language contributes nothing to the graph. + * + * The parsed count is the cheap signal that separates the two, and it is only + * meaningful if it excludes the `file` node every indexed file gets regardless. + */ + +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { initGrammars, loadAllGrammars, hasGrammar } from '../src/extraction/grammars'; + +beforeAll(async () => { + await initGrammars(); + await loadAllGrammars(); +}); + +function hasSqliteBindings(): boolean { + try { + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(':memory:'); + db.close(); + return true; + } catch { + return false; + } +} +const HAS_SQLITE = hasSqliteBindings(); + +describe('hasGrammar — which languages are expected to yield symbols', () => { + it('is true for grammar-backed languages', () => { + expect(hasGrammar('typescript')).toBe(true); + expect(hasGrammar('python')).toBe(true); + expect(hasGrammar('rust')).toBe(true); + }); + + it('is false for the formats tracked at file level on purpose', () => { + // These must never raise a "parsed nothing" warning — producing no symbols + // is their designed behaviour, not a broken grammar. + expect(hasGrammar('yaml')).toBe(false); + expect(hasGrammar('twig')).toBe(false); + expect(hasGrammar('xml')).toBe(false); + expect(hasGrammar('unknown')).toBe(false); + }); +}); + +describe.skipIf(!HAS_SQLITE)('getStats — parsedFilesByLanguage', () => { + let projectRoot: string; + let cg: any; + + beforeEach(() => { + projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-parsed-')); + }); + + afterEach(() => { + cg?.destroy(); + if (fs.existsSync(projectRoot)) fs.rmSync(projectRoot, { recursive: true, force: true }); + }); + + async function index(files: Record, include: string[]): Promise { + for (const [rel, body] of Object.entries(files)) { + const full = path.join(projectRoot, rel); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, body); + } + const CodeGraph = (await import('../src/index')).default; + cg = CodeGraph.initSync(projectRoot, { config: { include, exclude: [] } }); + await cg.indexAll(); + return cg.getStats(); + } + + it('counts a file that yielded symbols as parsed', async () => { + const stats = await index( + { 'src/a.ts': 'export function alpha(): number { return 1; }\n' }, + ['**/*.ts'] + ); + expect(stats.filesByLanguage.typescript).toBe(1); + expect(stats.parsedFilesByLanguage.typescript).toBe(1); + }, 60000); + + it('does NOT count a file whose only node is its own file node', async () => { + // An empty source file is indexed — it gets a `file` node like every other + // — but nothing was extracted from it. If the parsed count included the + // file node it would equal the seen count always, and detect nothing. + const stats = await index({ 'src/empty.ts': '\n\n' }, ['**/*.ts']); + expect(stats.filesByLanguage.typescript).toBe(1); + expect(stats.parsedFilesByLanguage.typescript ?? 0).toBe(0); + }, 60000); + + it('reports seen and parsed separately per language', async () => { + const stats = await index( + { + 'src/a.ts': 'export function alpha(): number { return 1; }\n', + 'src/empty.ts': '\n', + 'src/b.py': 'def beta():\n return 2\n', + }, + ['**/*.ts', '**/*.py'] + ); + expect(stats.filesByLanguage.typescript).toBe(2); + expect(stats.parsedFilesByLanguage.typescript).toBe(1); + expect(stats.filesByLanguage.python).toBe(1); + expect(stats.parsedFilesByLanguage.python).toBe(1); + }, 60000); + + it('a language that parsed nothing is detectable from the stats alone', async () => { + // The shape `status --strict` gates on: files present, none parsed, and the + // language is one a grammar is supposed to handle. + const stats = await index({ 'src/empty.ts': '\n' }, ['**/*.ts']); + const unparsed = Object.entries(stats.filesByLanguage as Record) + .filter( + ([lang, count]) => + count > 0 && + ((stats.parsedFilesByLanguage as Record)[lang] ?? 0) === 0 && + hasGrammar(lang as never) + ) + .map(([lang]) => lang); + expect(unparsed).toEqual(['typescript']); + }, 60000); +}); diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index 19038df1b..3bf73deaa 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -53,6 +53,8 @@ import { relaunchWithWasmRuntimeFlagsIfNeeded } from '../extraction/wasm-runtime import { installCommandSupervision } from './command-supervision'; import { EXTRACTION_VERSION } from '../extraction/extraction-version'; import { getTelemetry, TELEMETRY_DOCS, recordIndexEvent } from '../telemetry'; +// Type-only: erased at runtime, so it adds nothing to CLI startup. +import type { Language } from '../types'; // Decided once, before `--color`/`--no-color` are stripped from argv below // (#1281). Piped/redirected stdout, NO_COLOR, or --no-color -> plain output. @@ -941,7 +943,8 @@ program .command('status [path]') .description('Show index status and statistics') .option('-j, --json', 'Output as JSON') - .action(async (pathArg: string | undefined, options: { json?: boolean }) => { + .option('--strict', 'Exit non-zero when a language was indexed but parsed no files') + .action(async (pathArg: string | undefined, options: { json?: boolean; strict?: boolean }) => { const projectPath = resolveProjectPath(pathArg); // The directory the user actually ran from, before walking up to the index // root. Used to detect when the resolved index lives in a different git @@ -975,6 +978,26 @@ program const backend = cg.getBackend(); const journalMode = cg.getJournalMode(); + // Languages whose files were all indexed and none parsed. The headline + // file count is keyed on the extension, so it stays reassuringly + // non-zero when a language's grammar is missing entirely — every obvious + // check (init succeeds, status lists the language, a symbol lookup + // "runs") stays green while the language contributes nothing to the + // graph. Restricted to grammar-backed languages: config formats are + // tracked at file level ON PURPOSE and must not raise this. + // Imported here rather than at module scope: grammars.ts pulls in + // web-tree-sitter, which every other command would then pay for at + // startup. + const { hasGrammar } = await import('../extraction/grammars'); + const parsedByLanguage = stats.parsedFilesByLanguage ?? ({} as Record); + const unparsedLanguages = Object.entries(stats.filesByLanguage) + .filter(([lang, count]) => + count > 0 && + (parsedByLanguage[lang as Language] ?? 0) === 0 && + hasGrammar(lang as Language) + ) + .map(([lang]) => lang); + const buildInfo = cg.getIndexBuildInfo(); const reindexRecommended = cg.isIndexStale(); const indexState = cg.getIndexState(); @@ -1000,6 +1023,10 @@ program journalMode, nodesByKind: stats.nodesByKind, languages: Object.entries(stats.filesByLanguage).filter(([, count]) => count > 0).map(([lang]) => lang), + filesByLanguage: stats.filesByLanguage, + // Files that yielded a symbol, vs the extension-keyed count above. + parsedFilesByLanguage: stats.parsedFilesByLanguage, + unparsedLanguages, pendingChanges: { added: changes.added.length, modified: changes.modified.length, @@ -1089,16 +1116,30 @@ program } console.log(); - // Language breakdown + // Language breakdown. The file count comes from the EXTENSION, so it says + // nothing about whether anything parsed — show the parsed count beside it + // so a language that produced no symbols is visible at a glance. console.log(chalk.bold('Files by Language:')); const filesByLang = Object.entries(stats.filesByLanguage) .filter(([, count]) => count > 0) .sort((a, b) => b[1] - a[1]); for (const [lang, count] of filesByLang) { - console.log(` ${lang.padEnd(15)} ${formatNumber(count)}`); + const parsed = parsedByLanguage[lang as Language] ?? 0; + const detail = parsed === count ? '' : ` ${parsed} parsed`; + const line = ` ${lang.padEnd(15)} ${formatNumber(count)}`; + console.log(unparsedLanguages.includes(lang) ? chalk.yellow(line + detail) : line + chalk.dim(detail)); } console.log(); + if (unparsedLanguages.length > 0) { + warn( + `no symbols were extracted from any ${unparsedLanguages.join(' / ')} file. ` + + 'Those files are indexed but their contents are not searchable — usually a ' + + 'missing or failed grammar for that language.' + ); + console.log(); + } + // Pending changes const totalChanges = changes.added.length + changes.modified.length + changes.removed.length; if (totalChanges > 0) { @@ -1128,6 +1169,10 @@ program } cg.destroy(); + // Opt-in gate for packaging/CI: a language that parsed nothing is a + // broken build, but making it fail by default would break every existing + // caller of `status`. + if (options.strict && unparsedLanguages.length > 0) process.exit(1); } catch (err) { error(`Failed to get status: ${err instanceof Error ? err.message : String(err)}`); process.exit(1); diff --git a/src/db/queries.ts b/src/db/queries.ts index af19b14cf..7636cc744 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -2834,6 +2834,23 @@ export class QueryBuilder { filesByLanguage[row.language as Language] = row.count; } + // Files that produced a symbol. Every indexed file gets a `file` node + // whether or not it parsed, so that one is excluded — the presence of + // anything else is what distinguishes a parsed file from a merely seen one. + const parsedFilesByLanguage = {} as Record; + const parsedRows = this.db + .prepare( + `SELECT f.language, COUNT(*) as count FROM files f + WHERE EXISTS ( + SELECT 1 FROM nodes n WHERE n.file_path = f.path AND n.kind <> 'file' + ) + GROUP BY f.language` + ) + .all() as Array<{ language: string; count: number }>; + for (const row of parsedRows) { + parsedFilesByLanguage[row.language as Language] = row.count; + } + return { nodeCount: counts.node_count, edgeCount: counts.edge_count, @@ -2841,6 +2858,7 @@ export class QueryBuilder { nodesByKind, edgesByKind, filesByLanguage, + parsedFilesByLanguage, dbSizeBytes: 0, // Set by caller using DatabaseConnection.getSize() walSizeBytes: 0, // Set by caller using DatabaseConnection.getWalSizeBytes() lastUpdated: Date.now(), diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 84647c3e4..a56ca8d26 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -172,6 +172,20 @@ export const EXTENSION_MAP: Record = { '.tofu': 'terraform', }; +/** + * Whether this language is backed by a grammar, i.e. whether files of this + * language are EXPECTED to yield symbols. + * + * Several indexed languages track files without extracting symbols from them + * (yaml/twig/xml config, and the SFC wrappers that delegate their script + * content elsewhere). Distinguishing them matters for any check on "this + * language produced nothing": for a grammar language that is a broken parse, + * for the others it is business as usual. + */ +export function hasGrammar(language: Language): boolean { + return language in WASM_GRAMMAR_FILES; +} + /** * Whether a file is one CodeGraph can parse, based purely on its extension. * This is the single source of truth for "should we index this file" — derived diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 5c23f675d..37a6d6c05 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -6464,10 +6464,19 @@ export class ToolHandler { } } + // The file count is keyed on the EXTENSION, so it stays non-zero for a + // language nothing could parse. Show the parsed count whenever it differs, + // so "indexed" is never mistaken for "searchable". lines.push('', '**Languages:**'); + const parsedByLanguage = stats.parsedFilesByLanguage ?? {}; for (const [lang, count] of Object.entries(stats.filesByLanguage)) { if ((count as number) > 0) { - lines.push(`- ${lang}: ${count}`); + const parsed = (parsedByLanguage as Record)[lang] ?? 0; + lines.push( + parsed === count + ? `- ${lang}: ${count}` + : `- ${lang}: ${count} files, ${parsed} with extracted symbols` + ); } } diff --git a/src/types.ts b/src/types.ts index 186f57adc..def7f5f82 100644 --- a/src/types.ts +++ b/src/types.ts @@ -578,9 +578,20 @@ export interface GraphStats { /** Edge counts by kind */ edgesByKind: Record; - /** File counts by language */ + /** File counts by language, keyed on the file's detected language. */ filesByLanguage: Record; + /** + * Files that actually yielded a symbol, by language. + * + * `filesByLanguage` is derived from the file EXTENSION, so it counts a file + * the parser never understood exactly the same as one it parsed perfectly — + * a language whose grammar is missing still reports its full file count. This + * is the count that separates *seen* from *parsed*, and the gap between the + * two is the only cheap signal that a language's extraction is broken. + */ + parsedFilesByLanguage: Record; + /** Database size in bytes */ dbSizeBytes: number;