diff --git a/CHANGELOG.md b/CHANGELOG.md index 392bf802b..581b20475 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `codegraph index` no longer stops with advice to run `codegraph daemon stop`, a command that doesn't exist, when the project's background-server lock names a process that isn't CodeGraph, such as a lock left by a crash whose process number the system has since given to another program: it now clears that stale lock, leaving the process alone, and rebuilds, and `codegraph unlock` clears it too. When a CodeGraph process that doesn't answer holds the lock, such as an older background server, the message names that process and the command that stops it, and the message about another server already serving a project now points to `codegraph daemon`. Thanks @inth3shadows for the report and @danusha2345 for the first fix. (#2770) - A program that uses CodeGraph as a library and indexes chosen files with `indexFiles()` now keeps other CodeGraph processes, like a `codegraph sync` run by a git hook, out of the index until those files are stored. Before, it let go of the write lock that keeps two writers apart as soon as indexing started, so another process could write to the index at the same time. - The shared background server is no longer killed with "Main thread unresponsive" while it brings another project's index up to date, such as one a session names with `projectPath`: only its own project's index counted as progress, so a long update of any other looked like a hang, and the server was stopped and restarted again and again. Every index the server keeps up to date now counts, and so does the one a session updates on its own when the background server is unavailable. Thanks @mradwankhalil for the report and the first fix. (#2404) +- A long-running session, like the background server behind your agent, now notices a framework that arrives or goes while it runs, such as Express in a new `package.json`, or one app replaced by another with as many files. Files it indexes from then on are read with the frameworks the project has now; before, they were read with the ones it had when the session started, so a new Express app's routes were missing until a full `codegraph index`. Files indexed before the change are not read again, so `codegraph status` now says which frameworks changed since the last full index and recommends running one. Thanks @nk33-dev for the report and the first fix. (#2516, #2519) - A project a session queries through `projectPath` is now let go once it has gone its own 10 minutes without a query. Before, when its index was still being brought up to date as the session went on to query another project, it was held, locked, until that other project's 10 minutes were up, so its own session and `codegraph index` could wait nearly twice as long to take over. Thanks @bompus. (#2087) - An index left incomplete by an indexing run that was stopped partway, by a crash, an out-of-memory kill or the server's own hang check, no longer gets the background server killed with "Main thread unresponsive" every time it starts: on a large project, restoring the database indexes the run had set aside took longer than the hang check allows, so each restart died at the same point and the index was never repaired. They are now restored in the background, and the project is served once they are back. Thanks @cbeaulieu-gt for the report and the first fix. (#1887) - A new agent session no longer waits up to about 12 minutes for its first answer when the background server is stuck: a server whose process still holds its connection open but never responds made the session wait out a long series of retries. The session now gives up within seconds and answers on its own, and switches back to the background server once that has been restarted. diff --git a/__tests__/sync-framework-detection.test.ts b/__tests__/sync-framework-detection.test.ts new file mode 100644 index 000000000..1ff54e23f --- /dev/null +++ b/__tests__/sync-framework-detection.test.ts @@ -0,0 +1,227 @@ +/** + * A long-lived graph keeps framework detection current across syncs (#2516, + * #2519). + * + * The MCP server's watcher holds one CodeGraph for the whole session. Its + * framework detection used to be cached from the first time it ran, so a + * framework that arrived mid-session, like Express in a new `package.json`, + * was missing from every file synced after it until `codegraph index`. A sync + * now detects again when what detection read changed, without walking the + * project on a watcher's scoped sync, and says when files indexed earlier + * were read without the change. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as cp from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { detectFrameworks } from '../src/resolution/frameworks'; +import { clearDeclaredDependencies, declaredDependencies } from '../src/resolution/frameworks/package-deps'; +import type { ResolutionContext } from '../src/resolution/types'; + +// Pass-through wrappers, so a test can count framework detections and see the git commands run. +vi.mock('child_process', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, execFileSync: vi.fn(actual.execFileSync) }; +}); +vi.mock('../src/resolution/frameworks', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, detectFrameworks: vi.fn(actual.detectFrameworks) }; +}); + +const EXPRESS = JSON.stringify({ dependencies: { express: '^4.18.0' } }); +const ROUTES = "const app = express();\napp.get('/hello', function hello(req, res) {});\n"; + +describe('framework detection in a long-lived graph (#2516)', () => { + let root: string; + let cg: CodeGraph | undefined; + const write = (rel: string, body: string) => { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), body); + }; + const routes = (rel: string) => cg!.getNodesInFile(rel).filter((n) => n.kind === 'route').map((n) => n.name); + + beforeEach(() => { + root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-frameworks-'))); + }); + + afterEach(() => { + cg?.close(); + cg = undefined; + fs.rmSync(root, { recursive: true, force: true }); + }); + + it.each(['full', 'scoped'] as const)('reads a file added with a new framework using it (%s sync)', async (mode) => { + write('src/app.ts', 'export function boot() { return 1; }\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + // An ordinary edit first, so this graph has detected (no frameworks) and cached it. + write('src/app.ts', 'export function boot() { return 2; }\n'); + await cg.sync(mode === 'scoped' ? { paths: ['src/app.ts'] } : {}); + expect(cg.getDetectedFrameworks()).not.toContain('express'); + + write('package.json', EXPRESS); + write('src/routes.js', ROUTES); + await cg.sync(mode === 'scoped' ? { paths: ['src/routes.js'] } : {}); + expect(cg.getDetectedFrameworks()).toContain('express'); + expect(routes('src/routes.js')).toEqual(['GET /hello']); + }); + + it('detects again when a file is added, with no manifest change', async () => { + write('src/app.ts', 'export function boot() { return 1; }\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(cg.getDetectedFrameworks()).not.toContain('svelte'); + // Svelte is detected from a .svelte file alone. + write('src/Widget.svelte', '\n\n'); + await cg.sync({ paths: ['src/Widget.svelte'] }); + expect(cg.getDetectedFrameworks()).toContain('svelte'); + }); + + it('detects again when a source file detection read is edited', async () => { + write('src/app.ts', 'export function boot() { return 1; }\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(cg.getDetectedFrameworks()).not.toContain('react-native-bridge'); + // The bridge is detected from a TurboModule spec in the first files' source. + write('src/app.ts', "export const Native = TurboModuleRegistry.getEnforcing('Native');\n"); + await cg.sync({ paths: ['src/app.ts'] }); + expect(cg.getDetectedFrameworks()).toContain('react-native-bridge'); + }); + + it('notices a manifest edit at the next sync, and says what the files indexed before it lack', async () => { + write('src/server.js', ROUTES); + write('src/util.js', 'function util() { return 1; }\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(routes('src/server.js')).toEqual([]); + expect(cg.getFrameworkChangesSinceIndex()).toBeNull(); + + // No source file changes with the manifest; the next edit anywhere syncs it. + write('package.json', EXPRESS); + write('src/util.js', 'function util() { return 2; }\n'); + await cg.sync({ paths: ['src/util.js'] }); + expect(cg.getDetectedFrameworks()).toContain('express'); + // server.js was indexed without Express and is unchanged: status recommends a re-index. + expect(routes('src/server.js')).toEqual([]); + expect(cg.getFrameworkChangesSinceIndex()).toEqual({ added: ['express'], removed: [] }); + + // Going back to what the index was built with clears it. + write('package.json', JSON.stringify({ dependencies: {} })); + write('src/util.js', 'function util() { return 3; }\n'); + await cg.sync({ paths: ['src/util.js'] }); + expect(cg.getDetectedFrameworks()).not.toContain('express'); + expect(cg.getFrameworkChangesSinceIndex()).toBeNull(); + + // A full re-index reads every file with the new frameworks, and the note goes. + write('package.json', EXPRESS); + cg.close(); + cg = await CodeGraph.recreate(root); + await cg.indexAll(); + expect(routes('src/server.js')).toEqual(['GET /hello']); + expect(cg.getFrameworkChangesSinceIndex()).toBeNull(); + }); +}); + +describe('a scoped sync in a fresh process keeps off the full scan (#2516)', () => { + let root: string; + let cg: CodeGraph | undefined; + const git = (...args: string[]) => cp.execFileSync('git', args, { cwd: root, stdio: 'pipe' }); + // The full scan's untracked listing. (The scope matcher's embedded-repo + // discovery lists with `--directory`, once per process, by design.) + const scans = () => vi.mocked(cp.execFileSync).mock.calls + .filter(([cmd, args]) => cmd === 'git' && Array.isArray(args) && args.join(' ') === 'ls-files -z -o --exclude-standard'); + + beforeEach(() => { + root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-noscan-'))); + }); + + afterEach(() => { + cg?.close(); + cg = undefined; + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('detects from the index, once, and not again while nothing it read changed', async () => { + git('init', '-q'); + git('config', 'user.email', 'test@example.invalid'); + git('config', 'user.name', 'Test'); + fs.writeFileSync(path.join(root, '.gitignore'), '.codegraph/\n'); + fs.writeFileSync(path.join(root, 'package.json'), EXPRESS); + fs.mkdirSync(path.join(root, 'src')); + fs.writeFileSync(path.join(root, 'src', 'routes.js'), ROUTES); + git('add', '-A'); + git('commit', '-qm', 'init'); + const built = CodeGraph.initSync(root); + await built.indexAll(); + built.close(); + + // The watcher's process: open, then sync one edited file at a time. + cg = CodeGraph.openSync(root); + vi.mocked(cp.execFileSync).mockClear(); + vi.mocked(detectFrameworks).mockClear(); + for (const n of [2, 3]) { + fs.writeFileSync(path.join(root, 'src', 'routes.js'), ROUTES + `// edit ${n}\n`); + await cg.sync({ paths: ['src/routes.js'] }); + } + expect(scans()).toEqual([]); + expect(vi.mocked(detectFrameworks)).toHaveBeenCalledTimes(1); + expect(cg.getNodesInFile('src/routes.js').filter((n) => n.kind === 'route').map((n) => n.name)).toEqual(['GET /hello']); + }); +}); + +describe('declared dependencies in a long-lived graph (#2519)', () => { + let root: string; + let cg: CodeGraph | undefined; + const write = (rel: string, body: string) => { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), body); + }; + + beforeEach(() => { + root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-deps-'))); + }); + + afterEach(() => { + cg?.close(); + cg = undefined; + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('sees one app swapped for another with as many files', async () => { + write('web/package.json', JSON.stringify({ dependencies: { react: '^18.0.0' } })); + write('web/a.js', 'export function a() { return 1; }\n'); + write('web/b.js', 'export function b() { return 2; }\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(cg.getDetectedFrameworks()).toContain('react'); + + fs.rmSync(path.join(root, 'web'), { recursive: true, force: true }); + write('server/package.json', EXPRESS); + write('server/a.js', ROUTES); + write('server/b.js', 'export function b() { return 2; }\n'); + await cg.sync(); + expect(cg.getDetectedFrameworks()).toContain('express'); + expect(cg.getDetectedFrameworks()).not.toContain('react'); + cg.reinitializeResolver(); + expect(cg.getDetectedFrameworks()).toContain('express'); + expect(cg.getDetectedFrameworks()).not.toContain('react'); + }); + + it('reads the manifests again once told to forget them', () => { + write('package.json', JSON.stringify({ dependencies: { react: '^18.0.0' } })); + const files = ['a.js']; + const context = { + getAllFiles: () => files, + fileExists: (rel: string) => fs.existsSync(path.join(root, rel)), + readFile: (rel: string) => (fs.existsSync(path.join(root, rel)) ? fs.readFileSync(path.join(root, rel), 'utf8') : null), + } as unknown as ResolutionContext; + expect([...declaredDependencies(context)]).toEqual(['react']); + write('package.json', EXPRESS); + // Same file count: the count alone keeps the old answer. + expect([...declaredDependencies(context)]).toEqual(['react']); + clearDeclaredDependencies(context); + expect([...declaredDependencies(context)]).toEqual(['express']); + }); +}); diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index c4bbc4eb4..b041843ff 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -1167,6 +1167,9 @@ program const buildInfo = cg.getIndexBuildInfo(); const reindexRecommended = cg.isIndexStale(); + // Frameworks a sync detected since the last full index (#2516): files + // indexed before then were extracted without them. + const frameworkChanges = cg.getFrameworkChangesSinceIndex(); const indexState = cg.getIndexState(); // Zero on a healthy index; non-zero at rest means a resolution pass was // interrupted, so some files' call edges are missing (#1187). @@ -1208,7 +1211,9 @@ program builtWithVersion: buildInfo.version, builtWithExtractionVersion: buildInfo.extractionVersion, currentExtractionVersion: EXTRACTION_VERSION, - reindexRecommended, + reindexRecommended: reindexRecommended || frameworkChanges !== null, + // { added, removed } frameworks since the last full index, or null. + frameworksChangedSinceIndex: frameworkChanges, // 'complete' | 'partial' (files silently dropped) | 'indexing' // (a run was killed mid-index — the index is truncated) | // 'failed' | null (predates the marker). @@ -1343,6 +1348,15 @@ program info('Run "codegraph index" (full rebuild) or "codegraph sync"'); console.log(); } + if (frameworkChanges) { + const changed = [ + ...(frameworkChanges.added.length > 0 ? [`added ${frameworkChanges.added.join(', ')}`] : []), + ...(frameworkChanges.removed.length > 0 ? [`removed ${frameworkChanges.removed.join(', ')}`] : []), + ].join('; '); + warn(`Frameworks changed since the last full index (${changed}); files indexed before then were read without the change.`); + info('Run "codegraph index" to re-index every file with them'); + console.log(); + } cg.destroy(); } catch (err) { diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 5b8d8cffa..6096864c8 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -166,6 +166,12 @@ export interface SyncResult { definitionDelta?: string[]; /** Tracked files this sync found gone and deleted from the index. */ removedFilePaths?: string[]; + /** + * The frameworks extraction detected, when this sync had to detect them + * again because what detection read had changed (#2516). Absent when the + * cached detection still held. + */ + detectedFrameworks?: string[]; } /** @@ -190,6 +196,27 @@ function readSourceOrStamp(fullPath: string): string | null { return isMpegTsBytes(fullPath, bytes) ? null : bytes.toString('utf8'); } +/** + * What framework detection found at a path, to tell later whether it changed: + * a file's size and modification time, a folder's sub-folders, or '-' when + * nothing is there. + */ +function diskSignature(fullPath: string, isFolder = false): string { + try { + if (isFolder) { + return fs.readdirSync(fullPath, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort() + .join('/'); + } + const stats = fs.statSync(fullPath, { throwIfNoEntry: false }); + return stats ? `${stats.size}:${stats.mtimeMs}` : '-'; + } catch { + return '-'; + } +} + /** Whether these bytes, read from `filePath`, are a `.ts` MPEG transport stream (#1910). */ function isMpegTsBytes(filePath: string, bytes: Buffer): boolean { if (!hasMpegTsExtension(filePath) || !isMpegTransportStream(bytes.subarray(0, MPEG_TS_SNIFF_BYTES))) return false; @@ -2042,6 +2069,23 @@ export class ExtractionOrchestrator { * hasn't run yet so single-file re-index paths can detect on the spot. */ private detectedFrameworkNames: string[] | null = null; + /** + * What the last detection read, so a sync can tell when it went stale + * (#2516): the indexed source files it read or probed, which a sync's own + * change lists cover, and every other path with what was there then + * ({@link diskSignature}): the manifests, like `package.json`, that no sync + * reports. Null until detection runs. + */ + private detectionInputs: { sourceFiles: Set; diskPaths: Map } | null = null; + /** Files were added or removed since the last detection: the file list it read is out of date. */ + private detectionFileListChanged = false; + /** + * What detection read, by project-relative path, kept between detections: a + * source file until a sync changes it, any other path while its signature + * holds. Detecting again then reads only what changed, not the couple of + * hundred files some detectors sample (react-native-bridge reads up to 200). + */ + private detectionReads = new Map(); /** * Scope matcher for SCOPED syncs, memoized on the mtimes of the two root * files it is derived from (`codegraph.json`, `.gitignore`). See @@ -2098,9 +2142,30 @@ export class ExtractionOrchestrator { * detection. Graph-query methods (getNodesByName etc.) return empty because * the DB hasn't been populated yet, but detect() only uses readFile, * fileExists, and getAllFiles, so that's fine. + * + * With `inputs`, every path a detector reads or probes is recorded there, + * and answered from {@link detectionReads} when it was read before. */ - private buildDetectionContext(files: string[]): ResolutionContext { + private buildDetectionContext(files: string[], inputs?: NonNullable): ResolutionContext { const rootDir = this.rootDir; + const listed = inputs ? new Set(files) : null; + const reads = inputs ? this.detectionReads : null; + // What is at a path, once per path until a sync says it changed. + const probe = (relativePath: string): { full: string; signature: string; content?: string | null } | null => { + const rel = normalizePath(relativePath); + let known = reads?.get(rel); + if (!known) { + const full = validatePathWithinRoot(rootDir, relativePath); + if (!full) return null; + known = { full, signature: diskSignature(full) }; + reads?.set(rel, known); + } + if (inputs) { + if (listed!.has(rel)) inputs.sourceFiles.add(rel); + else if (!inputs.diskPaths.has(rel)) inputs.diskPaths.set(rel, known.signature); + } + return known; + }; return { getNodesInFile: () => [], getNodesByName: () => [], @@ -2111,24 +2176,22 @@ export class ExtractionOrchestrator { getAllFiles: () => files, getProjectRoot: () => rootDir, fileExists: (relativePath: string) => { - const full = validatePathWithinRoot(rootDir, relativePath); - if (!full) return false; - try { - return fs.existsSync(full); - } catch { - return false; - } + const known = probe(relativePath); + return known !== null && known.signature !== '-'; }, readFile: (relativePath: string) => { - const full = validatePathWithinRoot(rootDir, relativePath); - if (!full) return null; - try { - // Framework detectors scan source by name; a file over the size - // limit was never indexed and must not be decoded here either (#1910). - return readBoundedSourceSync(full).bytes?.toString('utf8') ?? null; - } catch { - return null; + const known = probe(relativePath); + if (!known) return null; + if (known.content === undefined) { + try { + // Framework detectors scan source by name; a file over the size + // limit was never indexed and must not be decoded here either (#1910). + known.content = readBoundedSourceSync(known.full).bytes?.toString('utf8') ?? null; + } catch { + known.content = null; + } } + return known.content; }, // Monorepo support — needed by framework detect()s that probe // subpackage manifests (e.g. fabric-view looking at @@ -2139,6 +2202,11 @@ export class ExtractionOrchestrator { relativePath === '.' || relativePath === '' ? rootDir : path.join(rootDir, relativePath); + // A folder appearing or going changes what a monorepo probe sees. + if (inputs) { + const key = normalizePath(path.relative(rootDir, target)) + '/'; + if (!inputs.diskPaths.has(key)) inputs.diskPaths.set(key, diskSignature(target, true)); + } try { return fs .readdirSync(target, { withFileTypes: true }) @@ -2154,14 +2222,22 @@ export class ExtractionOrchestrator { /** * Detect frameworks on demand using the current scanned files (or a fresh * scan if none are provided). Cached on the orchestrator so repeat calls - * inside a single run don't re-scan. + * inside a single run don't re-scan; a sync drops the cache when what it + * read has changed ({@link refreshFrameworkDetection}). */ private ensureDetectedFrameworks(files?: string[]): string[] { if (this.detectedFrameworkNames !== null) return this.detectedFrameworkNames; const fileList = files ?? scanDirectory(this.rootDir); - const context = this.buildDetectionContext(fileList); + const inputs = { sourceFiles: new Set(), diskPaths: new Map() }; + const context = this.buildDetectionContext(fileList, inputs); + this.detectionInputs = inputs; + this.detectionFileListChanged = false; this.detectedFrameworkNames = detectFrameworks(context).map((r) => r.name); const declared = declaredDependencies(context); + // Keep only what this detection read. + for (const rel of this.detectionReads.keys()) { + if (!inputs.sourceFiles.has(rel) && !inputs.diskPaths.has(rel)) this.detectionReads.delete(rel); + } this.gatedFrameworks = new Map(); for (const name of this.detectedFrameworkNames) { const deps = getFrameworkResolver(name)?.appDependencies; @@ -2221,8 +2297,12 @@ export class ExtractionOrchestrator { private dependenciesDeclaredIn(dir: string): Set | null { if (this.manifestDependencies.has(dir)) return this.manifestDependencies.get(dir)!; let declared: Set | null = null; + const manifest = path.join(this.rootDir, dir, 'package.json'); + // Kept until the next detection, so an edit of this manifest must bring one about. + const rel = dir ? `${dir}/package.json` : 'package.json'; + if (this.detectionInputs && !this.detectionInputs.diskPaths.has(rel)) this.detectionInputs.diskPaths.set(rel, diskSignature(manifest)); try { - const pkg = JSON.parse(fs.readFileSync(path.join(this.rootDir, dir, 'package.json'), 'utf8')) as Record; + const pkg = JSON.parse(fs.readFileSync(manifest, 'utf8')) as Record; declared = new Set(); for (const field of ['dependencies', 'devDependencies', 'peerDependencies']) { const group = pkg[field]; @@ -2233,6 +2313,43 @@ export class ExtractionOrchestrator { return declared; } + /** The frameworks extraction last detected, or null before any detection. */ + getDetectedFrameworkNames(): string[] | null { + return this.detectedFrameworkNames; + } + + /** + * Bring framework detection up to date before a sync extracts `filesToIndex` + * (#2516). Detection is cached for the life of this orchestrator, so in a + * long-lived process (the MCP server's watcher) a framework that arrives + * mid-session, like Express in a new `package.json`, would otherwise be + * missing from every file synced after it. It runs again only when what it + * read changed: files were added or removed since, a source file it read is + * among those changed, or a manifest it read or probed changed on disk. + * Otherwise this costs a set lookup per changed file and a stat per manifest, + * and when it runs, it reads again only what changed ({@link detectionReads}). + * + * `scannedFiles` is a full reconcile's scan. A scoped sync never walks the + * project: the file list is the index's, which already holds this sync's + * removals, plus the files it adds. Returns the frameworks detected when + * detection ran. + */ + private refreshFrameworkDetection(scannedFiles: string[] | null, filesToIndex: string[], addedFiles: string[], removedFiles: string[]): string[] | undefined { + for (const file of filesToIndex) this.detectionReads.delete(file); + for (const file of removedFiles) this.detectionReads.delete(file); + const inputs = this.detectionInputs; + let stale = this.detectedFrameworkNames === null || inputs === null || this.detectionFileListChanged || + filesToIndex.some((f) => inputs.sourceFiles.has(f)); + for (const [rel, signature] of inputs?.diskPaths ?? []) { + if (diskSignature(path.join(this.rootDir, rel), rel.endsWith('/')) === signature) continue; + stale = true; + this.detectionReads.delete(rel); + } + if (!stale) return undefined; + this.detectedFrameworkNames = null; + return this.ensureDetectedFrameworks(scannedFiles ?? [...this.queries.getAllFilePaths(), ...addedFiles]); + } + /** * Index all files in the project */ @@ -2315,8 +2432,10 @@ export class ExtractionOrchestrator { // Names are passed to each parse call so framework-specific extractors // (route nodes, middleware, etc.) run after the tree-sitter pass. // Framework detection is reset each run so adding e.g. requirements.txt - // between runs is picked up without restarting the process. + // between runs is picked up without restarting the process. It reads + // everything afresh: no sync has told this process what changed on disk. this.detectedFrameworkNames = null; + this.detectionReads.clear(); const tFw = Date.now(); const frameworkNames = this.ensureDetectedFrameworks(files); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] framework-detect: ${Date.now() - tFw}ms`); @@ -3830,6 +3949,15 @@ export class ExtractionOrchestrator { for (const pair of this.queries.getNodeNamePairsByFiles(filesToIndex)) pairsBefore.add(pair); } + // The frameworks every file below is extracted with, current (#2516). + if (addedFilePaths.length > 0 || removedFilePaths.length > 0) this.detectionFileListChanged = true; + const tDetect = Date.now(); + const detectedFrameworks = filesToIndex.length === 0 ? undefined : this.refreshFrameworkDetection( + scopedPaths && scopedPaths.length > 0 ? null : currentFiles, filesToIndex, addedFilePaths, removedFilePaths); + if (process.env.CODEGRAPH_SYNTH_TIMINGS && filesToIndex.length > 0) { + console.error(`[phase-timing] sync-framework-detect: ${Date.now() - tDetect}ms (${detectedFrameworks ? 'detected again' : 'unchanged'})`); + } + // Load only grammars needed for changed files if (filesToIndex.length > 0) { const overrides = loadExtensionOverrides(this.rootDir); @@ -3886,6 +4014,7 @@ export class ExtractionOrchestrator { ...(failedFilePaths.length > 0 ? { failedFilePaths } : {}), definitionDelta: definitionDelta.length > 0 ? definitionDelta : undefined, ...(removedFilePaths.length > 0 ? { removedFilePaths } : {}), + ...(detectedFrameworks ? { detectedFrameworks } : {}), }; } diff --git a/src/index.ts b/src/index.ts index 5d9c25ce9..ae27c17aa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -153,6 +153,24 @@ export interface IndexOptions { const PYTHON_ROOTS_METADATA_KEY = 'python_root_fingerprint'; /** Metadata key: the digest of the workspace packages and path dependencies the index was resolved by. */ const WORKSPACE_METADATA_KEY = 'js_workspace_fingerprint'; +/** Metadata key: the frameworks (sorted JSON names) the last full index extracted every file with. */ +const FRAMEWORKS_AT_INDEX_KEY = 'frameworks_at_index'; +/** + * Metadata key: `{ added, removed }` — the frameworks a sync detected since the + * last full index, against {@link FRAMEWORKS_AT_INDEX_KEY}; '' when none. + */ +const FRAMEWORKS_SINCE_INDEX_KEY = 'frameworks_since_index'; + +/** A JSON list of names from metadata, or null when absent or unreadable. */ +function parseNameList(value: string | null): string[] | null { + if (!value) return null; + try { + const parsed: unknown = JSON.parse(value); + return Array.isArray(parsed) && parsed.every((n) => typeof n === 'string') ? parsed : null; + } catch { + return null; + } +} export class CodeGraph { private db: DatabaseConnection; @@ -799,6 +817,7 @@ export class CodeGraph { // extraction stamp (the bulk would still be stale). See extraction-version.ts. if (result.success) this.stampPythonRootFingerprint(); if (result.success) this.stampWorkspaceFingerprint(); + if (result.success) this.stampIndexedFrameworks(); if (result.success && result.filesIndexed > 0) { try { @@ -923,6 +942,64 @@ export class CodeGraph { } catch { /* metadata is advisory */ } } + /** Record the frameworks a full index extracted every file with; none has come or gone since. */ + private stampIndexedFrameworks(): void { + const names = this.orchestrator.getDetectedFrameworkNames(); + if (names === null) return; + try { + this.queries.setMetadata(FRAMEWORKS_AT_INDEX_KEY, JSON.stringify([...names].sort())); + this.queries.setMetadata(FRAMEWORKS_SINCE_INDEX_KEY, ''); + } catch { /* metadata is advisory */ } + } + + /** + * A sync detected frameworks again because what detection read had changed + * (#2516): a framework arrived mid-session, like Express in a new + * `package.json`, or one left. Files the sync extracted were read with the + * new set already. The resolver detects again when its set differs, so + * resolution and the passes after it use the new one too. Files indexed + * before the change were extracted without it, so the change against the + * last full index is recorded for `codegraph status`, which recommends a + * re-index: re-extracting every file a framework can touch would mean + * re-parsing most of the project inside a watcher's sync. + */ + private followDetectedFrameworks(detected: string[]): void { + const now = [...detected].sort(); + const resolverNow = [...this.resolver.getDetectedFrameworks()].sort(); + if (now.join('\0') !== resolverNow.join('\0')) this.resolver.initialize(); + try { + const atIndex = parseNameList(this.queries.getMetadata(FRAMEWORKS_AT_INDEX_KEY)); + if (!atIndex) return; // indexed before this was recorded: nothing to compare with + const added = now.filter((name) => !atIndex.includes(name)); + const removed = atIndex.filter((name) => !now.includes(name)); + const value = added.length > 0 || removed.length > 0 ? JSON.stringify({ added, removed }) : ''; + if ((this.queries.getMetadata(FRAMEWORKS_SINCE_INDEX_KEY) ?? '') !== value) { + this.queries.setMetadata(FRAMEWORKS_SINCE_INDEX_KEY, value); + } + } catch { /* metadata is advisory */ } + } + + /** + * The frameworks a sync detected since the last full index, against the ones + * that index extracted every file with (#2516): files indexed before the + * change were read without the frameworks added, and with the ones removed, + * so a full re-index brings them in line. Null when nothing changed, or for + * an index built before this was recorded. + */ + getFrameworkChangesSinceIndex(): { added: string[]; removed: string[] } | null { + try { + const value = this.queries.getMetadata(FRAMEWORKS_SINCE_INDEX_KEY); + if (!value) return null; + const parsed = JSON.parse(value) as { added?: unknown; removed?: unknown }; + const names = (list: unknown): string[] => (Array.isArray(list) ? list.filter((n): n is string => typeof n === 'string') : []); + const added = names(parsed.added); + const removed = names(parsed.removed); + return added.length > 0 || removed.length > 0 ? { added, removed } : null; + } catch { + return null; + } + } + async sync(options: IndexOptions = {}): Promise { return this.indexMutex.withLock(async () => { const startedAt = Date.now(); @@ -1032,6 +1109,10 @@ export class CodeGraph { } }); + // Frameworks detected again because what detection read changed: the + // passes below use the new set (#2516). + if (result.detectedFrameworks) this.followDetectedFrameworks(result.detectedFrameworks); + // A component a tag in an unchanged file may render: `` links // to the `Team` this sync adds. The gate above reads each changed file // on its own, and a component with no markup of its own (`return null`, diff --git a/src/resolution/frameworks/package-deps.ts b/src/resolution/frameworks/package-deps.ts index 907c67943..e1a5c4bc4 100644 --- a/src/resolution/frameworks/package-deps.ts +++ b/src/resolution/frameworks/package-deps.ts @@ -31,9 +31,19 @@ const SCRIPT_FILE = /\.(?:[cm]?[jt]sx?|vue|svelte|astro)$/; * proshop-shaped repo indexed its React Router routes (extraction is not * gated on detection) and then resolved none of its navigation. Re-reading * when the file count changes costs one manifest sweep per index. + * + * The count alone misses a manifest edit and a swap of one app for another + * of the same size (#2519), so a long-lived context also forgets its answer + * with {@link clearDeclaredDependencies}, which the resolver calls with its + * other per-context memos, before it detects frameworks again. */ const cache = new WeakMap }>(); +/** Forget what `context`'s manifests declared, so the next question reads them again. */ +export function clearDeclaredDependencies(context: ResolutionContext): void { + cache.delete(context); +} + /** Every dependency name declared in the manifests read (see the file comment), de-duplicated. */ export function declaredDependencies(context: ResolutionContext): Set { const files = context.getAllFiles(); diff --git a/src/resolution/index.ts b/src/resolution/index.ts index 45538ccc9..516a93577 100644 --- a/src/resolution/index.ts +++ b/src/resolution/index.ts @@ -46,6 +46,7 @@ import { resolveViaImport, resolvePhpImportedStaticCall, resolvePhpQualifiedClas import { ResolverPool, minRefsForPool, shouldEngageAdaptively } from './resolver-pool'; import { resolveAliasBinding } from './alias-binding'; import { detectFrameworks, getResolvingFrameworks } from './frameworks'; +import { clearDeclaredDependencies } from './frameworks/package-deps'; import { synthesizeCallbackEdges } from './callback-synthesizer'; import { createYielder, type MaybeYield } from './cooperative-yield'; import { MAX_SOURCE_FILE_SIZE_BYTES } from '../file-limits'; @@ -389,9 +390,12 @@ export class ReferenceResolver { // A full index resolves by the package.json files on disk now, as the // fingerprint it stamps says, even in a process that read them earlier. this.workspacePackages = undefined; + // Detection reads through the caches (file contents, existence probes, + // declared dependencies): clear them first, or a process that read the + // manifests earlier detects by what they said then (#2519). + this.clearCaches(); this.frameworks = detectFrameworks(this.context); this.frameworksByLanguage.clear(); - this.clearCaches(); } /** The detected frameworks whose `resolve()` reads a reference written in `language`. */ @@ -625,6 +629,7 @@ export class ReferenceResolver { clearCppIncluderMemos(this.context); clearCFeatureVariantMemos(this.context); clearAmbientModuleMemos(this.context); + clearDeclaredDependencies(this.context); } }