diff --git a/CHANGELOG.md b/CHANGELOG.md index 59fdc88f0..58844f9a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -196,6 +196,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - In JavaScript and TypeScript, a call to a method whose name contains a `$` now links to that method through its receiver's type, like any other method call: RxJS-style `this.organizationService.organizations$(userId)`, or vscode's `this._proxy.$acceptModelChanged(…)` on the interface its RPC proxy is declared with. Before, such a call linked nothing, however well its receiver was typed. Re-index JavaScript and TypeScript projects after upgrading. - In JavaScript and TypeScript, an import written from a jsconfig or tsconfig `baseUrl`, like `import MainLayout from 'layout/MainLayout'` under `"baseUrl": "src"`, or through a catch-all `"paths": { "*": ["./src/*"] }`, now links to the file it names, read with the config of the app the importing file belongs to, and so does an import of a single name like `'config'` or `'themes'`. Create React App and Vite admin templates such as Berry, Mantis and MatX write nearly every import this way. Before, each one was taken for an npm package, so it linked nothing, its names were matched by name alone, which could reach a same-named component in another app of the same repository, and a React Router route whose page loads through `lazy(() => import('views/dashboard/Default'))` pointed at that loader instead of the page and its layout. A package the app's `package.json` declares, or a Node built-in, never links to a project module of the same name, only to type declarations the project keeps for it, and only a file the index holds counts, never a folder or something in `node_modules`. `codegraph sync` and the file watcher link such an import too when the file it names is added later or gains what it imports. Re-index JavaScript and TypeScript projects after upgrading. - `codegraph sync` and the file watcher now relink an import that reaches a module through a barrel file passing it on under another name, like `import { MatxLoading } from './components'` when `components/index.js` says `export { default as MatxLoading } from './MatxLoading'`, once an edit gives that module what is imported. Before, the import and every call through it stayed unlinked, or stayed on something else in the project with the same name, until the importing file itself changed or the project was indexed again. +- In JavaScript and TypeScript projects, a folder named `build` inside `src/` is now indexed for its TypeScript, JSX, Vue, Svelte and Astro files, so a feature kept there, like the `src/build/` pages of luci-go's milo, shows up in `codegraph_explore` and the routes that lazily load its pages link to them. Before, every folder named `build` was skipped as build output. The JavaScript, declaration and other files such a folder holds are still left out, since a build may have written them, and so is every `build` folder outside `src/`. ## [1.6.2] - 2026-10-03 diff --git a/README.md b/README.md index ae3750c3b..a3c2e1441 100644 --- a/README.md +++ b/README.md @@ -650,7 +650,10 @@ What it skips out of the box: - **Dependency, build, and cache directories** — `node_modules`, `vendor`, `dist`, `build`, `target`, `.venv`, `Pods`, `.next`, and the like across every [supported stack](#supported-languages) — so the graph is your code, not - third-party noise. This holds even with no `.gitignore`. + third-party noise. This holds even with no `.gitignore`. A `build` folder + inside `src/` is the exception: no build step writes TypeScript, JSX, Vue, + Svelte or Astro files, so those are indexed from it, and a Java, Kotlin or + Scala package named `build` is indexed whole. - **Anything in your `.gitignore`** — honored in git repos via git, and in non-git projects by reading `.gitignore` directly (root and nested). - **Files larger than 1 MB** — generated bundles, minified JS, vendored blobs. diff --git a/__tests__/react-router-lazy-barrel.test.ts b/__tests__/react-router-lazy-barrel.test.ts index 055276b1c..e60d5659c 100644 --- a/__tests__/react-router-lazy-barrel.test.ts +++ b/__tests__/react-router-lazy-barrel.test.ts @@ -4,8 +4,8 @@ * `invocation_page/index.ts`, whose `export * from './invocation_page'` * forwards the `Component` that `invocation_page.tsx` declares. The route * renders that component; before, the barrel was read for an export of its - * own, found none, and the route linked nothing. (The fixture's folder is - * `builds/`: a `build/` folder is skipped as build output.) + * own, found none, and the route linked nothing. (milo's own folder is + * `src/build/`; src-build-folders.test.ts covers routes into it.) */ import { describe, it, expect, afterAll, afterEach, beforeAll } from 'vitest'; import * as fs from 'fs'; diff --git a/__tests__/src-build-folders.test.ts b/__tests__/src-build-folders.test.ts new file mode 100644 index 000000000..89f51d9ba --- /dev/null +++ b/__tests__/src-build-folders.test.ts @@ -0,0 +1,271 @@ +/** + * A folder named `build` below `src/` is a source folder in a JS/TS app: + * luci-go's milo/ui keeps its whole LUCI "build" feature in `src/build/`, and + * its router lazily loads pages from `@/build/...`. The default ignore list + * skipped every folder named `build` as build output, so none of that feature + * was indexed and those routes linked nothing. + * + * Such a folder is now walked and its hand-written source (TypeScript, JSX, + * Vue, Svelte, Astro) indexed, while what a build step writes there — + * JavaScript, declarations, CSS, HTML, maps — stays out, as does every `build/` + * folder outside `src/`. Both enumeration paths, the git-status sync and the + * watcher share the rule (#1728). + */ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { buildDefaultIgnore, buildScopeIgnore, getGitChangedFiles, scanDirectory, scanDirectoryAsync } from '../src/extraction'; +import { FileWatcher, __emitWatchEventForTests } from '../src/sync/watcher'; + +const git = (cwd: string, ...args: string[]): void => { + execFileSync('git', args, { cwd, stdio: 'pipe' }); +}; + +let dirs: string[] = []; +let graphs: CodeGraph[] = []; +let watchers: FileWatcher[] = []; +afterEach(() => { + for (const watcher of watchers) watcher.stop(); + for (const graph of graphs) graph.close(); + for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true }); + watchers = []; + graphs = []; + dirs = []; +}); + +const tempDir = (): string => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-src-build-')); + dirs.push(dir); + return dir; +}; + +const write = (dir: string, files: Record): void => { + for (const [rel, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); + fs.writeFileSync(path.join(dir, rel), content); + } +}; + +const gitRepo = (dir: string): void => { + git(dir, 'init', '-q'); + git(dir, 'config', 'user.email', 'test@test.com'); + git(dir, 'config', 'user.name', 'Test'); +}; + +describe('the scope of a build folder below src', () => { + const SOURCES = [ + 'src/build/constants.ts', + 'src/build/legacy/build_page/build_page.tsx', + 'src/build/nested/build/deep.ts', + // Any case, like the other defaults (its own parent: Windows and macOS + // file systems would merge it into `src/build/`). + 'web/src/Build/upper_case.ts', + 'packages/app/src/core/build/runner.mts', + 'packages/app/src/core/build/loader.cts', + 'src/views/tool/build/index.vue', + 'src/routes/build/+page.svelte', + 'src/pages/build/BuildPage.jsx', + 'src/content/build/Card.astro', + // A JVM package named `build` keeps every file, as before (#1642). + 'src/main/java/com/acme/build/Script.js', + ]; + const IGNORED = [ + // What a build step writes, below src/ too. + 'src/build/bundle.js', + 'src/build/chunk.mjs', + 'src/build/legacy.cjs', + 'src/build/types/index.d.ts', + 'src/build/types/module.d.mts', + 'src/build/routes.yml', + 'src/client/build/static/js/main.4f3a2b1c.js', + 'src/main/resources/static/build/bundle.js', + 'src/main/resources/build/Generated.java', + // The other defaults still apply inside such a folder. + 'src/build/node_modules/pkg/index.ts', + 'src/build/dist/index.ts', + 'src/build/vendor/lib.ts', + // A build folder outside src/ stays excluded, whatever it holds. + 'build/index.ts', + 'build/static/js/main.4f3a2b1c.js', + 'packages/app/build/index.ts', + 'tools/build/release.ts', + ]; + + it.each(['filesystem', 'untracked', 'tracked'])( + 'indexes its hand-written source and nothing a build writes (%s)', + async (mode) => { + const root = tempDir(); + write(root, Object.fromEntries([...SOURCES, ...IGNORED].map((rel) => [rel, 'export const x = 1;\n']))); + write(root, { 'src/client/package.json': '{"name":"client"}' }); + if (mode !== 'filesystem') { + git(root, 'init', '-q'); + if (mode === 'tracked') git(root, 'add', '-f', '.'); + } + + const defaults = buildDefaultIgnore(root); + const scope = buildScopeIgnore(root); + const files = scanDirectory(root); + expect(await scanDirectoryAsync(root)).toEqual(files); + for (const rel of SOURCES) { + expect(defaults.ignores(rel), rel).toBe(false); + expect(scope.ignores(rel), rel).toBe(false); + // The watcher and the filesystem walk must reach every folder on the way. + const parts = rel.split('/'); + for (let i = 1; i < parts.length; i++) { + expect(scope.ignores(parts.slice(0, i).join('/') + '/'), rel).toBe(false); + } + expect(files).toContain(rel); + } + for (const rel of IGNORED) { + expect(defaults.ignores(rel), rel).toBe(true); + expect(scope.ignores(rel), rel).toBe(true); + expect(files).not.toContain(rel); + } + }, + ); + + it("follows the project's own rules first", () => { + const files = { + 'src/build/page.tsx': 'export const page = 1;\n', + 'src/build/bundle.js': 'export const bundle = 1;\n', + 'src/build/legacy/helper.js': 'export const helper = 1;\n', + 'src/build/node_modules/pkg/index.js': 'export const dep = 1;\n', + }; + const scope = (ignoreFile: string, rules: string): ReturnType => { + const root = tempDir(); + write(root, files); + gitRepo(root); + write(root, { [ignoreFile]: rules }); + return buildScopeIgnore(root); + }; + + // An explicit exclusion wins over the folder's source. + expect(scope('.gitignore', 'build/\n').ignores('src/build/page.tsx')).toBe(true); + expect(scope('codegraph.json', JSON.stringify({ exclude: ['src/build/'] })).ignores('src/build/page.tsx')).toBe(true); + + // Re-including the folder — the documented opt-in — keeps every type in it, as before. + const optedIn = scope('.gitignore', '!src/build/\n'); + expect(optedIn.ignores('src/build/bundle.js')).toBe(false); + expect(optedIn.ignores('src/build/legacy/helper.js')).toBe(false); + expect(optedIn.ignores('src/build/node_modules/pkg/index.js')).toBe(true); + + // So does re-including a folder inside it, or the file itself. + const legacy = scope('.gitignore', '!src/build/legacy/\n'); + expect(legacy.ignores('src/build/legacy/helper.js')).toBe(false); + expect(legacy.ignores('src/build/bundle.js')).toBe(true); + expect(scope('.gitignore', '!src/build/bundle.js\n').ignores('src/build/bundle.js')).toBe(false); + + // Without project rules, the folder's JavaScript stays out. + const root = tempDir(); + write(root, files); + expect(scanDirectory(root).sort()).toEqual(['src/build/page.tsx']); + }); +}); + +describe('a JS/TS feature kept in src/build', () => { + const FILES: Record = { + 'package.json': JSON.stringify({ name: 'ui', private: true, dependencies: { react: '^18', 'react-router': '^7' } }), + 'tsconfig.json': JSON.stringify({ compilerOptions: { baseUrl: '.', paths: { '@/*': ['./src/*'] } } }), + 'src/core/routes/routes.ts': `import { createBrowserRouter } from 'react-router'; + +export const router = createBrowserRouter([ + { path: '/p/:project/builders/:bucket/:builder', lazy: () => import('@/build/pages/builder_page') }, + { path: '/b/:buildId', lazy: () => import('@/build/legacy/build_page') }, +]); +`, + 'src/build/pages/builder_page/index.ts': `export * from './builder_page'; +`, + 'src/build/pages/builder_page/builder_page.tsx': `export function BuilderPage() { + return null; +} + +export function Component() { + return ; +} +`, + 'src/build/legacy/build_page/index.ts': `export * from './build_page'; +`, + 'src/build/legacy/build_page/build_page.tsx': `export const BuildPage = () => null; + +export function Component() { + return ; +} +`, + }; + + /** Each route's `references` edges, as `name@file`. */ + const routeLinks = (graph: CodeGraph): Record => Object.fromEntries( + graph.getNodesByKind('route').map((route) => [route.name, graph.getOutgoingEdges(route.id) + .filter((e) => e.kind === 'references') + .map((e) => graph.getNode(e.target)!) + .map((n) => `${n.name}@${n.filePath}`) + .sort()]) + ); + + it('is indexed, so its lazy routes link, and sync keeps it', async () => { + const root = tempDir(); + write(root, FILES); + gitRepo(root); + git(root, 'add', '.'); + git(root, 'commit', '-q', '-m', 'init'); + const cg = await CodeGraph.init(root, { index: true }); + graphs.push(cg); + + expect(routeLinks(cg)).toEqual({ + '/p/:project/builders/:bucket/:builder': ['Component@src/build/pages/builder_page/builder_page.tsx'], + '/b/:buildId': ['Component@src/build/legacy/build_page/build_page.tsx'], + }); + expect(cg.getNodesInFile('src/build/pages/builder_page/builder_page.tsx').map((n) => n.name)).toContain('BuilderPage'); + + // An edit, a new page and new build output below src/build: the git-status + // sync reads them by the same rule as the full index. + write(root, { + 'src/build/pages/builder_page/builder_page.tsx': FILES['src/build/pages/builder_page/builder_page.tsx']!.replace(/BuilderPage/g, 'BuilderView'), + 'src/build/pages/search_page/search_page.tsx': 'export function Component() {\n return null;\n}\n', + 'src/build/pages/bundle.js': 'export function bundled() {}\n', + 'src/build/pages/types.d.ts': 'export declare function bundled(): void;\n', + }); + const changes = getGitChangedFiles(root)!; + expect([...changes.added, ...changes.modified].sort()).toEqual([ + 'src/build/pages/builder_page/builder_page.tsx', + 'src/build/pages/search_page/search_page.tsx', + ]); + const synced = await cg.sync(); + expect(synced.filesAdded).toBe(1); + expect(synced.filesModified).toBe(1); + expect(cg.getNodesInFile('src/build/pages/builder_page/builder_page.tsx').map((n) => n.name)).toContain('BuilderView'); + expect(cg.getNodesInFile('src/build/pages/bundle.js')).toEqual([]); + }, 60_000); + + it('is watched: its source edits sync, build output there does not', async () => { + const root = tempDir(); + write(root, FILES); + const syncFn = vi.fn().mockResolvedValue({ filesChanged: 0, durationMs: 0 }); + const watcher = new FileWatcher(root, syncFn, { inertForTests: true, debounceMs: 100 }); + watchers.push(watcher); + watcher.start(); + await watcher.waitUntilReady(); + + write(root, { + 'src/build/pages/bundle.js': 'export function bundled() {}\n', + 'build/main.ts': 'export const out = 1;\n', + 'src/build/pages/search_page/search_page.tsx': 'export function Component() {\n return null;\n}\n', + }); + __emitWatchEventForTests(root, 'src/build/pages/bundle.js'); + __emitWatchEventForTests(root, 'build/main.ts'); + __emitWatchEventForTests(root, 'src/build/pages/search_page/search_page.tsx'); + expect(watcher.getPendingFiles().map((p) => p.path)).toEqual(['src/build/pages/search_page/search_page.tsx']); + await vi.waitFor(() => expect(syncFn).toHaveBeenCalledTimes(1)); + expect(syncFn.mock.calls[0]![0]).toEqual(['src/build/pages/search_page/search_page.tsx']); + + // A folder removed inside it arrives as one event on the folder's path: + // that schedules the full reconcile which drops its files (#1285). + fs.rmSync(path.join(root, 'src/build/legacy'), { recursive: true, force: true }); + __emitWatchEventForTests(root, 'src/build/legacy'); + await vi.waitFor(() => expect(syncFn).toHaveBeenCalledTimes(2)); + expect(syncFn.mock.calls[1]![0]).toBeUndefined(); + }); +}); diff --git a/site/src/content/docs/getting-started/configuration.md b/site/src/content/docs/getting-started/configuration.md index d63784e72..8f8e7dabc 100644 --- a/site/src/content/docs/getting-started/configuration.md +++ b/site/src/content/docs/getting-started/configuration.md @@ -7,7 +7,7 @@ Next to none — CodeGraph is **zero-config by default**, with nothing to write ## What it skips out of the box -- **Dependency, build, and cache directories** — `node_modules`, `vendor`, `dist`, `build`, `target`, `.venv`, `Pods`, `.next`, and the like across every [supported stack](/codegraph/reference/languages/) — so the graph is your code, not third-party noise. This holds even with no `.gitignore`. +- **Dependency, build, and cache directories** — `node_modules`, `vendor`, `dist`, `build`, `target`, `.venv`, `Pods`, `.next`, and the like across every [supported stack](/codegraph/reference/languages/) — so the graph is your code, not third-party noise. This holds even with no `.gitignore`. A `build` folder inside `src/` is the exception: no build step writes TypeScript, JSX, Vue, Svelte or Astro files, so those are indexed from it, and a Java, Kotlin or Scala package named `build` is indexed whole. - **Anything in your `.gitignore`** — honored in git repos via git, and in non-git projects by reading `.gitignore` directly (root and nested). - **Files larger than 1 MB** — generated bundles, minified JS, vendored blobs. diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 8623d7203..5d0bed0ed 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -268,6 +268,17 @@ const ANDROID_RES_TYPES: readonly string[] = [ 'menu', 'mipmap', 'navigation', 'transition', 'values', 'xml', ]; +/** + * `build` is also a legal JVM package segment: directories named `build` under + * a conventional source root (any source set: main, test, androidTest, ...) are + * packages, not module/build output (#1642). + */ +const JVM_BUILD_PACKAGE_DIRS: readonly string[] = [ + '**/src/*/java/**/build/', + '**/src/*/kotlin/**/build/', + '**/src/*/scala/**/build/', +]; + /** Gitignore-style patterns for the `ignore` matcher: the dirs above plus a few globs. */ const DEFAULT_IGNORE_PATTERNS: string[] = [ ...Array.from(DEFAULT_IGNORE_DIRS, (d) => `${d}/`), @@ -276,15 +287,89 @@ const DEFAULT_IGNORE_PATTERNS: string[] = [ 'bazel-*/', // Bazel output symlink trees // Android resource dirs at any depth, with their qualifier variants (#1047). ...ANDROID_RES_TYPES.map((t) => `**/res/${t}*/`), - // `build` is also a legal JVM package segment. Keep it under conventional - // source roots (any source set: main, test, androidTest, ...) while continuing - // to exclude module/build output (#1642). Unignore only the directory, not its - // subtree: other defaults still apply. - '!**/src/*/java/**/build/', - '!**/src/*/kotlin/**/build/', - '!**/src/*/scala/**/build/', + // Keep JVM packages named `build` (#1642). Unignore only the directory, not + // its subtree: other defaults still apply. + ...JVM_BUILD_PACKAGE_DIRS.map((d) => `!${d}`), + // Walk a `build` folder below `src/` too: a JS/TS app can keep a whole + // feature there. Only its hand-written source files are indexed — see + // buildOutputFolderUnderSrc. Other defaults still apply inside it. + '!**/src/**/build/', ]; +/** + * Source files no compiler or bundler writes: TypeScript (with its module and + * ArkTS variants), JSX, and Vue, Svelte and Astro components. Build steps emit + * JavaScript, declarations, CSS, HTML and source maps instead. + */ +const HAND_WRITTEN_SOURCE = /\.(?:[mc]?ts|tsx|ets|jsx|vue|svelte|astro)$/i; +/** TypeScript declarations, which `tsc` writes beside its JavaScript. */ +const DECLARATION_FILE = /\.d\.(?:[mc]?ts|ets)$/i; +/** Matches the JVM packages named `build`, whose whole subtree stays indexed. */ +const jvmBuildPackages = ignore().add(JVM_BUILD_PACKAGE_DIRS); + +/** + * The `build` folder below a `src/` folder that file `rel` sits in, with a + * trailing slash (`src/build/` for `src/build/legacy/page.tsx`), or null. A JVM + * package named `build` (#1642) does not count. + */ +function srcBuildFolderOf(rel: string): string | null { + if (!/(?:^|\/)build\//i.test(rel)) return null; + const parts = rel.split('/'); + let inSrc = false; + for (let i = 0; i < parts.length - 1; i++) { + const part = parts[i]!.toLowerCase(); + if (part === 'src') { + inSrc = true; + } else if (part === 'build' && inSrc) { + const folder = parts.slice(0, i + 1).join('/') + '/'; + return jvmBuildPackages.ignores(folder) ? null : folder; + } + } + return null; +} + +/** + * When `rel` is a file in a `build` folder below `src/` that a build step could + * have written, that folder (see {@link srcBuildFolderOf}); otherwise null. + * + * The folder itself is walked: JS/TS apps keep real source there (luci-go + * milo's whole `src/build/` feature). But an output tree built or copied into + * `src/` — a nested CRA app's `build/`, a vendored pdf.js, a Laravel + * `public/build` — holds JavaScript, CSS, HTML and maps, never TypeScript, JSX + * or components, so only those hand-written types are indexed from it. Paths + * without an extension are left alone: they are directories (a watcher event + * for a removed folder carries no trailing slash), never files a build writes. + */ +function buildOutputFolderUnderSrc(rel: string): string | null { + const name = rel.slice(rel.lastIndexOf('/') + 1); + if (!name.includes('.')) return null; + if (HAND_WRITTEN_SOURCE.test(name) && !DECLARATION_FILE.test(name)) return null; + return srcBuildFolderOf(rel); +} + +/** + * `ig` with {@link buildOutputFolderUnderSrc} applied on top of its patterns, + * for every caller of the defaults alike. `projectRules` are the project's own + * root ignore rules: a file they re-include by name, or whose folder they + * re-include from the `build` folder down (`!src/build/`, the documented + * opt-in), keeps every type, as it did before this rule. + */ +function excludeBuildOutputUnderSrc(ig: Ignore, projectRules: Ignore | null): Ignore { + const ignoredByPatterns = ig.ignores.bind(ig); + ig.ignores = (rel: string): boolean => { + if (ignoredByPatterns(rel)) return true; + const folder = buildOutputFolderUnderSrc(rel); + if (folder === null) return false; + if (!projectRules) return true; + // The file, then each folder above it up to the `build` folder. + for (let end = rel.length; end >= folder.length; end = rel.lastIndexOf('/', end - 2) + 1) { + if (projectRules.test(rel.slice(0, end)).unignored) return false; + } + return true; + }; + return ig; +} + /** True if `buf` decodes as strict UTF-8 (no invalid byte sequences). */ function isValidUtf8(buf: Buffer): boolean { try { @@ -465,11 +550,19 @@ function listGitIgnoredDirectories(rootDir: string): string[] { */ export function buildDefaultIgnore(rootDir: string): Ignore { const ig = ignore().add(DEFAULT_IGNORE_PATTERNS); + const projectRules = ignore(); const rootGitignore = path.join(rootDir, '.gitignore'); - if (fs.existsSync(rootGitignore)) ig.add(readGitignorePatterns(rootGitignore)); + if (fs.existsSync(rootGitignore)) { + const patterns = readGitignorePatterns(rootGitignore); + ig.add(patterns); + projectRules.add(patterns); + } const extra = readGitExcludeExtraPatterns(rootDir); - if (extra) ig.add(extra); - return ig; + if (extra) { + ig.add(extra); + projectRules.add(extra); + } + return excludeBuildOutputUnderSrc(ig, projectRules); } /** @@ -478,7 +571,7 @@ export function buildDefaultIgnore(rootDir: string): Ignore { * whose gitignore semantics their own `git ls-files` already enforced (#514). */ function defaultsOnlyIgnore(): Ignore { - return ignore().add(DEFAULT_IGNORE_PATTERNS); + return excludeBuildOutputUnderSrc(ignore().add(DEFAULT_IGNORE_PATTERNS), null); } /**