From 43e69ba61ca3b9f1b28d57a5e4da3795b9a0f791 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Fri, 23 Oct 2026 07:39:35 -0500 Subject: [PATCH] fix(ts): a re-export's import node keeps export type * as the file writes it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #2837 made `export type * from './x'` parse by blanking its `type` to spaces before either engine parses the file. Both engines read an import node's stored statement from the parsed bytes, so every such re-export stored `export * from './x'` (883 nodes over five repos). Nothing in resolution reads that text, but it is the statement's record in the graph. The source as written now reaches every place that builds those nodes: - TreeSitterExtractor keeps it whenever preParse changed the text, and a kernel-deferred file hands it over with the preParsed bytes; - a language opts in per file with `reExportAsWritten` (TypeScript: when its `export type *` blank changed something), and an `export … from` import node then reads its statement from the source as written; - the kernel's decoded result takes those nodes' signatures from the original source by each node's span, and the undecoded worker fast path steps aside for such files, as it does for C++ files with macro methods. Pascal's `signatureAsWritten` is unchanged: getSignature reads the source as written only for an extractor that asks. Validation: - ts-export-type-star.test.ts gains three tests and a signature check in each of its eight barrel cases (18 tests). 11 fail with main's four source files restored. Breaking the deferred-file hand-over, or the raw-path opt-out, fails exactly its own test. All pass with the fix. Pascal's as-written signature tests and the C, C++ and C# preParse parity suites pass unchanged. - A/B against main ef05970b, kernel engine, dumps by natural keys: masto.js 129, kysely 29, TanStack/query 5, graffle 622 and strapi 98 import nodes read their statement as written again, and nothing else changes (script-checked, 0 unexpected). masto.js and kysely nodes, edges and refs are byte-identical to main before PR #2837 (51c4124d). gin and requests are byte-identical. The wasm engine gives the same delta, and its graphs equal the kernel's. - Kernel parity, typescript and tsx: 0 diffs; deferrals unchanged. - Index time, median of 3 interleaved: graffle 10.17 s vs 10.01 s, strapi 32.87 s vs 28.05 s. Co-Authored-By: Claude Opus 5.5 --- __tests__/ts-export-type-star.test.ts | 39 ++++++++++++++++++++++++-- src/extraction/kernel/index.ts | 33 +++++++++++++++++++++- src/extraction/languages/typescript.ts | 13 ++++++++- src/extraction/tree-sitter-types.ts | 7 +++++ src/extraction/tree-sitter.ts | 36 ++++++++++++++++++------ 5 files changed, 116 insertions(+), 12 deletions(-) diff --git a/__tests__/ts-export-type-star.test.ts b/__tests__/ts-export-type-star.test.ts index eec3300f24..4c43299bba 100644 --- a/__tests__/ts-export-type-star.test.ts +++ b/__tests__/ts-export-type-star.test.ts @@ -22,8 +22,8 @@ import * as path from 'path'; import { CodeGraph } from '../src'; import { extractFromSource } from '../src/extraction'; import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; -import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel'; -import { blankExportTypeStar } from '../src/extraction/languages/typescript'; +import { tryKernelExtract, tryKernelExtractRaw, resetKernelForTests } from '../src/extraction/kernel'; +import { blankExportTypeStar, blanksExportTypeStar } from '../src/extraction/languages/typescript'; import type { ExtractionResult, Language } from '../src/types'; const KERNEL_PATH = path.join( @@ -45,6 +45,9 @@ const BARREL = [ '', ].join('\n'); +/** Each statement of the barrel, as its import node keeps it. */ +const BARREL_STATEMENTS = BARREL.split('\n').filter(Boolean); + describe('blankExportTypeStar', () => { it('blanks the `type` of a type-only star re-export, keeping every offset', () => { const cases: Array<[string, string]> = [ @@ -150,10 +153,37 @@ describe('a type-only star re-export parses in both engines', () => { 'FooClient@1', 'FooClientOptions@1', ]); + // Each import node keeps its statement as written, `type` included. + expect(result.nodes.filter((n) => n.kind === 'import').map((n) => n.signature)).toEqual(BARREL_STATEMENTS); }); } } + it.runIf(kernelAvailable)('keeps a re-export as written in a file the kernel hands to the wasm parser', () => { + // `export const = ;` is an error, so the kernel defers the whole file. + const source = 'export type * from "./types.js";\nexport const = ;\n'; + delete process.env.CODEGRAPH_KERNEL; + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + expect(tryKernelExtract('src/broken.ts', source, 'typescript')).toBeNull(); + const result = extractFromSource('src/broken.ts', source, 'typescript'); + expect(result.nodes.filter((n) => n.kind === 'import').map((n) => n.signature)).toEqual([ + 'export type * from "./types.js";', + ]); + }); + + it.runIf(kernelAvailable)('leaves the undecoded kernel path to files with nothing to keep as written', () => { + delete process.env.CODEGRAPH_KERNEL; + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + // Its import nodes' text is restored on the decoded path, which the parse worker falls back to. + expect(tryKernelExtractRaw('src/index.ts', BARREL, 'typescript')).toBeNull(); + const plain = 'export { FooClient } from "./client.js";\nexport * from "./types.js";\n'; + expect(tryKernelExtractRaw('src/plain.ts', plain, 'typescript')).not.toBeNull(); + expect(blanksExportTypeStar(BARREL)).toBe(true); + expect(blanksExportTypeStar(plain)).toBe(false); + // A mention only in a comment blanks nothing. + expect(blanksExportTypeStar('// prefer export type * from "./types"\nexport * from "./types";\n')).toBe(false); + }); + it.each(backends)('in a declaration file\'s declare module body: %s', (backend) => { const source = 'declare module "sdk" {\n export type * from "./types";\n export function connect(): void;\n}\n'; const result = extract(backend, 'typings/sdk.d.ts', source, 'typescript'); @@ -222,6 +252,11 @@ describe('indexing a barrel with `export type *` (#2474)', () => { expect(barrel!.errors ?? []).toEqual([]); }); + it('stores each re-export as written', () => { + const imports = cg.getNodesInFile('packages/sdk/src/index.ts').filter((n) => n.kind === 'import'); + expect(imports.sort((a, b) => a.startLine - b.startLine).map((n) => n.signature)).toEqual(BARREL_STATEMENTS); + }); + it('links each name imported through the barrel to the file that declares it', () => { const ids = cg.getNodesInFile('app/main.ts').map((n) => n.id); const imported = cg diff --git a/src/extraction/kernel/index.ts b/src/extraction/kernel/index.ts index 1e71d170c3..17281ffa71 100644 --- a/src/extraction/kernel/index.ts +++ b/src/extraction/kernel/index.ts @@ -141,6 +141,32 @@ function preParsedSource(filePath: string, source: string, language: Language): return pre ? pre(source, filePath) : source; } +/** + * Whether the file's re-exports keep their statements as written while the + * kernel reads the preParsed bytes (`reExportAsWritten`: TypeScript's + * `export type *`, #2474). The decoded result then takes those import nodes' + * signatures from `source`, as TreeSitterExtractor does, and the undecoded + * fast path steps aside for it. + */ +function keepsReExportsAsWritten(source: string, pre: string, language: Language): boolean { + return pre !== source && EXTRACTORS[language]?.reExportAsWritten?.(source) === true; +} + +/** Each re-export import node's signature, read from `source` by the node's span. */ +function restoreReExportSignatures(result: ExtractionResult, source: string): void { + let lineStarts: number[] | null = null; + for (const node of result.nodes) { + if (node.kind !== 'import' || !node.signature?.startsWith('export')) continue; + if (!lineStarts) { + lineStarts = [0]; + for (let i = source.indexOf('\n'); i !== -1; i = source.indexOf('\n', i + 1)) lineStarts.push(i + 1); + } + const start = lineStarts[node.startLine - 1]! + node.startColumn; + const end = lineStarts[node.endLine - 1]! + node.endColumn; + node.signature = source.slice(start, end).trim(); + } +} + function isRouted(language: Language): boolean { const env = process.env.CODEGRAPH_KERNEL_LANGS; if (env === undefined || env === '') return DEFAULT_ROUTED.has(language); @@ -203,7 +229,9 @@ export interface KernelRawResult { * so the main thread never materializes per-node objects. Returns null under * exactly the conditions tryKernelExtract does, PLUS when the language has a * registered post() pass (post passes operate on decoded results, so those - * languages keep the decoded path). + * languages keep the decoded path), and for a file whose re-exports keep their + * statements as written (keepsReExportsAsWritten), which the decoded path + * restores. */ export function tryKernelExtractRaw( filePath: string, @@ -215,6 +243,8 @@ export function tryKernelExtractRaw( if (!kernel) return null; if (takeDeferredPreParse(filePath, source, language) !== null) return null; // already deferred const pre = preParsedSource(filePath, source, language); + // Its signatures come from the source as written, which needs the decoded path. + if (keepsReExportsAsWritten(source, pre, language)) return null; try { const buffers = kernel.extractFile(filePath, pre, language); const meta = buffers.meta; @@ -296,6 +326,7 @@ export function tryKernelExtract( try { const buffers = kernel.extractFile(filePath, pre, language); const result = decodeExtractBuffers(buffers, filePath, language); + if (keepsReExportsAsWritten(source, pre, language)) restoreReExportSignatures(result, source); POST_PASSES[language]?.(result, source); // Read beside the tables, as TreeSitterExtractor does (a CommonJS `require`). result.unresolvedReferences.push(...commonJsRequireRefs(filePath, source, language)); diff --git a/src/extraction/languages/typescript.ts b/src/extraction/languages/typescript.ts index 5b435d9141..cff6b9349a 100644 --- a/src/extraction/languages/typescript.ts +++ b/src/extraction/languages/typescript.ts @@ -87,7 +87,8 @@ const EXPORT_TYPE_STAR = /(^|[;{}]|\*\/)([ \t]*export\s+)type(?=\s*\*\s*(?:from| * kernel handed every such file to the wasm parser (#2474). Only a statement * start is blanked, so a mention in a comment or a doc keeps its text. The * import resolver reads the statement as written, `type` included, from the - * file itself (`extractReExports`). Applied before both engines parse (the + * file itself (`extractReExports`), and so does the import node the statement + * makes ({@link blanksExportTypeStar}). Applied before both engines parse (the * kernel's preParse hoist), so they agree. */ export function blankExportTypeStar(source: string): string { @@ -95,6 +96,15 @@ export function blankExportTypeStar(source: string): string { return source.replace(EXPORT_TYPE_STAR, '$1$2 '); } +/** + * Whether {@link blankExportTypeStar} blanks anything in `source`. Such a + * file's re-exports keep their statements as written in the graph, `type` + * included (`reExportAsWritten`). + */ +export function blanksExportTypeStar(source: string): boolean { + return EXPORT_TYPE_STAR_HINT.test(source) && blankExportTypeStar(source) !== source; +} + /** * Whether `node` is declared in a `declare module 'x' { … }` or `declare * global { … }` body, which exports it without an `export` keyword. @@ -174,6 +184,7 @@ export function isAmbientFunctionSignature(node: SyntaxNode, filePath: string): export const typescriptExtractor: LanguageExtractor = { preParse: (source, filePath) => blankExportTypeStar(blankFlowSyntax(source, filePath)), + reExportAsWritten: blanksExportTypeStar, functionTypes: ['function_declaration', 'generator_function_declaration', 'arrow_function', 'function_expression', 'generator_function'], isAmbientFunction: isAmbientFunctionSignature, // `class` is a class expression, `register(class Widget extends Base {…})`: diff --git a/src/extraction/tree-sitter-types.ts b/src/extraction/tree-sitter-types.ts index ade9fff8b9..e67dddaad2 100644 --- a/src/extraction/tree-sitter-types.ts +++ b/src/extraction/tree-sitter-types.ts @@ -87,6 +87,13 @@ export interface LanguageExtractor { * or `const [MVCFromBody] Person` (#2545). Offsets are the same in both. */ signatureAsWritten?: boolean; + /** + * Whether a re-export's import node (`export … from`) keeps its statement as + * the file writes it, not as `preParse` left it, given the source as + * written: TypeScript asks for a file whose `export type *` it blanks + * (#2474). The kernel route reads the same answer (kernel/index.ts). + */ + reExportAsWritten?: (source: string) => boolean; // --- Node type mappings --- diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index dd6a81869e..d78099c7ae 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -816,15 +816,19 @@ export class TreeSitterExtractor { // point (this instance is the wasm fallback for a kernel-deferred file) — // don't blank it a second time. private sourceIsPreParsed = false; - // The source as the file has it, kept when this instance's preParse changed - // it, for an extractor whose signatures read it (`signatureAsWritten`). + // The source as the file has it, kept when preParse changed it (by this + // instance, or at the kernel route point for a deferred file), for an + // extractor whose signatures read it (`signatureAsWritten`) or whose + // re-exports keep their statements as written (`reExportAsWritten`). private sourceAsWritten: string | null = null; + // Memoized reExportAsWritten verdict (per-extractor = per-file). + private reExportsAsWritten: boolean | null = null; constructor( filePath: string, source: string, language?: Language, - options?: { sourceIsPreParsed?: boolean } + options?: { sourceIsPreParsed?: boolean; sourceAsWritten?: string } ) { this.filePath = filePath; this.source = source; @@ -832,11 +836,25 @@ export class TreeSitterExtractor { this.extractor = EXTRACTORS[this.language] || null; this.fnRefSpec = FN_REF_SPECS[this.language]; this.sourceIsPreParsed = options?.sourceIsPreParsed === true; + if (options?.sourceAsWritten !== undefined && options.sourceAsWritten !== source) { + this.sourceAsWritten = options.sourceAsWritten; + } } /** The text `getSignature` reads: the source as written when the extractor asks for it. */ private get signatureSource(): string { - return this.sourceAsWritten ?? this.source; + return this.extractor?.signatureAsWritten && this.sourceAsWritten !== null ? this.sourceAsWritten : this.source; + } + + /** + * The text an import node reads its statement from: for a re-export, the + * source as written when the extractor asks (`reExportAsWritten`), so + * `export type * from './x'` keeps its `type` (#2474). + */ + private importSource(node: SyntaxNode): string { + if (node.type !== 'export_statement' || this.sourceAsWritten === null) return this.source; + this.reExportsAsWritten ??= this.extractor?.reExportAsWritten?.(this.sourceAsWritten) === true; + return this.reExportsAsWritten ? this.sourceAsWritten : this.source; } /** @@ -889,7 +907,7 @@ export class TreeSitterExtractor { // kernel route point already applied it (sourceIsPreParsed). if (this.extractor?.preParse && !this.sourceIsPreParsed) { const preParsed = this.extractor.preParse(this.source, this.filePath); - if (preParsed !== this.source && this.extractor.signatureAsWritten) this.sourceAsWritten = this.source; + if (preParsed !== this.source) this.sourceAsWritten = this.source; this.source = preParsed; } this.tree = parser.parse(this.source) ?? null; @@ -986,6 +1004,7 @@ export class TreeSitterExtractor { } // Release source string to reduce GC pressure this.source = ''; + this.sourceAsWritten = null; } return { @@ -4625,11 +4644,12 @@ export class TreeSitterExtractor { private extractImport(node: SyntaxNode): void { if (!this.extractor) return; - const importText = getNodeText(node, this.source).trim(); + const statementSource = this.importSource(node); + const importText = getNodeText(node, statementSource).trim(); // Try language-specific hook first if (this.extractor.extractImport) { - const info = this.extractor.extractImport(node, this.source); + const info = this.extractor.extractImport(node, statementSource); if (info) { this.createNode('import', info.moduleName, node, { signature: info.signature, @@ -9551,7 +9571,7 @@ export function extractFromSource( filePath, deferredPre ?? source, detectedLanguage, - { sourceIsPreParsed: deferredPre != null } + deferredPre != null ? { sourceIsPreParsed: true, sourceAsWritten: source } : undefined ); result = extractor.extract(); }