diff --git a/__tests__/ts-export-type-star.test.ts b/__tests__/ts-export-type-star.test.ts index eec3300f2..4c43299bb 100644 --- a/__tests__/ts-export-type-star.test.ts +++ b/__tests__/ts-export-type-star.test.ts @@ -22,8 +22,8 @@ import * as path from 'path'; import { CodeGraph } from '../src'; import { extractFromSource } from '../src/extraction'; import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; -import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel'; -import { blankExportTypeStar } from '../src/extraction/languages/typescript'; +import { tryKernelExtract, tryKernelExtractRaw, resetKernelForTests } from '../src/extraction/kernel'; +import { blankExportTypeStar, blanksExportTypeStar } from '../src/extraction/languages/typescript'; import type { ExtractionResult, Language } from '../src/types'; const KERNEL_PATH = path.join( @@ -45,6 +45,9 @@ const BARREL = [ '', ].join('\n'); +/** Each statement of the barrel, as its import node keeps it. */ +const BARREL_STATEMENTS = BARREL.split('\n').filter(Boolean); + describe('blankExportTypeStar', () => { it('blanks the `type` of a type-only star re-export, keeping every offset', () => { const cases: Array<[string, string]> = [ @@ -150,10 +153,37 @@ describe('a type-only star re-export parses in both engines', () => { 'FooClient@1', 'FooClientOptions@1', ]); + // Each import node keeps its statement as written, `type` included. + expect(result.nodes.filter((n) => n.kind === 'import').map((n) => n.signature)).toEqual(BARREL_STATEMENTS); }); } } + it.runIf(kernelAvailable)('keeps a re-export as written in a file the kernel hands to the wasm parser', () => { + // `export const = ;` is an error, so the kernel defers the whole file. + const source = 'export type * from "./types.js";\nexport const = ;\n'; + delete process.env.CODEGRAPH_KERNEL; + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + expect(tryKernelExtract('src/broken.ts', source, 'typescript')).toBeNull(); + const result = extractFromSource('src/broken.ts', source, 'typescript'); + expect(result.nodes.filter((n) => n.kind === 'import').map((n) => n.signature)).toEqual([ + 'export type * from "./types.js";', + ]); + }); + + it.runIf(kernelAvailable)('leaves the undecoded kernel path to files with nothing to keep as written', () => { + delete process.env.CODEGRAPH_KERNEL; + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + // Its import nodes' text is restored on the decoded path, which the parse worker falls back to. + expect(tryKernelExtractRaw('src/index.ts', BARREL, 'typescript')).toBeNull(); + const plain = 'export { FooClient } from "./client.js";\nexport * from "./types.js";\n'; + expect(tryKernelExtractRaw('src/plain.ts', plain, 'typescript')).not.toBeNull(); + expect(blanksExportTypeStar(BARREL)).toBe(true); + expect(blanksExportTypeStar(plain)).toBe(false); + // A mention only in a comment blanks nothing. + expect(blanksExportTypeStar('// prefer export type * from "./types"\nexport * from "./types";\n')).toBe(false); + }); + it.each(backends)('in a declaration file\'s declare module body: %s', (backend) => { const source = 'declare module "sdk" {\n export type * from "./types";\n export function connect(): void;\n}\n'; const result = extract(backend, 'typings/sdk.d.ts', source, 'typescript'); @@ -222,6 +252,11 @@ describe('indexing a barrel with `export type *` (#2474)', () => { expect(barrel!.errors ?? []).toEqual([]); }); + it('stores each re-export as written', () => { + const imports = cg.getNodesInFile('packages/sdk/src/index.ts').filter((n) => n.kind === 'import'); + expect(imports.sort((a, b) => a.startLine - b.startLine).map((n) => n.signature)).toEqual(BARREL_STATEMENTS); + }); + it('links each name imported through the barrel to the file that declares it', () => { const ids = cg.getNodesInFile('app/main.ts').map((n) => n.id); const imported = cg diff --git a/src/extraction/kernel/index.ts b/src/extraction/kernel/index.ts index 1e71d170c..17281ffa7 100644 --- a/src/extraction/kernel/index.ts +++ b/src/extraction/kernel/index.ts @@ -141,6 +141,32 @@ function preParsedSource(filePath: string, source: string, language: Language): return pre ? pre(source, filePath) : source; } +/** + * Whether the file's re-exports keep their statements as written while the + * kernel reads the preParsed bytes (`reExportAsWritten`: TypeScript's + * `export type *`, #2474). The decoded result then takes those import nodes' + * signatures from `source`, as TreeSitterExtractor does, and the undecoded + * fast path steps aside for it. + */ +function keepsReExportsAsWritten(source: string, pre: string, language: Language): boolean { + return pre !== source && EXTRACTORS[language]?.reExportAsWritten?.(source) === true; +} + +/** Each re-export import node's signature, read from `source` by the node's span. */ +function restoreReExportSignatures(result: ExtractionResult, source: string): void { + let lineStarts: number[] | null = null; + for (const node of result.nodes) { + if (node.kind !== 'import' || !node.signature?.startsWith('export')) continue; + if (!lineStarts) { + lineStarts = [0]; + for (let i = source.indexOf('\n'); i !== -1; i = source.indexOf('\n', i + 1)) lineStarts.push(i + 1); + } + const start = lineStarts[node.startLine - 1]! + node.startColumn; + const end = lineStarts[node.endLine - 1]! + node.endColumn; + node.signature = source.slice(start, end).trim(); + } +} + function isRouted(language: Language): boolean { const env = process.env.CODEGRAPH_KERNEL_LANGS; if (env === undefined || env === '') return DEFAULT_ROUTED.has(language); @@ -203,7 +229,9 @@ export interface KernelRawResult { * so the main thread never materializes per-node objects. Returns null under * exactly the conditions tryKernelExtract does, PLUS when the language has a * registered post() pass (post passes operate on decoded results, so those - * languages keep the decoded path). + * languages keep the decoded path), and for a file whose re-exports keep their + * statements as written (keepsReExportsAsWritten), which the decoded path + * restores. */ export function tryKernelExtractRaw( filePath: string, @@ -215,6 +243,8 @@ export function tryKernelExtractRaw( if (!kernel) return null; if (takeDeferredPreParse(filePath, source, language) !== null) return null; // already deferred const pre = preParsedSource(filePath, source, language); + // Its signatures come from the source as written, which needs the decoded path. + if (keepsReExportsAsWritten(source, pre, language)) return null; try { const buffers = kernel.extractFile(filePath, pre, language); const meta = buffers.meta; @@ -296,6 +326,7 @@ export function tryKernelExtract( try { const buffers = kernel.extractFile(filePath, pre, language); const result = decodeExtractBuffers(buffers, filePath, language); + if (keepsReExportsAsWritten(source, pre, language)) restoreReExportSignatures(result, source); POST_PASSES[language]?.(result, source); // Read beside the tables, as TreeSitterExtractor does (a CommonJS `require`). result.unresolvedReferences.push(...commonJsRequireRefs(filePath, source, language)); diff --git a/src/extraction/languages/typescript.ts b/src/extraction/languages/typescript.ts index 5b435d914..cff6b9349 100644 --- a/src/extraction/languages/typescript.ts +++ b/src/extraction/languages/typescript.ts @@ -87,7 +87,8 @@ const EXPORT_TYPE_STAR = /(^|[;{}]|\*\/)([ \t]*export\s+)type(?=\s*\*\s*(?:from| * kernel handed every such file to the wasm parser (#2474). Only a statement * start is blanked, so a mention in a comment or a doc keeps its text. The * import resolver reads the statement as written, `type` included, from the - * file itself (`extractReExports`). Applied before both engines parse (the + * file itself (`extractReExports`), and so does the import node the statement + * makes ({@link blanksExportTypeStar}). Applied before both engines parse (the * kernel's preParse hoist), so they agree. */ export function blankExportTypeStar(source: string): string { @@ -95,6 +96,15 @@ export function blankExportTypeStar(source: string): string { return source.replace(EXPORT_TYPE_STAR, '$1$2 '); } +/** + * Whether {@link blankExportTypeStar} blanks anything in `source`. Such a + * file's re-exports keep their statements as written in the graph, `type` + * included (`reExportAsWritten`). + */ +export function blanksExportTypeStar(source: string): boolean { + return EXPORT_TYPE_STAR_HINT.test(source) && blankExportTypeStar(source) !== source; +} + /** * Whether `node` is declared in a `declare module 'x' { … }` or `declare * global { … }` body, which exports it without an `export` keyword. @@ -174,6 +184,7 @@ export function isAmbientFunctionSignature(node: SyntaxNode, filePath: string): export const typescriptExtractor: LanguageExtractor = { preParse: (source, filePath) => blankExportTypeStar(blankFlowSyntax(source, filePath)), + reExportAsWritten: blanksExportTypeStar, functionTypes: ['function_declaration', 'generator_function_declaration', 'arrow_function', 'function_expression', 'generator_function'], isAmbientFunction: isAmbientFunctionSignature, // `class` is a class expression, `register(class Widget extends Base {…})`: diff --git a/src/extraction/tree-sitter-types.ts b/src/extraction/tree-sitter-types.ts index ade9fff8b..e67dddaad 100644 --- a/src/extraction/tree-sitter-types.ts +++ b/src/extraction/tree-sitter-types.ts @@ -87,6 +87,13 @@ export interface LanguageExtractor { * or `const [MVCFromBody] Person` (#2545). Offsets are the same in both. */ signatureAsWritten?: boolean; + /** + * Whether a re-export's import node (`export … from`) keeps its statement as + * the file writes it, not as `preParse` left it, given the source as + * written: TypeScript asks for a file whose `export type *` it blanks + * (#2474). The kernel route reads the same answer (kernel/index.ts). + */ + reExportAsWritten?: (source: string) => boolean; // --- Node type mappings --- diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index dd6a81869..d78099c7a 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -816,15 +816,19 @@ export class TreeSitterExtractor { // point (this instance is the wasm fallback for a kernel-deferred file) — // don't blank it a second time. private sourceIsPreParsed = false; - // The source as the file has it, kept when this instance's preParse changed - // it, for an extractor whose signatures read it (`signatureAsWritten`). + // The source as the file has it, kept when preParse changed it (by this + // instance, or at the kernel route point for a deferred file), for an + // extractor whose signatures read it (`signatureAsWritten`) or whose + // re-exports keep their statements as written (`reExportAsWritten`). private sourceAsWritten: string | null = null; + // Memoized reExportAsWritten verdict (per-extractor = per-file). + private reExportsAsWritten: boolean | null = null; constructor( filePath: string, source: string, language?: Language, - options?: { sourceIsPreParsed?: boolean } + options?: { sourceIsPreParsed?: boolean; sourceAsWritten?: string } ) { this.filePath = filePath; this.source = source; @@ -832,11 +836,25 @@ export class TreeSitterExtractor { this.extractor = EXTRACTORS[this.language] || null; this.fnRefSpec = FN_REF_SPECS[this.language]; this.sourceIsPreParsed = options?.sourceIsPreParsed === true; + if (options?.sourceAsWritten !== undefined && options.sourceAsWritten !== source) { + this.sourceAsWritten = options.sourceAsWritten; + } } /** The text `getSignature` reads: the source as written when the extractor asks for it. */ private get signatureSource(): string { - return this.sourceAsWritten ?? this.source; + return this.extractor?.signatureAsWritten && this.sourceAsWritten !== null ? this.sourceAsWritten : this.source; + } + + /** + * The text an import node reads its statement from: for a re-export, the + * source as written when the extractor asks (`reExportAsWritten`), so + * `export type * from './x'` keeps its `type` (#2474). + */ + private importSource(node: SyntaxNode): string { + if (node.type !== 'export_statement' || this.sourceAsWritten === null) return this.source; + this.reExportsAsWritten ??= this.extractor?.reExportAsWritten?.(this.sourceAsWritten) === true; + return this.reExportsAsWritten ? this.sourceAsWritten : this.source; } /** @@ -889,7 +907,7 @@ export class TreeSitterExtractor { // kernel route point already applied it (sourceIsPreParsed). if (this.extractor?.preParse && !this.sourceIsPreParsed) { const preParsed = this.extractor.preParse(this.source, this.filePath); - if (preParsed !== this.source && this.extractor.signatureAsWritten) this.sourceAsWritten = this.source; + if (preParsed !== this.source) this.sourceAsWritten = this.source; this.source = preParsed; } this.tree = parser.parse(this.source) ?? null; @@ -986,6 +1004,7 @@ export class TreeSitterExtractor { } // Release source string to reduce GC pressure this.source = ''; + this.sourceAsWritten = null; } return { @@ -4625,11 +4644,12 @@ export class TreeSitterExtractor { private extractImport(node: SyntaxNode): void { if (!this.extractor) return; - const importText = getNodeText(node, this.source).trim(); + const statementSource = this.importSource(node); + const importText = getNodeText(node, statementSource).trim(); // Try language-specific hook first if (this.extractor.extractImport) { - const info = this.extractor.extractImport(node, this.source); + const info = this.extractor.extractImport(node, statementSource); if (info) { this.createNode('import', info.moduleName, node, { signature: info.signature, @@ -9551,7 +9571,7 @@ export function extractFromSource( filePath, deferredPre ?? source, detectedLanguage, - { sourceIsPreParsed: deferredPre != null } + deferredPre != null ? { sourceIsPreParsed: true, sourceAsWritten: source } : undefined ); result = extractor.extract(); }