From 57fe80df4a074895a0946956f7a351dbc225f297 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Thu, 19 Feb 2026 17:00:46 +0100 Subject: [PATCH 1/2] meta: move generators --- src/generators.mjs | 8 +- src/generators/__tests__/index.test.mjs | 23 ++- src/generators/addon-verify/generate.mjs | 81 ++++++++++ src/generators/addon-verify/index.mjs | 78 ---------- src/generators/addon-verify/types.d.ts | 5 +- .../api-links/__tests__/fixtures.test.mjs | 8 +- src/generators/api-links/generate.mjs | 64 ++++++++ src/generators/api-links/index.mjs | 64 -------- src/generators/api-links/types.d.ts | 5 +- src/generators/ast-js/generate.mjs | 56 +++++++ src/generators/ast-js/index.mjs | 51 +------ src/generators/ast-js/types.d.ts | 5 +- src/generators/ast/generate.mjs | 58 +++++++ src/generators/ast/index.mjs | 52 ------- src/generators/ast/types.d.ts | 5 +- src/generators/index.mjs | 58 +++---- src/generators/json-simple/generate.mjs | 46 ++++++ src/generators/json-simple/index.mjs | 43 ------ src/generators/json-simple/types.d.ts | 5 +- src/generators/jsx-ast/generate.mjs | 67 ++++++++ src/generators/jsx-ast/index.mjs | 64 +------- src/generators/jsx-ast/types.d.ts | 5 +- .../jsx-ast/utils/buildBarProps.mjs | 6 +- src/generators/jsx-ast/utils/buildContent.mjs | 6 +- src/generators/legacy-html-all/generate.mjs | 74 +++++++++ src/generators/legacy-html-all/index.mjs | 70 --------- src/generators/legacy-html-all/types.d.ts | 9 +- src/generators/legacy-html/generate.mjs | 142 +++++++++++++++++ src/generators/legacy-html/index.mjs | 143 +----------------- src/generators/legacy-html/types.d.ts | 11 +- .../legacy-html/utils/buildContent.mjs | 4 +- .../utils/replaceTemplateValues.mjs | 6 +- src/generators/legacy-json-all/generate.mjs | 80 ++++++++++ src/generators/legacy-json-all/index.mjs | 79 ---------- src/generators/legacy-json-all/types.d.ts | 5 +- src/generators/legacy-json/generate.mjs | 66 ++++++++ src/generators/legacy-json/index.mjs | 63 -------- src/generators/legacy-json/types.d.ts | 5 +- src/generators/llms-txt/generate.mjs | 31 ++++ src/generators/llms-txt/index.mjs | 28 +--- src/generators/llms-txt/types.d.ts | 9 +- src/generators/man-page/generate.mjs | 84 ++++++++++ src/generators/man-page/index.mjs | 79 ---------- src/generators/man-page/types.d.ts | 15 +- src/generators/metadata/generate.mjs | 38 +++++ src/generators/metadata/index.mjs | 32 ---- src/generators/metadata/types.d.ts | 9 +- src/generators/orama-db/generate.mjs | 59 ++++++++ src/generators/orama-db/index.mjs | 56 ------- src/generators/orama-db/types.d.ts | 5 +- src/generators/sitemap/generate.mjs | 64 ++++++++ src/generators/sitemap/index.mjs | 61 +------- src/generators/sitemap/types.d.ts | 5 +- src/generators/types.d.ts | 31 ++-- src/generators/web/generate.mjs | 53 +++++++ src/generators/web/index.mjs | 54 +------ src/generators/web/types.d.ts | 13 +- src/generators/web/utils/bundle.mjs | 14 +- src/threading/chunk-worker.mjs | 7 +- src/threading/parallel.mjs | 11 +- .../configuration/__tests__/index.test.mjs | 6 +- src/utils/configuration/index.mjs | 9 +- src/utils/configuration/templates.mjs | 4 +- 63 files changed, 1245 insertions(+), 1152 deletions(-) create mode 100644 src/generators/addon-verify/generate.mjs create mode 100644 src/generators/api-links/generate.mjs create mode 100644 src/generators/ast-js/generate.mjs create mode 100644 src/generators/ast/generate.mjs create mode 100644 src/generators/json-simple/generate.mjs create mode 100644 src/generators/jsx-ast/generate.mjs create mode 100644 src/generators/legacy-html-all/generate.mjs create mode 100644 src/generators/legacy-html/generate.mjs create mode 100644 src/generators/legacy-json-all/generate.mjs create mode 100644 src/generators/legacy-json/generate.mjs create mode 100644 src/generators/llms-txt/generate.mjs create mode 100644 src/generators/man-page/generate.mjs create mode 100644 src/generators/metadata/generate.mjs create mode 100644 src/generators/orama-db/generate.mjs create mode 100644 src/generators/sitemap/generate.mjs create mode 100644 src/generators/web/generate.mjs diff --git a/src/generators.mjs b/src/generators.mjs index 5245f534..81e90526 100644 --- a/src/generators.mjs +++ b/src/generators.mjs @@ -53,8 +53,12 @@ const createGenerator = () => { return; } - const { dependsOn, generate, processChunk } = - await allGenerators[generatorName](); + const { dependsOn } = allGenerators[generatorName]; + + // Lazy-load the generator implementation + const { generate, processChunk } = await import( + `./generators/${generatorName}/generate.mjs` + ); // Schedule dependency first if (dependsOn && !(dependsOn in cachedGenerators)) { diff --git a/src/generators/__tests__/index.test.mjs b/src/generators/__tests__/index.test.mjs index bc9ef584..8efd3c9a 100644 --- a/src/generators/__tests__/index.test.mjs +++ b/src/generators/__tests__/index.test.mjs @@ -7,16 +7,11 @@ import { allGenerators } from '../index.mjs'; const validDependencies = Object.keys(allGenerators); -const allGeneratorsReaolved = await Promise.all( - Object.entries(allGenerators).map(async ([key, loader]) => [ - key, - await loader(), - ]) -); +const allGeneratorsEntries = Object.entries(allGenerators); describe('All Generators', () => { - it('should have keys matching their name property', async () => { - allGeneratorsReaolved.forEach(([key, generator]) => { + it('should have keys matching their name property', () => { + allGeneratorsEntries.forEach(([key, generator]) => { assert.equal( key, generator.name, @@ -25,8 +20,8 @@ describe('All Generators', () => { }); }); - it('should have valid semver versions', async () => { - allGeneratorsReaolved.forEach(([key, generator]) => { + it('should have valid semver versions', () => { + allGeneratorsEntries.forEach(([key, generator]) => { const isValid = semver.valid(generator.version); assert.ok( isValid, @@ -35,8 +30,8 @@ describe('All Generators', () => { }); }); - it('should have valid dependsOn references', async () => { - allGeneratorsReaolved.forEach(([key, generator]) => { + it('should have valid dependsOn references', () => { + allGeneratorsEntries.forEach(([key, generator]) => { if (generator.dependsOn) { assert.ok( validDependencies.includes(generator.dependsOn), @@ -46,8 +41,8 @@ describe('All Generators', () => { }); }); - it('should have ast generator as a top-level generator with no dependencies', async () => { - const ast = await allGenerators.ast(); + it('should have ast generator as a top-level generator with no dependencies', () => { + const ast = allGenerators.ast; assert.ok(ast, 'ast generator should exist'); assert.equal( ast.dependsOn, diff --git a/src/generators/addon-verify/generate.mjs b/src/generators/addon-verify/generate.mjs new file mode 100644 index 00000000..074c7211 --- /dev/null +++ b/src/generators/addon-verify/generate.mjs @@ -0,0 +1,81 @@ +'use strict'; + +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { visit } from 'unist-util-visit'; + +import { EXTRACT_CODE_FILENAME_COMMENT } from './constants.mjs'; +import { generateFileList } from './utils/generateFileList.mjs'; +import { + generateSectionFolderName, + isBuildableSection, + normalizeSectionName, +} from './utils/section.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; + +/** + * Generates a file list from code blocks. + * + * @type {import('./types').Implementation['generate']} + */ +export async function generate(input) { + const config = getConfig('addon-verify'); + + const sectionsCodeBlocks = input.reduce((addons, node) => { + const sectionName = node.heading.data.name; + + const content = node.content; + + visit(content, childNode => { + if (childNode.type === 'code') { + const filename = childNode.value.match(EXTRACT_CODE_FILENAME_COMMENT); + + if (filename === null) { + return; + } + + if (!addons[sectionName]) { + addons[sectionName] = []; + } + + addons[sectionName].push({ + name: filename[1], + content: childNode.value, + }); + } + }); + + return addons; + }, {}); + + const files = await Promise.all( + Object.entries(sectionsCodeBlocks) + .filter(([, codeBlocks]) => isBuildableSection(codeBlocks)) + .flatMap(async ([sectionName, codeBlocks], index) => { + const files = generateFileList(codeBlocks); + + if (config.output) { + const normalizedSectionName = normalizeSectionName(sectionName); + + const folderName = generateSectionFolderName( + normalizedSectionName, + index + ); + + await mkdir(join(config.output, folderName), { recursive: true }); + + for (const file of files) { + await writeFile( + join(config.output, folderName, file.name), + file.content + ); + } + } + + return files; + }) + ); + + return files; +} diff --git a/src/generators/addon-verify/index.mjs b/src/generators/addon-verify/index.mjs index 4fb47a5f..6df1b576 100644 --- a/src/generators/addon-verify/index.mjs +++ b/src/generators/addon-verify/index.mjs @@ -1,19 +1,5 @@ 'use strict'; -import { mkdir, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { visit } from 'unist-util-visit'; - -import { EXTRACT_CODE_FILENAME_COMMENT } from './constants.mjs'; -import { generateFileList } from './utils/generateFileList.mjs'; -import { - generateSectionFolderName, - isBuildableSection, - normalizeSectionName, -} from './utils/section.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; - /** * This generator generates a file list from code blocks extracted from * `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime @@ -30,68 +16,4 @@ export default { 'Generates a file list from code blocks extracted from `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime validations', dependsOn: 'metadata', - - /** - * Generates a file list from code blocks. - */ - async generate(input) { - const config = getConfig('addon-verify'); - - const sectionsCodeBlocks = input.reduce((addons, node) => { - const sectionName = node.heading.data.name; - - const content = node.content; - - visit(content, childNode => { - if (childNode.type === 'code') { - const filename = childNode.value.match(EXTRACT_CODE_FILENAME_COMMENT); - - if (filename === null) { - return; - } - - if (!addons[sectionName]) { - addons[sectionName] = []; - } - - addons[sectionName].push({ - name: filename[1], - content: childNode.value, - }); - } - }); - - return addons; - }, {}); - - const files = await Promise.all( - Object.entries(sectionsCodeBlocks) - .filter(([, codeBlocks]) => isBuildableSection(codeBlocks)) - .flatMap(async ([sectionName, codeBlocks], index) => { - const files = generateFileList(codeBlocks); - - if (config.output) { - const normalizedSectionName = normalizeSectionName(sectionName); - - const folderName = generateSectionFolderName( - normalizedSectionName, - index - ); - - await mkdir(join(config.output, folderName), { recursive: true }); - - for (const file of files) { - await writeFile( - join(config.output, folderName, file.name), - file.content - ); - } - } - - return files; - }) - ); - - return files; - }, }; diff --git a/src/generators/addon-verify/types.d.ts b/src/generators/addon-verify/types.d.ts index 44520a3d..25af16cb 100644 --- a/src/generators/addon-verify/types.d.ts +++ b/src/generators/addon-verify/types.d.ts @@ -1,4 +1,5 @@ -export type Generator = GeneratorMetadata< - {}, +export type Generator = GeneratorMetadata<{}>; + +export type Implementation = GeneratorImpl< Generate, Promise>> >; diff --git a/src/generators/api-links/__tests__/fixtures.test.mjs b/src/generators/api-links/__tests__/fixtures.test.mjs index 60e88208..6259aeb3 100644 --- a/src/generators/api-links/__tests__/fixtures.test.mjs +++ b/src/generators/api-links/__tests__/fixtures.test.mjs @@ -6,8 +6,8 @@ import { globSync } from 'tinyglobby'; import createWorkerPool from '../../../threading/index.mjs'; import createParallelWorker from '../../../threading/parallel.mjs'; import { setConfig } from '../../../utils/configuration/index.mjs'; -import astJs from '../../ast-js/index.mjs'; -import apiLinks from '../index.mjs'; +import { generate as astJsGenerate } from '../../ast-js/generate.mjs'; +import { generate as apiLinksGenerate } from '../generate.mjs'; const relativePath = relative(process.cwd(), import.meta.dirname); @@ -40,11 +40,11 @@ describe('api links', () => { // Collect results from the async generator const astJsResults = []; - for await (const chunk of astJs.generate(undefined, worker)) { + for await (const chunk of astJsGenerate(undefined, worker)) { astJsResults.push(...chunk); } - const actualOutput = await apiLinks.generate(astJsResults); + const actualOutput = await apiLinksGenerate(astJsResults); for (const [k, v] of Object.entries(actualOutput)) { actualOutput[k] = v.replace(/.*(?=lib\/)/, ''); diff --git a/src/generators/api-links/generate.mjs b/src/generators/api-links/generate.mjs new file mode 100644 index 00000000..5f27de3b --- /dev/null +++ b/src/generators/api-links/generate.mjs @@ -0,0 +1,64 @@ +'use strict'; + +import { writeFile } from 'node:fs/promises'; +import { basename, join } from 'node:path'; + +import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs'; +import { extractExports } from './utils/extractExports.mjs'; +import { findDefinitions } from './utils/findDefinitions.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { GH_BLOB_URL, populate } from '../../utils/configuration/templates.mjs'; + +/** + * Generates the `apilinks.json` file. + * + * @type {import('./types').Implementation['generate']} + */ +export async function generate(input) { + const config = getConfig('api-links'); + /** + * @type Record + */ + const definitions = {}; + + input.forEach(program => { + /** + * Mapping of definitions to their line number + * + * @type {Record} + * @example { 'someclass.foo': 10 } + */ + const nameToLineNumberMap = {}; + + // `http.js` -> `http` + const baseName = basename(program.path, '.js'); + + const exports = extractExports(program, baseName, nameToLineNumberMap); + + findDefinitions(program, baseName, nameToLineNumberMap, exports); + + checkIndirectReferences(program, exports, nameToLineNumberMap); + + const fullGitUrl = `${populate(GH_BLOB_URL, config)}lib/${baseName}.js`; + + // Add the exports we found in this program to our output + Object.keys(nameToLineNumberMap).forEach(key => { + const lineNumber = nameToLineNumberMap[key]; + + definitions[key] = `${fullGitUrl}#L${lineNumber}`; + }); + }); + + if (config.output) { + const out = join(config.output, 'apilinks.json'); + + await writeFile( + out, + config.minify + ? JSON.stringify(definitions) + : JSON.stringify(definitions, null, 2) + ); + } + + return definitions; +} diff --git a/src/generators/api-links/index.mjs b/src/generators/api-links/index.mjs index 1c96be62..9252be95 100644 --- a/src/generators/api-links/index.mjs +++ b/src/generators/api-links/index.mjs @@ -1,17 +1,5 @@ 'use strict'; -import { writeFile } from 'node:fs/promises'; -import { basename, join } from 'node:path'; - -import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs'; -import { extractExports } from './utils/extractExports.mjs'; -import { findDefinitions } from './utils/findDefinitions.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { - GITHUB_BLOB_URL, - populate, -} from '../../utils/configuration/templates.mjs'; - /** * This generator is responsible for mapping publicly accessible functions in * Node.js to their source locations in the Node.js repository. @@ -33,56 +21,4 @@ export default { // Unlike the rest of the generators, this utilizes Javascript sources being // passed into the input field rather than Markdown. dependsOn: 'ast-js', - - /** - * Generates the `apilinks.json` file. - */ - async generate(input) { - const config = getConfig('api-links'); - /** - * @type Record - */ - const definitions = {}; - - input.forEach(program => { - /** - * Mapping of definitions to their line number - * - * @type {Record} - * @example { 'someclass.foo': 10 } - */ - const nameToLineNumberMap = {}; - - // `http.js` -> `http` - const baseName = basename(program.path, '.js'); - - const exports = extractExports(program, baseName, nameToLineNumberMap); - - findDefinitions(program, baseName, nameToLineNumberMap, exports); - - checkIndirectReferences(program, exports, nameToLineNumberMap); - - const fullGitUrl = `${populate(GITHUB_BLOB_URL, config)}lib/${baseName}.js`; - - // Add the exports we found in this program to our output - Object.keys(nameToLineNumberMap).forEach(key => { - const lineNumber = nameToLineNumberMap[key]; - - definitions[key] = `${fullGitUrl}#L${lineNumber}`; - }); - }); - - if (config.output) { - const out = join(config.output, 'apilinks.json'); - - await writeFile( - out, - config.minify - ? JSON.stringify(definitions) - : JSON.stringify(definitions, null, 2) - ); - } - - return definitions; - }, }; diff --git a/src/generators/api-links/types.d.ts b/src/generators/api-links/types.d.ts index 48a75b65..e0d8124d 100644 --- a/src/generators/api-links/types.d.ts +++ b/src/generators/api-links/types.d.ts @@ -4,7 +4,8 @@ export interface ProgramExports { indirects: Record; } -export type Generator = GeneratorMetadata< - {}, +export type Generator = GeneratorMetadata<{}>; + +export type Implementation = GeneratorImpl< Generate>> >; diff --git a/src/generators/ast-js/generate.mjs b/src/generators/ast-js/generate.mjs new file mode 100644 index 00000000..25f688ca --- /dev/null +++ b/src/generators/ast-js/generate.mjs @@ -0,0 +1,56 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { extname } from 'node:path'; + +import { parse } from 'acorn'; +import { globSync } from 'tinyglobby'; + +import getConfig from '../../utils/configuration/index.mjs'; + +/** + * Process a chunk of JavaScript files in a worker thread. + * Parses JS source files into AST representations. + * + * @type {import('./types').Implementation['processChunk']} + */ +export async function processChunk(inputSlice, itemIndices) { + const filePaths = itemIndices.map(idx => inputSlice[idx]); + + const results = []; + + for (const path of filePaths) { + const value = await readFile(path, 'utf-8'); + + const parsed = parse(value, { + allowReturnOutsideFunction: true, + ecmaVersion: 'latest', + locations: true, + }); + + parsed.path = path; + + results.push(parsed); + } + + return results; +} + +/** + * Generates a JavaScript AST from the input files. + * + * @type {import('./types').Implementation['generate']} + */ +export async function* generate(_, worker) { + const config = getConfig('ast-js'); + + const files = globSync(config.input, { ignore: config.ignore }).filter( + p => extname(p) === '.js' + ); + + // Parse the Javascript sources into ASTs in parallel using worker threads + // source is both the items list and the fullInput since we use sliceInput + for await (const chunkResult of worker.stream(files, files)) { + yield chunkResult; + } +} diff --git a/src/generators/ast-js/index.mjs b/src/generators/ast-js/index.mjs index 1e453930..6a771ccc 100644 --- a/src/generators/ast-js/index.mjs +++ b/src/generators/ast-js/index.mjs @@ -1,10 +1,4 @@ -import { readFile } from 'node:fs/promises'; -import { extname } from 'node:path'; - -import { parse } from 'acorn'; -import { globSync } from 'tinyglobby'; - -import getConfig from '../../utils/configuration/index.mjs'; +'use strict'; /** * This generator parses Javascript sources passed into the generator's input @@ -22,47 +16,4 @@ export default { version: '1.0.0', description: 'Parses Javascript source files passed into the input.', - - /** - * Process a chunk of JavaScript files in a worker thread. - * Parses JS source files into AST representations. - */ - async processChunk(inputSlice, itemIndices) { - const filePaths = itemIndices.map(idx => inputSlice[idx]); - - const results = []; - - for (const path of filePaths) { - const value = await readFile(path, 'utf-8'); - - const parsed = parse(value, { - allowReturnOutsideFunction: true, - ecmaVersion: 'latest', - locations: true, - }); - - parsed.path = path; - - results.push(parsed); - } - - return results; - }, - - /** - * Generates a JavaScript AST from the input files. - */ - async *generate(_, worker) { - const config = getConfig('ast-js'); - - const files = globSync(config.input, { ignore: config.ignore }).filter( - p => extname(p) === '.js' - ); - - // Parse the Javascript sources into ASTs in parallel using worker threads - // source is both the items list and the fullInput since we use sliceInput - for await (const chunkResult of worker.stream(files, files)) { - yield chunkResult; - } - }, }; diff --git a/src/generators/ast-js/types.d.ts b/src/generators/ast-js/types.d.ts index 70358fc9..2e992a22 100644 --- a/src/generators/ast-js/types.d.ts +++ b/src/generators/ast-js/types.d.ts @@ -1,5 +1,6 @@ -export type Generator = GeneratorMetadata< - {}, +export type Generator = GeneratorMetadata<{}>; + +export type Implementation = GeneratorImpl< Generate>, ProcessChunk >; diff --git a/src/generators/ast/generate.mjs b/src/generators/ast/generate.mjs new file mode 100644 index 00000000..b4da54e0 --- /dev/null +++ b/src/generators/ast/generate.mjs @@ -0,0 +1,58 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { extname } from 'node:path'; + +import { globSync } from 'tinyglobby'; +import { VFile } from 'vfile'; + +import getConfig from '../../utils/configuration/index.mjs'; +import createQueries from '../../utils/queries/index.mjs'; +import { getRemark } from '../../utils/remark.mjs'; + +const { updateStabilityPrefixToLink } = createQueries(); + +const remarkProcessor = getRemark(); + +/** + * Process a chunk of markdown files in a worker thread. + * Loads and parses markdown files into AST representations. + * + * @type {import('./types').Implementation['processChunk']} + */ +export async function processChunk(inputSlice, itemIndices) { + const filePaths = itemIndices.map(idx => inputSlice[idx]); + + const results = []; + + for (const path of filePaths) { + const vfile = new VFile({ path, value: await readFile(path, 'utf-8') }); + + updateStabilityPrefixToLink(vfile); + + results.push({ + tree: remarkProcessor.parse(vfile), + file: { stem: vfile.stem, basename: vfile.basename }, + }); + } + + return results; +} + +/** + * Generates AST trees from markdown input files. + * + * @type {import('./types').Implementation['generate']} + */ +export async function* generate(_, worker) { + const { ast: config } = getConfig(); + + const files = globSync(config.input, { ignore: config.ignore }).filter( + p => extname(p) === '.md' + ); + + // Parse markdown files in parallel using worker threads + for await (const chunkResult of worker.stream(files, files)) { + yield chunkResult; + } +} diff --git a/src/generators/ast/index.mjs b/src/generators/ast/index.mjs index 30ca8c97..6d3fbfdc 100644 --- a/src/generators/ast/index.mjs +++ b/src/generators/ast/index.mjs @@ -1,18 +1,5 @@ 'use strict'; -import { readFile } from 'node:fs/promises'; -import { extname } from 'node:path'; - -import { globSync } from 'tinyglobby'; -import { VFile } from 'vfile'; - -import getConfig from '../../utils/configuration/index.mjs'; -import createQueries from '../../utils/queries/index.mjs'; -import { getRemark } from '../../utils/remark.mjs'; - -const { updateStabilityPrefixToLink } = createQueries(); -const remarkProcessor = getRemark(); - /** * This generator parses Markdown API doc files into AST trees. * It parallelizes the parsing across worker threads for better performance. @@ -25,43 +12,4 @@ export default { version: '1.0.0', description: 'Parses Markdown API doc files into AST trees', - - /** - * Process a chunk of markdown files in a worker thread. - * Loads and parses markdown files into AST representations. - */ - async processChunk(inputSlice, itemIndices) { - const filePaths = itemIndices.map(idx => inputSlice[idx]); - - const results = []; - - for (const path of filePaths) { - const vfile = new VFile({ path, value: await readFile(path, 'utf-8') }); - - updateStabilityPrefixToLink(vfile); - - results.push({ - tree: remarkProcessor.parse(vfile), - file: { stem: vfile.stem, basename: vfile.basename }, - }); - } - - return results; - }, - - /** - * Generates AST trees from markdown input files. - */ - async *generate(_, worker) { - const { ast: config } = getConfig(); - - const files = globSync(config.input, { ignore: config.ignore }).filter( - p => extname(p) === '.md' - ); - - // Parse markdown files in parallel using worker threads - for await (const chunkResult of worker.stream(files, files)) { - yield chunkResult; - } - }, }; diff --git a/src/generators/ast/types.d.ts b/src/generators/ast/types.d.ts index 256ce4e7..03cba205 100644 --- a/src/generators/ast/types.d.ts +++ b/src/generators/ast/types.d.ts @@ -1,7 +1,8 @@ import type { Root } from 'mdast'; -export type Generator = GeneratorMetadata< - {}, +export type Generator = GeneratorMetadata<{}>; + +export type Implementation = GeneratorImpl< Generate>>, ProcessChunk> >; diff --git a/src/generators/index.mjs b/src/generators/index.mjs index cb6e503b..24294f92 100644 --- a/src/generators/index.mjs +++ b/src/generators/index.mjs @@ -1,38 +1,44 @@ 'use strict'; -import { lazy } from '../utils/misc.mjs'; - -/** - * Wraps a dynamic import into a lazy loader that resolves to the default export. - * - * @template T - * @param {() => Promise<{default: T}>} loader - * @returns {() => Promise} - */ -const lazyDefault = loader => lazy(() => loader().then(m => m.default)); +import addonVerify from './addon-verify/index.mjs'; +import apiLinks from './api-links/index.mjs'; +import ast from './ast/index.mjs'; +import astJs from './ast-js/index.mjs'; +import jsonSimple from './json-simple/index.mjs'; +import jsxAst from './jsx-ast/index.mjs'; +import legacyHtml from './legacy-html/index.mjs'; +import legacyHtmlAll from './legacy-html-all/index.mjs'; +import legacyJson from './legacy-json/index.mjs'; +import legacyJsonAll from './legacy-json-all/index.mjs'; +import llmsTxt from './llms-txt/index.mjs'; +import manPage from './man-page/index.mjs'; +import metadata from './metadata/index.mjs'; +import oramaDb from './orama-db/index.mjs'; +import sitemap from './sitemap/index.mjs'; +import web from './web/index.mjs'; export const publicGenerators = { - 'json-simple': lazyDefault(() => import('./json-simple/index.mjs')), - 'legacy-html': lazyDefault(() => import('./legacy-html/index.mjs')), - 'legacy-html-all': lazyDefault(() => import('./legacy-html-all/index.mjs')), - 'man-page': lazyDefault(() => import('./man-page/index.mjs')), - 'legacy-json': lazyDefault(() => import('./legacy-json/index.mjs')), - 'legacy-json-all': lazyDefault(() => import('./legacy-json-all/index.mjs')), - 'addon-verify': lazyDefault(() => import('./addon-verify/index.mjs')), - 'api-links': lazyDefault(() => import('./api-links/index.mjs')), - 'orama-db': lazyDefault(() => import('./orama-db/index.mjs')), - 'llms-txt': lazyDefault(() => import('./llms-txt/index.mjs')), - sitemap: lazyDefault(() => import('./sitemap/index.mjs')), - web: lazyDefault(() => import('./web/index.mjs')), + 'json-simple': jsonSimple, + 'legacy-html': legacyHtml, + 'legacy-html-all': legacyHtmlAll, + 'man-page': manPage, + 'legacy-json': legacyJson, + 'legacy-json-all': legacyJsonAll, + 'addon-verify': addonVerify, + 'api-links': apiLinks, + 'orama-db': oramaDb, + 'llms-txt': llmsTxt, + sitemap, + web, }; // These ones are special since they don't produce standard output, // and hence, we don't expose them to the CLI. const internalGenerators = { - ast: lazyDefault(() => import('./ast/index.mjs')), - metadata: lazyDefault(() => import('./metadata/index.mjs')), - 'jsx-ast': lazyDefault(() => import('./jsx-ast/index.mjs')), - 'ast-js': lazyDefault(() => import('./ast-js/index.mjs')), + ast, + metadata, + 'jsx-ast': jsxAst, + 'ast-js': astJs, }; export const allGenerators = { diff --git a/src/generators/json-simple/generate.mjs b/src/generators/json-simple/generate.mjs new file mode 100644 index 00000000..8e9188e0 --- /dev/null +++ b/src/generators/json-simple/generate.mjs @@ -0,0 +1,46 @@ +'use strict'; + +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { remove } from 'unist-util-remove'; + +import getConfig from '../../utils/configuration/index.mjs'; +import createQueries from '../../utils/queries/index.mjs'; + +/** + * Generates the simplified JSON version of the API docs + * + * @type {import('./types').Implementation['generate']} + */ +export async function generate(input) { + const config = getConfig('json-simple'); + + // Iterates the input (ApiDocMetadataEntry) and performs a few changes + const mappedInput = input.map(node => { + // Deep clones the content nodes to avoid affecting upstream nodes + const content = JSON.parse(JSON.stringify(node.content)); + + // Removes numerous nodes from the content that should not be on the "body" + // of the JSON version of the API docs as they are already represented in the metadata + remove(content, [ + createQueries.UNIST.isStabilityNode, + createQueries.UNIST.isHeading, + ]); + + return { ...node, content }; + }); + + if (config.output) { + // Writes all the API docs stringified content into one file + // Note: The full JSON generator in the future will create one JSON file per top-level API doc file + await writeFile( + join(config.output, 'api-docs.json'), + config.minify + ? JSON.stringify(mappedInput) + : JSON.stringify(mappedInput, null, 2) + ); + } + + return mappedInput; +} diff --git a/src/generators/json-simple/index.mjs b/src/generators/json-simple/index.mjs index f5ae6e02..b0fa3341 100644 --- a/src/generators/json-simple/index.mjs +++ b/src/generators/json-simple/index.mjs @@ -1,13 +1,5 @@ 'use strict'; -import { writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { remove } from 'unist-util-remove'; - -import getConfig from '../../utils/configuration/index.mjs'; -import createQueries from '../../utils/queries/index.mjs'; - /** * This generator generates a simplified JSON version of the API docs and returns it as a string * this is not meant to be used for the final API docs, but for debugging and testing purposes @@ -26,39 +18,4 @@ export default { 'Generates the simple JSON version of the API docs, and returns it as a string', dependsOn: 'metadata', - - /** - * Generates the simplified JSON version of the API docs - */ - async generate(input) { - const config = getConfig('json-simple'); - - // Iterates the input (ApiDocMetadataEntry) and performs a few changes - const mappedInput = input.map(node => { - // Deep clones the content nodes to avoid affecting upstream nodes - const content = JSON.parse(JSON.stringify(node.content)); - - // Removes numerous nodes from the content that should not be on the "body" - // of the JSON version of the API docs as they are already represented in the metadata - remove(content, [ - createQueries.UNIST.isStabilityNode, - createQueries.UNIST.isHeading, - ]); - - return { ...node, content }; - }); - - if (config.output) { - // Writes all the API docs stringified content into one file - // Note: The full JSON generator in the future will create one JSON file per top-level API doc file - await writeFile( - join(config.output, 'api-docs.json'), - config.minify - ? JSON.stringify(mappedInput) - : JSON.stringify(mappedInput, null, 2) - ); - } - - return mappedInput; - }, }; diff --git a/src/generators/json-simple/types.d.ts b/src/generators/json-simple/types.d.ts index 72960787..c1e53df6 100644 --- a/src/generators/json-simple/types.d.ts +++ b/src/generators/json-simple/types.d.ts @@ -1,4 +1,5 @@ -export type Generator = GeneratorMetadata< - {}, +export type Generator = GeneratorMetadata<{}>; + +export type Implementation = GeneratorImpl< Generate, Promise>> >; diff --git a/src/generators/jsx-ast/generate.mjs b/src/generators/jsx-ast/generate.mjs new file mode 100644 index 00000000..4e18ff74 --- /dev/null +++ b/src/generators/jsx-ast/generate.mjs @@ -0,0 +1,67 @@ +import { buildSideBarProps } from './utils/buildBarProps.mjs'; +import buildContent from './utils/buildContent.mjs'; +import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { groupNodesByModule } from '../../utils/generators.mjs'; +import { getRemarkRecma } from '../../utils/remark.mjs'; + +const remarkRecma = getRemarkRecma(); + +/** + * Process a chunk of items in a worker thread. + * Transforms metadata entries into JSX AST nodes. + * + * Each item is a SlicedModuleInput containing the head node + * and all entries for that module - no need to recompute grouping. + * + * @type {import('./types').Implementation['processChunk']} + */ +export async function processChunk(slicedInput, itemIndices, docPages) { + const results = []; + + for (const idx of itemIndices) { + const { head, entries } = slicedInput[idx]; + + const sideBarProps = buildSideBarProps(head, docPages); + + const content = await buildContent( + entries, + head, + sideBarProps, + remarkRecma + ); + + results.push(content); + } + + return results; +} + +/** + * Generates a JSX AST + * + * @type {import('./types').Implementation['generate']} + */ +export async function* generate(input, worker) { + const config = getConfig('jsx-ast'); + + const groupedModules = groupNodesByModule(input); + const headNodes = getSortedHeadNodes(input); + + // Pre-compute docPages once in main thread + // TODO(@avivkeller): Load the index file here instead of during configuration + const docPages = config.index + ? config.index.map(({ section, api }) => [section, `${api}.html`]) + : headNodes.map(node => [node.heading.data.name, `${node.api}.html`]); + + // Create sliced input: each item contains head + its module's entries + // This avoids sending all 4700+ entries to every worker + const entries = headNodes.map(head => ({ + head, + entries: groupedModules.get(head.api), + })); + + for await (const chunkResult of worker.stream(entries, entries, docPages)) { + yield chunkResult; + } +} diff --git a/src/generators/jsx-ast/index.mjs b/src/generators/jsx-ast/index.mjs index e1ee73a1..435f98e8 100644 --- a/src/generators/jsx-ast/index.mjs +++ b/src/generators/jsx-ast/index.mjs @@ -1,11 +1,4 @@ -import { buildSideBarProps } from './utils/buildBarProps.mjs'; -import buildContent from './utils/buildContent.mjs'; -import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { groupNodesByModule } from '../../utils/generators.mjs'; -import { getRemarkRecma } from '../../utils/remark.mjs'; - -const remarkRecma = getRemarkRecma(); +'use strict'; /** * Generator for converting MDAST to JSX AST. @@ -24,59 +17,4 @@ export default { defaultConfiguration: { ref: 'main', }, - - /** - * Process a chunk of items in a worker thread. - * Transforms metadata entries into JSX AST nodes. - * - * Each item is a SlicedModuleInput containing the head node - * and all entries for that module - no need to recompute grouping. - */ - async processChunk(slicedInput, itemIndices, docPages) { - const results = []; - - for (const idx of itemIndices) { - const { head, entries } = slicedInput[idx]; - - const sideBarProps = buildSideBarProps(head, docPages); - - const content = await buildContent( - entries, - head, - sideBarProps, - remarkRecma - ); - - results.push(content); - } - - return results; - }, - - /** - * Generates a JSX AST - */ - async *generate(input, worker) { - const config = getConfig('jsx-ast'); - - const groupedModules = groupNodesByModule(input); - const headNodes = getSortedHeadNodes(input); - - // Pre-compute docPages once in main thread - // TODO(@avivkeller): Load the index file here instead of during configuration - const docPages = config.index - ? config.index.map(({ section, api }) => [section, `${api}.html`]) - : headNodes.map(node => [node.heading.data.name, `${node.api}.html`]); - - // Create sliced input: each item contains head + its module's entries - // This avoids sending all 4700+ entries to every worker - const entries = headNodes.map(head => ({ - head, - entries: groupedModules.get(head.api), - })); - - for await (const chunkResult of worker.stream(entries, entries, docPages)) { - yield chunkResult; - } - }, }; diff --git a/src/generators/jsx-ast/types.d.ts b/src/generators/jsx-ast/types.d.ts index 9103bea1..4bb81b5d 100644 --- a/src/generators/jsx-ast/types.d.ts +++ b/src/generators/jsx-ast/types.d.ts @@ -1,7 +1,8 @@ import type { JSXContent } from './utils/buildContent.mjs'; -export type Generator = GeneratorMetadata< - {}, +export type Generator = GeneratorMetadata<{}>; + +export type Implementation = GeneratorImpl< Generate, AsyncGenerator>, ProcessChunk< { head: ApiDocMetadataEntry; entries: Array }, diff --git a/src/generators/jsx-ast/utils/buildBarProps.mjs b/src/generators/jsx-ast/utils/buildBarProps.mjs index c995dbf3..3e7c7b67 100644 --- a/src/generators/jsx-ast/utils/buildBarProps.mjs +++ b/src/generators/jsx-ast/utils/buildBarProps.mjs @@ -1,10 +1,12 @@ +'use strict'; + import readingTime from 'reading-time'; import { visit } from 'unist-util-visit'; import { getFullName } from './buildSignature.mjs'; import getConfig from '../../../utils/configuration/index.mjs'; import { - GITHUB_EDIT_URL, + GH_EDIT_URL, populate, } from '../../../utils/configuration/templates.mjs'; import { @@ -101,7 +103,7 @@ export const buildMetaBarProps = (head, entries) => { ['JSON', `${head.api}.json`], ['MD', `${head.api}.md`], ], - editThisPage: `${populate(GITHUB_EDIT_URL, config)}${head.api}.md`, + editThisPage: `${populate(GH_EDIT_URL, config)}${head.api}.md`, }; }; diff --git a/src/generators/jsx-ast/utils/buildContent.mjs b/src/generators/jsx-ast/utils/buildContent.mjs index a938cb4f..cd9736ab 100644 --- a/src/generators/jsx-ast/utils/buildContent.mjs +++ b/src/generators/jsx-ast/utils/buildContent.mjs @@ -1,3 +1,5 @@ +'use strict'; + import { h as createElement } from 'hastscript'; import { slice } from 'mdast-util-slice-markdown'; import { u as createTree } from 'unist-builder'; @@ -22,7 +24,7 @@ import { import insertSignature, { getFullName } from './buildSignature.mjs'; import getConfig from '../../../utils/configuration/index.mjs'; import { - GITHUB_BLOB_URL, + GH_BLOB_URL, populate, } from '../../../utils/configuration/templates.mjs'; @@ -81,7 +83,7 @@ export const createSourceLink = sourceLink => { createElement( 'a', { - href: `${populate(GITHUB_BLOB_URL, config)}${sourceLink}`, + href: `${populate(GH_BLOB_URL, config)}${sourceLink}`, target: '_blank', }, [ diff --git a/src/generators/legacy-html-all/generate.mjs b/src/generators/legacy-html-all/generate.mjs new file mode 100644 index 00000000..7d85bfd3 --- /dev/null +++ b/src/generators/legacy-html-all/generate.mjs @@ -0,0 +1,74 @@ +'use strict'; + +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import getConfig from '../../utils/configuration/index.mjs'; +import { minifyHTML } from '../../utils/html-minifier.mjs'; +import { getRemarkRehype } from '../../utils/remark.mjs'; +import { replaceTemplateValues } from '../legacy-html/utils/replaceTemplateValues.mjs'; +import tableOfContents from '../legacy-html/utils/tableOfContents.mjs'; + +/** + * Generates the `all.html` file from the `legacy-html` generator + * + * @type {import('./types').Implementation['generate']} + */ +export async function generate(input) { + const config = getConfig('legacy-html-all'); + + // Gets a Remark Processor that parses Markdown to minified HTML + const remarkWithRehype = getRemarkRehype(); + + // Reads the API template.html file to be used as a base for the HTML files + const apiTemplate = await readFile(config.templatePath, 'utf-8'); + + // Filter out index entries and extract needed properties + const entries = input.filter(entry => entry.api !== 'index'); + + // Aggregates all individual Table of Contents into one giant string + const aggregatedToC = entries.map(entry => entry.toc).join('\n'); + + // Aggregates all individual content into one giant string + const aggregatedContent = entries.map(entry => entry.content).join('\n'); + + // Creates a "mimic" of an `ApiDocMetadataEntry` which fulfils the requirements + // for generating the `tableOfContents` with the `tableOfContents.parseNavigationNode` parser + const sideNavigationFromValues = entries.map(entry => ({ + api: entry.api, + heading: { data: { depth: 1, name: entry.section } }, + })); + + // Generates the global Table of Contents (Sidebar Navigation) + const parsedSideNav = remarkWithRehype.processSync( + tableOfContents(sideNavigationFromValues, { + maxDepth: 1, + parser: tableOfContents.parseNavigationNode, + }) + ); + + const templateValues = { + api: 'all', + added: '', + section: 'All', + version: `v${config.version.version}`, + toc: aggregatedToC, + nav: String(parsedSideNav), + content: aggregatedContent, + }; + + let result = replaceTemplateValues(apiTemplate, templateValues, config, { + skipGitHub: true, + skipGtocPicker: true, + }); + + if (config.minify) { + result = Buffer.from(await minifyHTML(result)); + } + + if (config.output) { + await writeFile(join(config.output, 'all.html'), result); + } + + return result; +} diff --git a/src/generators/legacy-html-all/index.mjs b/src/generators/legacy-html-all/index.mjs index 96e2808e..ce364e78 100644 --- a/src/generators/legacy-html-all/index.mjs +++ b/src/generators/legacy-html-all/index.mjs @@ -1,14 +1,6 @@ 'use strict'; -import { readFile, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import getConfig from '../../utils/configuration/index.mjs'; -import { minifyHTML } from '../../utils/html-minifier.mjs'; -import { getRemarkRehype } from '../../utils/remark.mjs'; import legacyHtml from '../legacy-html/index.mjs'; -import { replaceTemplateValues } from '../legacy-html/utils/replaceTemplateValues.mjs'; -import tableOfContents from '../legacy-html/utils/tableOfContents.mjs'; /** * This generator generates the legacy HTML pages of the legacy API docs @@ -32,66 +24,4 @@ export default { defaultConfiguration: { templatePath: legacyHtml.defaultConfiguration.templatePath, }, - - /** - * Generates the `all.html` file from the `legacy-html` generator - */ - async generate(input) { - const config = getConfig('legacy-html-all'); - - // Gets a Remark Processor that parses Markdown to minified HTML - const remarkWithRehype = getRemarkRehype(); - - // Reads the API template.html file to be used as a base for the HTML files - const apiTemplate = await readFile(config.templatePath, 'utf-8'); - - // Filter out index entries and extract needed properties - const entries = input.filter(entry => entry.api !== 'index'); - - // Aggregates all individual Table of Contents into one giant string - const aggregatedToC = entries.map(entry => entry.toc).join('\n'); - - // Aggregates all individual content into one giant string - const aggregatedContent = entries.map(entry => entry.content).join('\n'); - - // Creates a "mimic" of an `ApiDocMetadataEntry` which fulfils the requirements - // for generating the `tableOfContents` with the `tableOfContents.parseNavigationNode` parser - const sideNavigationFromValues = entries.map(entry => ({ - api: entry.api, - heading: { data: { depth: 1, name: entry.section } }, - })); - - // Generates the global Table of Contents (Sidebar Navigation) - const parsedSideNav = remarkWithRehype.processSync( - tableOfContents(sideNavigationFromValues, { - maxDepth: 1, - parser: tableOfContents.parseNavigationNode, - }) - ); - - const templateValues = { - api: 'all', - added: '', - section: 'All', - version: `v${config.version.version}`, - toc: aggregatedToC, - nav: String(parsedSideNav), - content: aggregatedContent, - }; - - let result = replaceTemplateValues(apiTemplate, templateValues, config, { - skipGitHub: true, - skipGtocPicker: true, - }); - - if (config.minify) { - result = Buffer.from(await minifyHTML(result)); - } - - if (config.output) { - await writeFile(join(config.output, 'all.html'), result); - } - - return result; - }, }; diff --git a/src/generators/legacy-html-all/types.d.ts b/src/generators/legacy-html-all/types.d.ts index e005d1ff..5d69700a 100644 --- a/src/generators/legacy-html-all/types.d.ts +++ b/src/generators/legacy-html-all/types.d.ts @@ -8,9 +8,10 @@ export interface TemplateValues { content: string; } -export type Generator = GeneratorMetadata< - { - templatePath: string; - }, +export type Generator = GeneratorMetadata<{ + templatePath: string; +}>; + +export type Implementation = GeneratorImpl< Generate, Promise> >; diff --git a/src/generators/legacy-html/generate.mjs b/src/generators/legacy-html/generate.mjs new file mode 100644 index 00000000..210e045f --- /dev/null +++ b/src/generators/legacy-html/generate.mjs @@ -0,0 +1,142 @@ +'use strict'; + +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { basename, join } from 'node:path'; + +import buildContent from './utils/buildContent.mjs'; +import { replaceTemplateValues } from './utils/replaceTemplateValues.mjs'; +import { safeCopy } from './utils/safeCopy.mjs'; +import tableOfContents from './utils/tableOfContents.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { groupNodesByModule } from '../../utils/generators.mjs'; +import { minifyHTML } from '../../utils/html-minifier.mjs'; +import { getRemarkRehypeWithShiki } from '../../utils/remark.mjs'; + +/** + * Creates a heading object with the given name. + * @param {string} name - The name of the heading + * @returns {HeadingMetadataEntry} The heading object + */ +const getHeading = name => ({ data: { depth: 1, name } }); + +const remarkRehypeProcessor = getRemarkRehypeWithShiki(); + +/** + * Process a chunk of items in a worker thread. + * Builds HTML template objects - FS operations happen in generate(). + * + * Each item is pre-grouped {head, nodes, headNodes} - no need to + * recompute groupNodesByModule for every chunk. + * + * @type {import('./types').Implementation['processChunk']} + */ +export async function processChunk(slicedInput, itemIndices, navigation) { + const results = []; + + for (const idx of itemIndices) { + const { head, nodes, headNodes } = slicedInput[idx]; + + const nav = navigation.replace( + `class="nav-${head.api}`, + `class="nav-${head.api} active` + ); + + const toc = String( + remarkRehypeProcessor.processSync( + tableOfContents(nodes, { + maxDepth: 4, + parser: tableOfContents.parseToCNode, + }) + ) + ); + + const content = buildContent(headNodes, nodes, remarkRehypeProcessor); + + const apiAsHeading = head.api.charAt(0).toUpperCase() + head.api.slice(1); + + const template = { + api: head.api, + added: head.introduced_in ?? '', + section: head.heading.data.name || apiAsHeading, + toc, + nav, + content, + }; + + results.push(template); + } + + return results; +} + +/** + * Generates the legacy version of the API docs in HTML + * + * @type {import('./types').Implementation['generate']} + */ +export async function* generate(input, worker) { + const config = getConfig('legacy-html'); + + const apiTemplate = await readFile(config.templatePath, 'utf-8'); + + const groupedModules = groupNodesByModule(input); + + const headNodes = input + .filter(node => node.heading.depth === 1) + .toSorted((a, b) => a.heading.data.name.localeCompare(b.heading.data.name)); + + const indexOfFiles = config.index + ? config.index.map(({ api, section }) => ({ + api, + heading: getHeading(section), + })) + : headNodes; + + const navigation = String( + remarkRehypeProcessor.processSync( + tableOfContents(indexOfFiles, { + maxDepth: 1, + parser: tableOfContents.parseNavigationNode, + }) + ) + ); + + if (config.output) { + for (const path of config.additionalPathsToCopy) { + // Define the output folder for API docs assets + const assetsFolder = join(config.output, basename(path)); + + // Creates the assets folder if it does not exist + await mkdir(assetsFolder, { recursive: true }); + + // Copy all files from assets folder to output, skipping unchanged files + await safeCopy(path, assetsFolder); + } + } + + // Create sliced input: each item contains head + its module's entries + headNodes reference + // This avoids sending all ~4900 entries to every worker and recomputing groupings + const entries = headNodes.map(head => ({ + head, + nodes: groupedModules.get(head.api), + headNodes, + })); + + // Stream chunks as they complete - HTML files are written immediately + for await (const chunkResult of worker.stream(entries, entries, navigation)) { + // Write files for this chunk in the generate method (main thread) + if (config.output) { + for (const template of chunkResult) { + let result = replaceTemplateValues(apiTemplate, template, config); + + if (config.minify) { + result = Buffer.from(await minifyHTML(result)); + } + + await writeFile(join(config.output, `${template.api}.html`), result); + } + } + + yield chunkResult; + } +} diff --git a/src/generators/legacy-html/index.mjs b/src/generators/legacy-html/index.mjs index d647fe08..d5b3c56d 100644 --- a/src/generators/legacy-html/index.mjs +++ b/src/generators/legacy-html/index.mjs @@ -1,25 +1,6 @@ 'use strict'; -import { readFile, writeFile, mkdir } from 'node:fs/promises'; -import { basename, join } from 'node:path'; - -import buildContent from './utils/buildContent.mjs'; -import { replaceTemplateValues } from './utils/replaceTemplateValues.mjs'; -import { safeCopy } from './utils/safeCopy.mjs'; -import tableOfContents from './utils/tableOfContents.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { groupNodesByModule } from '../../utils/generators.mjs'; -import { minifyHTML } from '../../utils/html-minifier.mjs'; -import { getRemarkRehypeWithShiki } from '../../utils/remark.mjs'; - -/** - * Creates a heading object with the given name. - * @param {string} name - The name of the heading - * @returns {HeadingMetadataEntry} The heading object - */ -const getHeading = name => ({ data: { depth: 1, name } }); - -const remarkRehypeProcessor = getRemarkRehypeWithShiki(); +import { join } from 'node:path'; /** * @@ -46,126 +27,4 @@ export default { additionalPathsToCopy: [join(import.meta.dirname, 'assets')], ref: 'main', }, - - /** - * Process a chunk of items in a worker thread. - * Builds HTML template objects - FS operations happen in generate(). - * - * Each item is pre-grouped {head, nodes, headNodes} - no need to - * recompute groupNodesByModule for every chunk. - */ - async processChunk(slicedInput, itemIndices, navigation) { - const results = []; - - for (const idx of itemIndices) { - const { head, nodes, headNodes } = slicedInput[idx]; - - const nav = navigation.replace( - `class="nav-${head.api}`, - `class="nav-${head.api} active` - ); - - const toc = String( - remarkRehypeProcessor.processSync( - tableOfContents(nodes, { - maxDepth: 4, - parser: tableOfContents.parseToCNode, - }) - ) - ); - - const content = buildContent(headNodes, nodes, remarkRehypeProcessor); - - const apiAsHeading = head.api.charAt(0).toUpperCase() + head.api.slice(1); - - const template = { - api: head.api, - added: head.introduced_in ?? '', - section: head.heading.data.name || apiAsHeading, - toc, - nav, - content, - }; - - results.push(template); - } - - return results; - }, - - /** - * Generates the legacy version of the API docs in HTML - */ - async *generate(input, worker) { - const config = getConfig('legacy-html'); - - const apiTemplate = await readFile(config.templatePath, 'utf-8'); - - const groupedModules = groupNodesByModule(input); - - const headNodes = input - .filter(node => node.heading.depth === 1) - .toSorted((a, b) => - a.heading.data.name.localeCompare(b.heading.data.name) - ); - - const indexOfFiles = config.index - ? config.index.map(({ api, section }) => ({ - api, - heading: getHeading(section), - })) - : headNodes; - - const navigation = String( - remarkRehypeProcessor.processSync( - tableOfContents(indexOfFiles, { - maxDepth: 1, - parser: tableOfContents.parseNavigationNode, - }) - ) - ); - - if (config.output) { - for (const path of config.additionalPathsToCopy) { - // Define the output folder for API docs assets - const assetsFolder = join(config.output, basename(path)); - - // Creates the assets folder if it does not exist - await mkdir(assetsFolder, { recursive: true }); - - // Copy all files from assets folder to output, skipping unchanged files - await safeCopy(path, assetsFolder); - } - } - - // Create sliced input: each item contains head + its module's entries + headNodes reference - // This avoids sending all ~4900 entries to every worker and recomputing groupings - const entries = headNodes.map(head => ({ - head, - nodes: groupedModules.get(head.api), - headNodes, - })); - - // Stream chunks as they complete - HTML files are written immediately - for await (const chunkResult of worker.stream( - entries, - entries, - navigation - )) { - // Write files for this chunk in the generate method (main thread) - if (config.output) { - for (const template of chunkResult) { - let result = replaceTemplateValues(apiTemplate, template, config); - - if (config.minify) { - result = Buffer.from(await minifyHTML(result)); - } - - await writeFile(join(config.output, `${template.api}.html`), result); - } - } - - yield chunkResult; - } - }, }; diff --git a/src/generators/legacy-html/types.d.ts b/src/generators/legacy-html/types.d.ts index e0222dfa..c7732aae 100644 --- a/src/generators/legacy-html/types.d.ts +++ b/src/generators/legacy-html/types.d.ts @@ -7,11 +7,12 @@ export interface TemplateValues { content: string; } -export type Generator = GeneratorMetadata< - { - templatePath: string; - additionalPathsToCopy: Array; - }, +export type Generator = GeneratorMetadata<{ + templatePath: string; + additionalPathsToCopy: Array; +}>; + +export type Implementation = GeneratorImpl< Generate, AsyncGenerator>, ProcessChunk< { diff --git a/src/generators/legacy-html/utils/buildContent.mjs b/src/generators/legacy-html/utils/buildContent.mjs index af22e8e3..8ab0c7ef 100644 --- a/src/generators/legacy-html/utils/buildContent.mjs +++ b/src/generators/legacy-html/utils/buildContent.mjs @@ -7,7 +7,7 @@ import { SKIP, visit } from 'unist-util-visit'; import buildExtraContent from './buildExtraContent.mjs'; import getConfig from '../../../utils/configuration/index.mjs'; import { - GITHUB_BLOB_URL, + GH_BLOB_URL, populate, } from '../../../utils/configuration/templates.mjs'; import createQueries from '../../../utils/queries/index.mjs'; @@ -116,7 +116,7 @@ const buildMetadataElement = (node, remark) => { // We use a `span` element to display the source link as a clickable link to the source within Node.js if (typeof node.source_link === 'string') { // Creates the source link URL with the base URL and the source link - const sourceLink = `${populate(GITHUB_BLOB_URL, config)}${node.source_link}`; + const sourceLink = `${populate(GH_BLOB_URL, config)}${node.source_link}`; // Creates the source link element with the source link and the source link text const sourceLinkElement = createElement('span', [ diff --git a/src/generators/legacy-html/utils/replaceTemplateValues.mjs b/src/generators/legacy-html/utils/replaceTemplateValues.mjs index 460592d3..c2e39f09 100644 --- a/src/generators/legacy-html/utils/replaceTemplateValues.mjs +++ b/src/generators/legacy-html/utils/replaceTemplateValues.mjs @@ -8,7 +8,7 @@ import { } from './buildDropdowns.mjs'; import tableOfContents from './tableOfContents.mjs'; import { - GITHUB_EDIT_URL, + GH_EDIT_URL, populate, } from '../../../utils/configuration/templates.mjs'; @@ -39,8 +39,6 @@ export const replaceTemplateValues = ( .replace('__ALTDOCS__', buildVersions(api, added, config.changelog)) .replace( '__EDIT_ON_GITHUB__', - skipGitHub - ? '' - : buildGitHub(`${populate(GITHUB_EDIT_URL, config)}${api}.md`) + skipGitHub ? '' : buildGitHub(`${populate(GH_EDIT_URL, config)}${api}.md`) ); }; diff --git a/src/generators/legacy-json-all/generate.mjs b/src/generators/legacy-json-all/generate.mjs new file mode 100644 index 00000000..4deb7e68 --- /dev/null +++ b/src/generators/legacy-json-all/generate.mjs @@ -0,0 +1,80 @@ +'use strict'; + +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import getConfig from '../../utils/configuration/index.mjs'; +import { legacyToJSON } from '../../utils/generators.mjs'; + +/** + * Generates the legacy JSON `all.json` file. + * + * @type {import('./types').Implementation['generate']} + */ +export async function generate(input) { + const config = getConfig('legacy-json-all'); + + /** + * The consolidated output object that will contain + * combined data from all sections in the input. + * + * @type {import('./types.d.ts').Output} + */ + const generatedValue = { + miscs: [], + modules: [], + classes: [], + globals: [], + methods: [], + }; + + /** + * The properties to copy from each section in the input + */ + const propertiesToCopy = Object.keys(generatedValue); + + // Create a map of api name to index position for sorting + const indexOrder = new Map( + config.index?.map(({ api }, position) => [`doc/api/${api}.md`, position]) ?? + [] + ); + + // Sort input by index order (documents not in index go to the end) + const sortedInput = input.toSorted((a, b) => { + const aOrder = indexOrder.get(a.source) ?? Infinity; + const bOrder = indexOrder.get(b.source) ?? Infinity; + + return aOrder - bOrder; + }); + + // Aggregate all sections into the output + for (const section of sortedInput) { + // Skip index.json - it has no useful content, just navigation + if (section.api === 'index') { + continue; + } + + for (const property of propertiesToCopy) { + const items = section[property]; + + if (Array.isArray(items)) { + const enrichedItems = section.source + ? items.map(item => ({ ...item, source: section.source })) + : items; + + generatedValue[property].push(...enrichedItems); + } + } + } + + if (config.output) { + await writeFile( + join(config.output, 'all.json'), + config.minify + ? legacyToJSON(generatedValue) + : legacyToJSON(generatedValue, null, 2) + ); + } + + return generatedValue; +} diff --git a/src/generators/legacy-json-all/index.mjs b/src/generators/legacy-json-all/index.mjs index cfd71b5a..cce57ea0 100644 --- a/src/generators/legacy-json-all/index.mjs +++ b/src/generators/legacy-json-all/index.mjs @@ -1,11 +1,5 @@ 'use strict'; -import { writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import getConfig from '../../utils/configuration/index.mjs'; -import { legacyToJSON } from '../../utils/generators.mjs'; - /** * This generator consolidates data from the `legacy-json` generator into a single * JSON file (`all.json`). @@ -25,77 +19,4 @@ export default { defaultConfiguration: { minify: false, }, - - /** - * Generates the legacy JSON `all.json` file. - */ - async generate(input) { - const config = getConfig('legacy-json-all'); - - /** - * The consolidated output object that will contain - * combined data from all sections in the input. - * - * @type {import('./types.d.ts').Output} - */ - const generatedValue = { - miscs: [], - modules: [], - classes: [], - globals: [], - methods: [], - }; - - /** - * The properties to copy from each section in the input - */ - const propertiesToCopy = Object.keys(generatedValue); - - // Create a map of api name to index position for sorting - const indexOrder = new Map( - config.index?.map(({ api }, position) => [ - `doc/api/${api}.md`, - position, - ]) ?? [] - ); - - // Sort input by index order (documents not in index go to the end) - const sortedInput = input.toSorted((a, b) => { - const aOrder = indexOrder.get(a.source) ?? Infinity; - const bOrder = indexOrder.get(b.source) ?? Infinity; - - return aOrder - bOrder; - }); - - // Aggregate all sections into the output - for (const section of sortedInput) { - // Skip index.json - it has no useful content, just navigation - if (section.api === 'index') { - continue; - } - - for (const property of propertiesToCopy) { - const items = section[property]; - - if (Array.isArray(items)) { - const enrichedItems = section.source - ? items.map(item => ({ ...item, source: section.source })) - : items; - - generatedValue[property].push(...enrichedItems); - } - } - } - - if (config.output) { - await writeFile( - join(config.output, 'all.json'), - config.minify - ? legacyToJSON(generatedValue) - : legacyToJSON(generatedValue, null, 2) - ); - } - - return generatedValue; - }, }; diff --git a/src/generators/legacy-json-all/types.d.ts b/src/generators/legacy-json-all/types.d.ts index d95ffd14..77cf7365 100644 --- a/src/generators/legacy-json-all/types.d.ts +++ b/src/generators/legacy-json-all/types.d.ts @@ -13,7 +13,8 @@ export interface Output { methods: Array; } -export type Generator = GeneratorMetadata< - {}, +export type Generator = GeneratorMetadata<{}>; + +export type Implementation = GeneratorImpl< Generate, Promise> >; diff --git a/src/generators/legacy-json/generate.mjs b/src/generators/legacy-json/generate.mjs new file mode 100644 index 00000000..f41daf3d --- /dev/null +++ b/src/generators/legacy-json/generate.mjs @@ -0,0 +1,66 @@ +'use strict'; + +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { createSectionBuilder } from './utils/buildSection.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { groupNodesByModule, legacyToJSON } from '../../utils/generators.mjs'; + +const buildSection = createSectionBuilder(); + +/** + * Process a chunk of items in a worker thread. + * Builds JSON sections - FS operations happen in generate(). + * + * Each item is pre-grouped {head, nodes} - no need to + * recompute groupNodesByModule for every chunk. + * + * @type {import('./types').Implementation['processChunk']} + */ +export async function processChunk(slicedInput, itemIndices) { + const results = []; + + for (const idx of itemIndices) { + const { head, nodes } = slicedInput[idx]; + + results.push(buildSection(head, nodes)); + } + + return results; +} + +/** + * Generates a legacy JSON file. + * + * @type {import('./types').Implementation['generate']} + */ +export async function* generate(input, worker) { + const config = getConfig('legacy-json'); + + const groupedModules = groupNodesByModule(input); + + const headNodes = input.filter(node => node.heading.depth === 1); + + // Create sliced input: each item contains head + its module's entries + // This avoids sending all 4900+ entries to every worker + const entries = headNodes.map(head => ({ + head, + nodes: groupedModules.get(head.api), + })); + + for await (const chunkResult of worker.stream(entries, entries)) { + if (config.output) { + for (const section of chunkResult) { + const out = join(config.output, `${section.api}.json`); + + await writeFile( + out, + config.minify ? legacyToJSON(section) : legacyToJSON(section, null, 2) + ); + } + } + + yield chunkResult; + } +} diff --git a/src/generators/legacy-json/index.mjs b/src/generators/legacy-json/index.mjs index 5c4ef787..009542e8 100644 --- a/src/generators/legacy-json/index.mjs +++ b/src/generators/legacy-json/index.mjs @@ -1,14 +1,5 @@ 'use strict'; -import { writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { createSectionBuilder } from './utils/buildSection.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { groupNodesByModule, legacyToJSON } from '../../utils/generators.mjs'; - -const buildSection = createSectionBuilder(); - /** * This generator is responsible for generating the legacy JSON files for the * legacy API docs for retro-compatibility. It is to be replaced while we work @@ -33,58 +24,4 @@ export default { ref: 'main', minify: false, }, - - /** - * Process a chunk of items in a worker thread. - * Builds JSON sections - FS operations happen in generate(). - * - * Each item is pre-grouped {head, nodes} - no need to - * recompute groupNodesByModule for every chunk. - */ - async processChunk(slicedInput, itemIndices) { - const results = []; - - for (const idx of itemIndices) { - const { head, nodes } = slicedInput[idx]; - - results.push(buildSection(head, nodes)); - } - - return results; - }, - - /** - * Generates a legacy JSON file. - */ - async *generate(input, worker) { - const config = getConfig('legacy-json'); - - const groupedModules = groupNodesByModule(input); - - const headNodes = input.filter(node => node.heading.depth === 1); - - // Create sliced input: each item contains head + its module's entries - // This avoids sending all 4900+ entries to every worker - const entries = headNodes.map(head => ({ - head, - nodes: groupedModules.get(head.api), - })); - - for await (const chunkResult of worker.stream(entries, entries)) { - if (config.output) { - for (const section of chunkResult) { - const out = join(config.output, `${section.api}.json`); - - await writeFile( - out, - config.minify - ? legacyToJSON(section) - : legacyToJSON(section, null, 2) - ); - } - } - - yield chunkResult; - } - }, }; diff --git a/src/generators/legacy-json/types.d.ts b/src/generators/legacy-json/types.d.ts index 16649acc..0c207253 100644 --- a/src/generators/legacy-json/types.d.ts +++ b/src/generators/legacy-json/types.d.ts @@ -279,8 +279,9 @@ export interface ParameterList { options?: ParameterList; } -export type Generator = GeneratorMetadata< - {}, +export type Generator = GeneratorMetadata<{}>; + +export type Implementation = GeneratorImpl< Generate, AsyncGenerator
>, ProcessChunk< { head: ApiDocMetadataEntry; nodes: Array }, diff --git a/src/generators/llms-txt/generate.mjs b/src/generators/llms-txt/generate.mjs new file mode 100644 index 00000000..76e44cf0 --- /dev/null +++ b/src/generators/llms-txt/generate.mjs @@ -0,0 +1,31 @@ +'use strict'; + +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { buildApiDocLink } from './utils/buildApiDocLink.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; + +/** + * Generates a llms.txt file + * + * @type {import('./types').Implementation['generate']} + */ +export async function generate(input) { + const config = getConfig('llms-txt'); + + const template = await readFile(config.templatePath, 'utf-8'); + + const apiDocsLinks = input + .filter(entry => entry.heading.depth === 1) + .map(entry => `- ${buildApiDocLink(entry, config.baseURL)}`) + .join('\n'); + + const filledTemplate = `${template}${apiDocsLinks}`; + + if (config.output) { + await writeFile(join(config.output, 'llms.txt'), filledTemplate); + } + + return filledTemplate; +} diff --git a/src/generators/llms-txt/index.mjs b/src/generators/llms-txt/index.mjs index 3874519e..f94a1bbf 100644 --- a/src/generators/llms-txt/index.mjs +++ b/src/generators/llms-txt/index.mjs @@ -1,8 +1,6 @@ -import { readFile, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; +'use strict'; -import { buildApiDocLink } from './utils/buildApiDocLink.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; +import { join } from 'node:path'; /** * This generator generates a llms.txt file to provide information to LLMs at @@ -23,26 +21,4 @@ export default { defaultConfiguration: { templatePath: join(import.meta.dirname, 'template.txt'), }, - - /** - * Generates a llms.txt file - */ - async generate(input) { - const config = getConfig('llms-txt'); - - const template = await readFile(config.templatePath, 'utf-8'); - - const apiDocsLinks = input - .filter(entry => entry.heading.depth === 1) - .map(entry => `- ${buildApiDocLink(entry, config.baseURL)}`) - .join('\n'); - - const filledTemplate = `${template}${apiDocsLinks}`; - - if (config.output) { - await writeFile(join(config.output, 'llms.txt'), filledTemplate); - } - - return filledTemplate; - }, }; diff --git a/src/generators/llms-txt/types.d.ts b/src/generators/llms-txt/types.d.ts index 693195fe..80b58191 100644 --- a/src/generators/llms-txt/types.d.ts +++ b/src/generators/llms-txt/types.d.ts @@ -1,6 +1,7 @@ -export type Generator = GeneratorMetadata< - { - templatePath: string; - }, +export type Generator = GeneratorMetadata<{ + templatePath: string; +}>; + +export type Implementation = GeneratorImpl< Generate, Promise> >; diff --git a/src/generators/man-page/generate.mjs b/src/generators/man-page/generate.mjs new file mode 100644 index 00000000..3f8aae65 --- /dev/null +++ b/src/generators/man-page/generate.mjs @@ -0,0 +1,84 @@ +'use strict'; + +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { + convertOptionToMandoc, + convertEnvVarToMandoc, +} from './utils/converter.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; + +/** + * @param {Array} components + * @param {number} start + * @param {number} end + * @param {(element: ApiDocMetadataEntry) => string} convert + * @returns {string} + */ +function extractMandoc(components, start, end, convert) { + return components + .slice(start, end) + .filter(({ heading }) => heading.depth === 3) + .map(convert) + .join(''); +} + +/** + * Generates the Node.js man-page + * + * @type {import('./types').Implementation['generate']} + */ +export async function generate(input) { + const config = getConfig('man-page'); + + // Filter to only 'cli'. + const components = input.filter(({ api }) => api === 'cli'); + + if (!components.length) { + throw new Error('Could not find any `cli` documentation.'); + } + + // Find the appropriate headers + const optionsStart = components.findIndex( + ({ slug }) => slug === config.cliOptionsHeaderSlug + ); + + const environmentStart = components.findIndex( + ({ slug }) => slug === config.envVarsHeaderSlug + ); + + // The first header that is <3 in depth after environmentStart + const environmentEnd = components.findIndex( + ({ heading }, index) => heading.depth < 3 && index > environmentStart + ); + + const output = { + // Extract the CLI options. + options: extractMandoc( + components, + optionsStart + 1, + environmentStart, + convertOptionToMandoc + ), + // Extract the environment variables. + env: extractMandoc( + components, + environmentStart + 1, + environmentEnd, + convertEnvVarToMandoc + ), + }; + + const template = await readFile(config.templatePath, 'utf-8'); + + const filledTemplate = template + .replace('__OPTIONS__', output.options) + .replace('__ENVIRONMENT__', output.env); + + if (config.output) { + await writeFile(join(config.output, config.fileName), filledTemplate); + } + + return filledTemplate; +} diff --git a/src/generators/man-page/index.mjs b/src/generators/man-page/index.mjs index 177eaa57..56de3c7e 100644 --- a/src/generators/man-page/index.mjs +++ b/src/generators/man-page/index.mjs @@ -1,14 +1,7 @@ 'use strict'; -import { readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { - convertOptionToMandoc, - convertEnvVarToMandoc, -} from './utils/converter.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; - /** * This generator generates a man page version of the CLI.md file. * See https://man.openbsd.org/mdoc.7 for the formatting. @@ -30,76 +23,4 @@ export default { envVarsHeaderSlug: 'environment-variables-1', templatePath: join(import.meta.dirname, 'template.1'), }, - - /** - * Generates the Node.js man-page - */ - async generate(input) { - const config = getConfig('man-page'); - - // Filter to only 'cli'. - const components = input.filter(({ api }) => api === 'cli'); - - if (!components.length) { - throw new Error('Could not find any `cli` documentation.'); - } - - // Find the appropriate headers - const optionsStart = components.findIndex( - ({ slug }) => slug === config.cliOptionsHeaderSlug - ); - - const environmentStart = components.findIndex( - ({ slug }) => slug === config.envVarsHeaderSlug - ); - - // The first header that is <3 in depth after environmentStart - const environmentEnd = components.findIndex( - ({ heading }, index) => heading.depth < 3 && index > environmentStart - ); - - const output = { - // Extract the CLI options. - options: extractMandoc( - components, - optionsStart + 1, - environmentStart, - convertOptionToMandoc - ), - // Extract the environment variables. - env: extractMandoc( - components, - environmentStart + 1, - environmentEnd, - convertEnvVarToMandoc - ), - }; - - const template = await readFile(config.templatePath, 'utf-8'); - - const filledTemplate = template - .replace('__OPTIONS__', output.options) - .replace('__ENVIRONMENT__', output.env); - - if (config.output) { - await writeFile(join(config.output, config.fileName), filledTemplate); - } - - return filledTemplate; - }, }; - -/** - * @param {Array} components - * @param {number} start - * @param {number} end - * @param {(element: ApiDocMetadataEntry) => string} convert - * @returns {string} - */ -function extractMandoc(components, start, end, convert) { - return components - .slice(start, end) - .filter(({ heading }) => heading.depth === 3) - .map(convert) - .join(''); -} diff --git a/src/generators/man-page/types.d.ts b/src/generators/man-page/types.d.ts index a6e26f29..f4193773 100644 --- a/src/generators/man-page/types.d.ts +++ b/src/generators/man-page/types.d.ts @@ -1,9 +1,10 @@ -export type Generator = GeneratorMetadata< - { - fileName: string; - cliOptionsHeaderSlug: string; - envVarsHeaderSlug: string; - templatePath: string; - }, +export type Generator = GeneratorMetadata<{ + fileName: string; + cliOptionsHeaderSlug: string; + envVarsHeaderSlug: string; + templatePath: string; +}>; + +export type Implementation = GeneratorImpl< Generate, Promise> >; diff --git a/src/generators/metadata/generate.mjs b/src/generators/metadata/generate.mjs new file mode 100644 index 00000000..7b943793 --- /dev/null +++ b/src/generators/metadata/generate.mjs @@ -0,0 +1,38 @@ +'use strict'; + +import { parseApiDoc } from './utils/parse.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { importFromURL } from '../../utils/url.mjs'; + +/** + * Process a chunk of API doc files in a worker thread. + * Called by chunk-worker.mjs for parallel processing. + * + * @type {import('./types').Implementation['processChunk']} + */ +export async function processChunk(fullInput, itemIndices, typeMap) { + const results = []; + + for (const idx of itemIndices) { + results.push(...parseApiDoc(fullInput[idx], typeMap)); + } + + return results; +} + +/** + * Generates a flattened list of metadata entries from API docs. + * + * @type {import('./types').Implementation['generate']} + */ +export async function* generate(inputs, worker) { + const { metadata: config } = getConfig(); + + const typeMap = await importFromURL(config.typeMap); + + // Stream chunks as they complete - allows dependent generators + // to start collecting/preparing while we're still processing + for await (const chunkResult of worker.stream(inputs, inputs, typeMap)) { + yield chunkResult.flat(); + } +} diff --git a/src/generators/metadata/index.mjs b/src/generators/metadata/index.mjs index c1dfdec9..acffb2c2 100644 --- a/src/generators/metadata/index.mjs +++ b/src/generators/metadata/index.mjs @@ -1,9 +1,5 @@ 'use strict'; -import { parseApiDoc } from './utils/parse.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { importFromURL } from '../../utils/url.mjs'; - /** * This generator generates a flattened list of metadata entries from a API doc * @@ -21,32 +17,4 @@ export default { defaultConfiguration: { typeMap: import.meta.resolve('./typeMap.json'), }, - - /** - * Process a chunk of API doc files in a worker thread. - * Called by chunk-worker.mjs for parallel processing. - */ - async processChunk(fullInput, itemIndices, typeMap) { - const results = []; - - for (const idx of itemIndices) { - results.push(...parseApiDoc(fullInput[idx], typeMap)); - } - - return results; - }, - - /** - */ - async *generate(inputs, worker) { - const { metadata: config } = getConfig(); - - const typeMap = await importFromURL(config.typeMap); - - // Stream chunks as they complete - allows dependent generators - // to start collecting/preparing while we're still processing - for await (const chunkResult of worker.stream(inputs, inputs, typeMap)) { - yield chunkResult.flat(); - } - }, }; diff --git a/src/generators/metadata/types.d.ts b/src/generators/metadata/types.d.ts index 403170db..2c0919c0 100644 --- a/src/generators/metadata/types.d.ts +++ b/src/generators/metadata/types.d.ts @@ -1,9 +1,10 @@ import type { Root } from 'mdast'; -export type Generator = GeneratorMetadata< - { - typeMap: string | URL; - }, +export type Generator = GeneratorMetadata<{ + typeMap: string | URL; +}>; + +export type Implementation = GeneratorImpl< Generate, AsyncGenerator>, ProcessChunk> >; diff --git a/src/generators/orama-db/generate.mjs b/src/generators/orama-db/generate.mjs new file mode 100644 index 00000000..6149515e --- /dev/null +++ b/src/generators/orama-db/generate.mjs @@ -0,0 +1,59 @@ +'use strict'; + +import { writeFile } from 'node:fs/promises'; + +import { create, save, insertMultiple } from '@orama/orama'; + +import { SCHEMA } from './constants.mjs'; +import { buildHierarchicalTitle } from './utils/title.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { groupNodesByModule } from '../../utils/generators.mjs'; +import { transformNodeToString } from '../../utils/unist.mjs'; + +/** + * Generates the Orama database. + * + * @type {import('./types').Implementation['generate']} + */ +export async function generate(input) { + const config = getConfig('orama-db'); + + const db = create({ schema: SCHEMA }); + + const apiGroups = groupNodesByModule(input); + + // Process all API groups and flatten into a single document array + const documents = Array.from(apiGroups.values()).flatMap(headings => + headings.map((entry, index) => { + const hierarchicalTitle = buildHierarchicalTitle(headings, index); + + const paragraph = entry.content.children.find( + child => child.type === 'paragraph' + ); + + return { + title: hierarchicalTitle, + description: paragraph + ? transformNodeToString(paragraph, true) + : undefined, + href: `${entry.api}.html#${entry.slug}`, + siteSection: headings[0].heading.data.name, + }; + }) + ); + + // Insert all documents + await insertMultiple(db, documents); + + const result = save(db); + + // Persist + if (config.output) { + await writeFile( + `${config.output}/orama-db.json`, + config.minify ? JSON.stringify(result) : JSON.stringify(result, null, 2) + ); + } + + return result; +} diff --git a/src/generators/orama-db/index.mjs b/src/generators/orama-db/index.mjs index 551fad27..70bb6695 100644 --- a/src/generators/orama-db/index.mjs +++ b/src/generators/orama-db/index.mjs @@ -1,15 +1,5 @@ 'use strict'; -import { writeFile } from 'node:fs/promises'; - -import { create, save, insertMultiple } from '@orama/orama'; - -import { SCHEMA } from './constants.mjs'; -import { buildHierarchicalTitle } from './utils/title.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { groupNodesByModule } from '../../utils/generators.mjs'; -import { transformNodeToString } from '../../utils/unist.mjs'; - /** * This generator is responsible for generating the Orama database for the * API docs. It is based on the legacy-json generator. @@ -24,50 +14,4 @@ export default { description: 'Generates the Orama database for the API docs.', dependsOn: 'metadata', - - /** - * Generates the Orama database. - */ - async generate(input) { - const config = getConfig('orama-db'); - - const db = create({ schema: SCHEMA }); - - const apiGroups = groupNodesByModule(input); - - // Process all API groups and flatten into a single document array - const documents = Array.from(apiGroups.values()).flatMap(headings => - headings.map((entry, index) => { - const hierarchicalTitle = buildHierarchicalTitle(headings, index); - - const paragraph = entry.content.children.find( - child => child.type === 'paragraph' - ); - - return { - title: hierarchicalTitle, - description: paragraph - ? transformNodeToString(paragraph, true) - : undefined, - href: `${entry.api}.html#${entry.slug}`, - siteSection: headings[0].heading.data.name, - }; - }) - ); - - // Insert all documents - await insertMultiple(db, documents); - - const result = save(db); - - // Persist - if (config.output) { - await writeFile( - `${config.output}/orama-db.json`, - config.minify ? JSON.stringify(result) : JSON.stringify(result, null, 2) - ); - } - - return result; - }, }; diff --git a/src/generators/orama-db/types.d.ts b/src/generators/orama-db/types.d.ts index 6fe89926..b5f285e4 100644 --- a/src/generators/orama-db/types.d.ts +++ b/src/generators/orama-db/types.d.ts @@ -23,7 +23,8 @@ export interface OramaDbEntry { */ export type OramaDb = Orama; -export type Generator = GeneratorMetadata< - {}, +export type Generator = GeneratorMetadata<{}>; + +export type Implementation = GeneratorImpl< Generate, Promise> >; diff --git a/src/generators/sitemap/generate.mjs b/src/generators/sitemap/generate.mjs new file mode 100644 index 00000000..c163067c --- /dev/null +++ b/src/generators/sitemap/generate.mjs @@ -0,0 +1,64 @@ +'use strict'; + +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { createPageSitemapEntry } from './utils/createPageSitemapEntry.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; + +/** + * Generates a sitemap.xml file + * + * @type {import('./types').Implementation['generate']} + */ +export async function generate(entries) { + const { sitemap: config } = getConfig(); + + const template = await readFile( + join(import.meta.dirname, 'template.xml'), + 'utf-8' + ); + + const entryTemplate = await readFile( + join(import.meta.dirname, 'entry-template.xml'), + 'utf-8' + ); + + const lastmod = new Date().toISOString().split('T')[0]; + + const apiPages = entries + .filter(entry => entry.heading.depth === 1) + .map(entry => createPageSitemapEntry(entry, config.baseURL, lastmod)); + + const { href: loc } = new URL('latest/api/', config.baseURL); + + /** + * @typedef {import('./types').SitemapEntry} + */ + const mainPage = { + loc, + lastmod, + changefreq: 'daily', + priority: '1.0', + }; + + apiPages.push(mainPage); + + const urlset = apiPages + .map(page => + entryTemplate + .replace('__LOC__', page.loc) + .replace('__LASTMOD__', page.lastmod) + .replace('__CHANGEFREQ__', page.changefreq) + .replace('__PRIORITY__', page.priority) + ) + .join(''); + + const sitemap = template.replace('__URLSET__', urlset); + + if (config.output) { + await writeFile(join(config.output, 'sitemap.xml'), sitemap, 'utf-8'); + } + + return sitemap; +} diff --git a/src/generators/sitemap/index.mjs b/src/generators/sitemap/index.mjs index e58e6be9..e3aa4335 100644 --- a/src/generators/sitemap/index.mjs +++ b/src/generators/sitemap/index.mjs @@ -1,8 +1,4 @@ -import { readFile, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { createPageSitemapEntry } from './utils/createPageSitemapEntry.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; +'use strict'; /** * This generator generates a sitemap.xml file for search engine optimization @@ -17,59 +13,4 @@ export default { description: 'Generates a sitemap.xml file for search engine optimization', dependsOn: 'metadata', - - /** - * Generates a sitemap.xml file - */ - async generate(entries) { - const { sitemap: config } = getConfig(); - - const template = await readFile( - join(import.meta.dirname, 'template.xml'), - 'utf-8' - ); - - const entryTemplate = await readFile( - join(import.meta.dirname, 'entry-template.xml'), - 'utf-8' - ); - - const lastmod = new Date().toISOString().split('T')[0]; - - const apiPages = entries - .filter(entry => entry.heading.depth === 1) - .map(entry => createPageSitemapEntry(entry, config.baseURL, lastmod)); - - const { href: loc } = new URL('latest/api/', config.baseURL); - - /** - * @typedef {import('./types').SitemapEntry} - */ - const mainPage = { - loc, - lastmod, - changefreq: 'daily', - priority: '1.0', - }; - - apiPages.push(mainPage); - - const urlset = apiPages - .map(page => - entryTemplate - .replace('__LOC__', page.loc) - .replace('__LASTMOD__', page.lastmod) - .replace('__CHANGEFREQ__', page.changefreq) - .replace('__PRIORITY__', page.priority) - ) - .join(''); - - const sitemap = template.replace('__URLSET__', urlset); - - if (config.output) { - await writeFile(join(config.output, 'sitemap.xml'), sitemap, 'utf-8'); - } - - return sitemap; - }, }; diff --git a/src/generators/sitemap/types.d.ts b/src/generators/sitemap/types.d.ts index d9cb9c2f..ba849797 100644 --- a/src/generators/sitemap/types.d.ts +++ b/src/generators/sitemap/types.d.ts @@ -12,7 +12,8 @@ export interface SitemapEntry { priority?: string; } -export type Generator = GeneratorMetadata< - {}, +export type Generator = GeneratorMetadata<{}>; + +export type Implementation = GeneratorImpl< Generate, Promise> >; diff --git a/src/generators/types.d.ts b/src/generators/types.d.ts index bc7f24a8..af93204d 100644 --- a/src/generators/types.d.ts +++ b/src/generators/types.d.ts @@ -1,22 +1,14 @@ import type { publicGenerators, allGenerators } from './index.mjs'; declare global { - /** - * A lazy generator loader that returns a promise resolving to the generator metadata. - */ - export type LazyGenerator> = - () => Promise; - // Public generators exposed to the CLI export type AvailableGenerators = typeof publicGenerators; // All generators including internal ones (metadata, jsx-ast, ast-js) export type AllGenerators = typeof allGenerators; - // The resolved type of a loaded generator - export type ResolvedGenerator = Awaited< - ReturnType - >; + // The resolved type of a generator (now synchronous, not lazy) + export type ResolvedGenerator = AllGenerators[K]; /** * ParallelWorker interface for distributing work across Node.js worker threads. @@ -68,11 +60,11 @@ declare global { dependencies: D ) => Promise; - export type GeneratorMetadata< - C extends any, - G extends Generate, - P extends ProcessChunk | undefined = undefined, - > = { + /** + * Generator metadata - loaded synchronously from each generator's index.mjs. + * Contains only descriptive metadata and default configuration. + */ + export type GeneratorMetadata = { readonly defaultConfiguration: C; // The name of the Generator. Must match the Key in AllGenerators @@ -105,7 +97,16 @@ declare global { * passes the ASTs for any JavaScript files given in the input. */ dependsOn: keyof AllGenerators | undefined; + }; + /** + * Generator implementation - loaded dynamically from each generator's generate.mjs. + * Contains the generate function and optional processChunk for parallel processing. + */ + export type GeneratorImpl< + G extends Generate, + P extends ProcessChunk | undefined = undefined, + > = { /** * Generators are abstract and the different generators have different sort of inputs and outputs. * For example, a MDX generator would take the raw AST and output MDX with React Components; diff --git a/src/generators/web/generate.mjs b/src/generators/web/generate.mjs new file mode 100644 index 00000000..0c43914c --- /dev/null +++ b/src/generators/web/generate.mjs @@ -0,0 +1,53 @@ +'use strict'; + +import { readFile, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { join } from 'node:path'; + +import createASTBuilder from './utils/generate.mjs'; +import { processJSXEntries } from './utils/processing.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; + +/** + * Main generation function that processes JSX AST entries into web bundles. + * + * @type {import('./types').Implementation['generate']} + */ +export async function generate(input) { + const config = getConfig('web'); + + const template = await readFile(config.templatePath, 'utf-8'); + + // Create AST builders for server and client programs + const astBuilders = createASTBuilder(); + + // Create require function for resolving external packages in server code + const requireFn = createRequire(import.meta.url); + + // Process all entries: convert JSX to HTML/CSS/JS + const { results, css, chunks } = await processJSXEntries( + input, + template, + astBuilders, + requireFn, + config + ); + + // Process all entries together (required for code-split bundles) + if (config.output) { + // Write HTML files + for (const { html, api } of results) { + await writeFile(join(config.output, `${api}.html`), html, 'utf-8'); + } + + // Write code-split JavaScript chunks + for (const chunk of chunks) { + await writeFile(join(config.output, chunk.fileName), chunk.code, 'utf-8'); + } + + // Write CSS bundle + await writeFile(join(config.output, 'styles.css'), css, 'utf-8'); + } + + return results.map(({ html }) => ({ html: html.toString(), css })); +} diff --git a/src/generators/web/index.mjs b/src/generators/web/index.mjs index 70ebd7dc..9860c7d2 100644 --- a/src/generators/web/index.mjs +++ b/src/generators/web/index.mjs @@ -1,10 +1,6 @@ -import { readFile, writeFile } from 'node:fs/promises'; -import { createRequire } from 'node:module'; -import { join } from 'node:path'; +'use strict'; -import createASTBuilder from './utils/generate.mjs'; -import { processJSXEntries } from './utils/processing.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; +import { join } from 'node:path'; /** * Web generator - transforms JSX AST entries into complete web bundles. @@ -35,50 +31,4 @@ export default { '#config/Logo': '@node-core/ui-components/Common/NodejsLogo', }, }, - - /** - * Main generation function that processes JSX AST entries into web bundles. - */ - async generate(input) { - const config = getConfig('web'); - - const template = await readFile(config.templatePath, 'utf-8'); - - // Create AST builders for server and client programs - const astBuilders = createASTBuilder(); - - // Create require function for resolving external packages in server code - const requireFn = createRequire(import.meta.url); - - // Process all entries: convert JSX to HTML/CSS/JS - const { results, css, chunks } = await processJSXEntries( - input, - template, - astBuilders, - requireFn, - config - ); - - // Process all entries together (required for code-split bundles) - if (config.output) { - // Write HTML files - for (const { html, api } of results) { - await writeFile(join(config.output, `${api}.html`), html, 'utf-8'); - } - - // Write code-split JavaScript chunks - for (const chunk of chunks) { - await writeFile( - join(config.output, chunk.fileName), - chunk.code, - 'utf-8' - ); - } - - // Write CSS bundle - await writeFile(join(config.output, 'styles.css'), css, 'utf-8'); - } - - return results.map(({ html }) => ({ html: html.toString(), css })); - }, }; diff --git a/src/generators/web/types.d.ts b/src/generators/web/types.d.ts index 5feed60e..e0669944 100644 --- a/src/generators/web/types.d.ts +++ b/src/generators/web/types.d.ts @@ -1,10 +1,11 @@ import type { JSXContent } from '../jsx-ast/utils/buildContent.mjs'; -export type Generator = GeneratorMetadata< - { - templatePath: string; - title: string; - imports: Record; - }, +export type Generator = GeneratorMetadata<{ + templatePath: string; + title: string; + imports: Record; +}>; + +export type Implementation = GeneratorImpl< Generate, AsyncGenerator<{ html: string; css: string }>> >; diff --git a/src/generators/web/utils/bundle.mjs b/src/generators/web/utils/bundle.mjs index 69780372..874cb6a2 100644 --- a/src/generators/web/utils/bundle.mjs +++ b/src/generators/web/utils/bundle.mjs @@ -14,6 +14,14 @@ const DOC_KIT_NODE_MODULES = join( '../../../../node_modules' ); +// Modules expected to be provided at runtime in the server environment, and thus excluded from the bundle. +const SERVER_MODULES = [ + 'preact', + 'preact-render-to-string', + '@node-core/ui-components', + '@node-core/rehype-shiki', +]; + /** * Asynchronously bundles JavaScript source code (and its CSS imports), * targeting either browser (client) or server (Node.js) environments. @@ -32,6 +40,8 @@ export default async function bundleCode(codeMap, { server = false } = {}) { // Experimental features: import maps for client, none for server experimental: { chunkImportMap: !server, + + lazyBarrel: true, }, checks: { @@ -59,9 +69,7 @@ export default async function bundleCode(codeMap, { server = false } = {}) { // External dependencies to exclude from bundling. // These are expected to be available at runtime in the server environment. // This reduces bundle size and avoids bundling shared server libs. - external: server - ? ['preact', 'preact-render-to-string', '@node-core/ui-components'] - : [], + external: server ? SERVER_MODULES : [], transform: { // Inject global compile-time constants that will be replaced in code. diff --git a/src/threading/chunk-worker.mjs b/src/threading/chunk-worker.mjs index 358ba8a1..8e0ddf8f 100644 --- a/src/threading/chunk-worker.mjs +++ b/src/threading/chunk-worker.mjs @@ -1,4 +1,3 @@ -import { allGenerators } from '../generators/index.mjs'; import { setConfig } from '../utils/configuration/index.mjs'; /** @@ -17,7 +16,9 @@ export default async ({ }) => { await setConfig(configuration); - const generator = await allGenerators[generatorName](); + const { processChunk } = await import( + `../generators/${generatorName}/generate.mjs` + ); - return generator.processChunk(input, itemIndices, extra); + return processChunk(input, itemIndices, extra); }; diff --git a/src/threading/parallel.mjs b/src/threading/parallel.mjs index a362c10c..5b083a05 100644 --- a/src/threading/parallel.mjs +++ b/src/threading/parallel.mjs @@ -1,6 +1,5 @@ 'use strict'; -import { allGenerators } from '../generators/index.mjs'; import logger from '../logger/index.mjs'; const parallelLogger = logger.child('parallel'); @@ -70,7 +69,9 @@ export default async function createParallelWorker( ) { const { threads, chunkSize } = configuration; - const generator = await allGenerators[generatorName](); + const { processChunk } = await import( + `../generators/${generatorName}/generate.mjs` + ); return { /** @@ -100,9 +101,9 @@ export default async function createParallelWorker( const pending = new Set( chunks.map(indices => { if (runInOneGo) { - const promise = generator - .processChunk(fullInput, indices, extra) - .then(result => ({ promise, result })); + const promise = processChunk(fullInput, indices, extra).then( + result => ({ promise, result }) + ); return promise; } diff --git a/src/utils/configuration/__tests__/index.test.mjs b/src/utils/configuration/__tests__/index.test.mjs index 229c5677..293a468f 100644 --- a/src/utils/configuration/__tests__/index.test.mjs +++ b/src/utils/configuration/__tests__/index.test.mjs @@ -15,9 +15,9 @@ const createMockConfig = (overrides = {}) => ({ mock.module('../../../generators/index.mjs', { namedExports: { allGenerators: { - json: async () => ({ defaultConfiguration: { format: 'json' } }), - html: async () => ({ defaultConfiguration: { format: 'html' } }), - markdown: async () => ({}), + json: { defaultConfiguration: { format: 'json' } }, + html: { defaultConfiguration: { format: 'html' } }, + markdown: {}, }, }, }); diff --git a/src/utils/configuration/index.mjs b/src/utils/configuration/index.mjs index 7161b93a..3a465ee2 100644 --- a/src/utils/configuration/index.mjs +++ b/src/utils/configuration/index.mjs @@ -32,12 +32,9 @@ export const getDefaultConfig = lazy(async () => { chunkSize: 10, }); - await Promise.all( - Object.keys(allGenerators).map(async k => { - const generator = await allGenerators[k](); - defaults[k] = generator.defaultConfiguration ?? {}; - }) - ); + for (const k of Object.keys(allGenerators)) { + defaults[k] = allGenerators[k].defaultConfiguration ?? {}; + } return defaults; }); diff --git a/src/utils/configuration/templates.mjs b/src/utils/configuration/templates.mjs index 802e227b..778c610f 100644 --- a/src/utils/configuration/templates.mjs +++ b/src/utils/configuration/templates.mjs @@ -3,11 +3,11 @@ export const CHANGELOG_URL = 'https://raw.githubusercontent.com/{repository}/{ref}/CHANGELOG.md'; // This is the Base URL for viewing a file within GitHub UI -export const GITHUB_BLOB_URL = 'https://github.com/{repository}/blob/{ref}/'; +export const GH_BLOB_URL = 'https://github.com/{repository}/blob/{ref}/'; // This is the API docs base URL for editing a file on GitHub UI // TODO(@avivkeller): specify /doc/api in config -export const GITHUB_EDIT_URL = +export const GH_EDIT_URL = 'https://github.com/{repository}/edit/{ref}/doc/api/'; /** From 5722d1a7d6daad536aaa4b4e08b121203d236148 Mon Sep 17 00:00:00 2001 From: Aviv Keller Date: Fri, 20 Feb 2026 15:47:16 -0500 Subject: [PATCH 2/2] fixup! --- docs/generators.md | 401 +++++++++++------- src/generators.mjs | 12 +- src/generators/addon-verify/generate.mjs | 2 +- src/generators/addon-verify/index.mjs | 6 +- src/generators/addon-verify/types.d.ts | 5 +- src/generators/api-links/generate.mjs | 9 +- src/generators/api-links/index.mjs | 6 +- src/generators/api-links/types.d.ts | 5 +- src/generators/ast-js/generate.mjs | 4 +- src/generators/ast-js/index.mjs | 8 +- src/generators/ast-js/types.d.ts | 5 +- src/generators/ast/generate.mjs | 4 +- src/generators/ast/index.mjs | 8 +- src/generators/ast/types.d.ts | 5 +- src/generators/json-simple/generate.mjs | 2 +- src/generators/json-simple/index.mjs | 6 +- src/generators/json-simple/types.d.ts | 5 +- src/generators/jsx-ast/generate.mjs | 4 +- src/generators/jsx-ast/index.mjs | 8 +- src/generators/jsx-ast/types.d.ts | 5 +- .../jsx-ast/utils/buildBarProps.mjs | 4 +- src/generators/jsx-ast/utils/buildContent.mjs | 4 +- src/generators/legacy-html-all/generate.mjs | 2 +- src/generators/legacy-html-all/index.mjs | 5 +- src/generators/legacy-html-all/types.d.ts | 9 +- src/generators/legacy-html/generate.mjs | 4 +- src/generators/legacy-html/index.mjs | 8 +- src/generators/legacy-html/types.d.ts | 11 +- .../legacy-html/utils/buildContent.mjs | 4 +- .../utils/replaceTemplateValues.mjs | 6 +- src/generators/legacy-json-all/generate.mjs | 2 +- src/generators/legacy-json-all/index.mjs | 6 +- src/generators/legacy-json-all/types.d.ts | 5 +- src/generators/legacy-json/generate.mjs | 4 +- src/generators/legacy-json/index.mjs | 8 +- src/generators/legacy-json/types.d.ts | 5 +- src/generators/llms-txt/generate.mjs | 2 +- src/generators/llms-txt/index.mjs | 6 +- src/generators/llms-txt/types.d.ts | 9 +- src/generators/man-page/generate.mjs | 2 +- src/generators/man-page/index.mjs | 6 +- src/generators/man-page/types.d.ts | 15 +- src/generators/metadata/generate.mjs | 4 +- src/generators/metadata/index.mjs | 8 +- src/generators/metadata/types.d.ts | 9 +- src/generators/orama-db/generate.mjs | 2 +- src/generators/orama-db/index.mjs | 6 +- src/generators/orama-db/types.d.ts | 5 +- src/generators/sitemap/generate.mjs | 2 +- src/generators/sitemap/index.mjs | 6 +- src/generators/sitemap/types.d.ts | 5 +- src/generators/types.d.ts | 24 +- src/generators/web/generate.mjs | 2 +- src/generators/web/index.mjs | 6 +- src/generators/web/types.d.ts | 13 +- src/generators/web/utils/bundle.mjs | 14 +- src/threading/__tests__/parallel.test.mjs | 12 +- src/threading/chunk-worker.mjs | 7 +- src/threading/parallel.mjs | 13 +- src/utils/configuration/index.mjs | 46 +- src/utils/configuration/templates.mjs | 4 +- src/utils/configuration/types.d.ts | 2 +- src/utils/generators.mjs | 29 ++ 63 files changed, 499 insertions(+), 357 deletions(-) diff --git a/docs/generators.md b/docs/generators.md index cd44cf57..86d1d37b 100644 --- a/docs/generators.md +++ b/docs/generators.md @@ -32,13 +32,14 @@ A generator is defined as a module exporting an object conforming to the `Genera ## Creating a Basic Generator -### Step 1: Create the Generator File +### Step 1: Create the Generator Files Create a new directory in `src/generators/`: ``` src/generators/my-format/ -├── index.mjs # Main generator file (required) +├── index.mjs # Generator metadata (required) +├── generate.mjs # Generator implementation (required) ├── constants.mjs # Constants (optional) ├── types.d.ts # TypeScript types (required) └── utils/ # Utility functions (optional) @@ -66,22 +67,20 @@ export type Generator = GeneratorMetadata< >; ``` -### Step 3: Implement the Generator +### Step 3: Define Generator Metadata + +Create the generator metadata in `index.mjs` using `createLazyGenerator`: ```javascript // src/generators/my-format/index.mjs -import { writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; +import { createLazyGenerator } from '../../utils/generators.mjs'; /** * Generates output in MyFormat. * - * @typedef {Array} Input - * @typedef {string} Output - * - * @type {GeneratorMetadata} + * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'my-format', version: '1.0.0', @@ -93,36 +92,52 @@ export default { defaultConfiguration: { // If your generator supports a custom configuration, define the defaults here - myCustomOption: 'myDefaultValue' + myCustomOption: 'myDefaultValue', // All generators support options in the GlobalConfiguration object // To override the defaults, they can be specified here - ref: 'overriddenRef' - } + ref: 'overriddenRef', + }, +}); +``` - /** - * Main generation function - * - * @param {Input} input - Metadata entries from previous generator - * @param {Partial} options - Configuration - * @returns {Promise} - */ - async generate(input, { output, version }) { - // Transform input to your format - const result = transformToMyFormat(input, version); - - // Write to file if output directory specified - if (output) { - await writeFile(join(output, 'documentation.myformat'), result, 'utf-8'); - } +### Step 4: Implement the Generator Logic - return result; - }, -}; +Create the generator implementation in `generate.mjs`: + +```javascript +// src/generators/my-format/generate.mjs +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import getConfig from '../../utils/configuration/index.mjs'; + +/** + * Main generation function + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input, worker) { + const config = getConfig('my-format'); + + // Transform input to your format + const result = transformToMyFormat(input, config.version); + + // Write to file if output directory specified + if (config.output) { + await writeFile( + join(config.output, 'documentation.myformat'), + result, + 'utf-8' + ); + } + + return result; +} /** * Transform metadata entries to MyFormat - * @param {Input} entries + * @param {Array} entries * @param {import('semver').SemVer} version * @returns {string} */ @@ -134,7 +149,7 @@ function transformToMyFormat(entries, version) { } ``` -### Step 4: Register the Generator +### Step 5: Register the Generator Add your generator to the exports in `src/generators/index.mjs`: @@ -162,58 +177,75 @@ For generators processing large datasets, implement parallel processing using wo ### Implementing Worker-Based Processing +First, define the generator metadata in `index.mjs`: + ```javascript -export default { +// src/generators/parallel-generator/index.mjs +import { createLazyGenerator } from '../../utils/generators.mjs'; + +/** + * @type {import('./types').Generator} + */ +export default createLazyGenerator({ name: 'parallel-generator', + version: '1.0.0', + description: 'Processes data in parallel', + dependsOn: 'metadata', - /** - * Process a chunk of items in a worker thread. - * This function runs in isolated worker threads. - * - * @param {Array} fullInput - Complete input array - * @param {number[]} itemIndices - Indices of items to process - * @param {Object} deps - Serializable dependencies - * @returns {Promise>} - */ - async processChunk(fullInput, itemIndices, deps) { - const results = []; - - // Process only the items at specified indices - for (const idx of itemIndices) { - const item = fullInput[idx]; - const result = await processItem(item, deps); - results.push(result); - } - - return results; - }, + // Indicates this generator has a processChunk implementation + hasParallelProcessor: true, +}); +``` - /** - * Main generation function that orchestrates worker threads - * - * @param {Input} input - * @param {ParallelWorker} options - */ - async *generate(input, worker) { - // Configuration for this generator is based on it's name - const config = getConfig('parallel-generator'); - - // Prepare serializable dependencies - const deps = { - version: config.version, - ...someConfig, - }; - - // Stream chunks as they complete - for await (const chunkResult of worker.stream(input, input, deps)) { - // Process chunk result if needed - yield chunkResult; - } - }, -}; +Then, implement both `processChunk` and `generate` in `generate.mjs`: + +```javascript +// src/generators/parallel-generator/generate.mjs +import getConfig from '../../utils/configuration/index.mjs'; + +/** + * Process a chunk of items in a worker thread. + * This function runs in isolated worker threads. + * + * @type {import('./types').Generator['processChunk']} + */ +export async function processChunk(fullInput, itemIndices, deps) { + const results = []; + + // Process only the items at specified indices + for (const idx of itemIndices) { + const item = fullInput[idx]; + const result = await processItem(item, deps); + results.push(result); + } + + return results; +} + +/** + * Main generation function that orchestrates worker threads + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input, worker) { + // Configuration for this generator is based on its name + const config = getConfig('my-format'); + + // Prepare serializable dependencies + const deps = { + version: config.version, + // ...other config + }; + + // Stream chunks as they complete + for await (const chunkResult of worker.stream(input, input, deps)) { + // Process chunk result if needed + yield chunkResult; + } +} ``` ### Key Points for Worker Processing @@ -239,31 +271,56 @@ Don't use workers when: ## Streaming Results -Generators can yield results as they're produced using async generators: +Generators can yield results as they're produced using async generators. + +Define the generator metadata in `index.mjs`: ```javascript -export default { +// src/generators/streaming-generator/index.mjs +import { createLazyGenerator } from '../../utils/generators.mjs'; + +/** + * @type {import('./types').Generator} + */ +export default createLazyGenerator({ name: 'streaming-generator', + version: '1.0.0', + description: 'Streams results as they are ready', + dependsOn: 'metadata', - async processChunk(fullInput, itemIndices, deps) { - // Process chunk - return results; - }, + hasParallelProcessor: true, +}); +``` - /** - * Generator function that yields results incrementally - */ - async *generate(input, worker) { - // Stream results as workers complete chunks - for await (const chunkResult of worker.stream(input, input, {})) { - // Yield immediately - downstream can start processing - yield chunkResult; - } - }, -}; +Implement the generator in `generate.mjs`: + +```javascript +// src/generators/streaming-generator/generate.mjs +/** + * Process a chunk of data + * + * @type {import('./types').Generator['processChunk']} + */ +export async function processChunk(fullInput, itemIndices, deps) { + // Process chunk + return results; +} + +/** + * Generator function that yields results incrementally + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input, worker) { + // Stream results as workers complete chunks + for await (const chunkResult of worker.stream(input, input, {})) { + // Yield immediately - downstream can start processing + yield chunkResult; + } +} ``` ### Benefits of Streaming @@ -274,28 +331,46 @@ export default { ### Non-Streaming Generators -Some generators must collect all input before processing: +Some generators must collect all input before processing. + +Generator metadata in `index.mjs`: ```javascript -export default { +// src/generators/batch-generator/index.mjs +import { createLazyGenerator } from '../../utils/generators.mjs'; + +/** + * @type {import('./types').Generator} + */ +export default createLazyGenerator({ name: 'batch-generator', + version: '1.0.0', + description: 'Requires all input at once', + dependsOn: 'jsx-ast', +}); +``` - /** - * Non-streaming - returns Promise instead of AsyncGenerator - */ - async generate(input, worker) { - // Collect all input (if dependency is streaming, this waits for completion) - const allData = await collectAll(input); +Implementation in `generate.mjs`: - // Process everything together - const result = processBatch(allData); +```javascript +// src/generators/batch-generator/generate.mjs +/** + * Non-streaming - returns Promise instead of AsyncGenerator + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input, worker) { + // Collect all input (if dependency is streaming, this waits for completion) + const allData = await collectAll(input); - return result; - }, -}; + // Process everything together + const result = processBatch(allData); + + return result; +} ``` Use non-streaming when: @@ -308,40 +383,54 @@ Use non-streaming when: ### Declaring Dependencies +In `index.mjs`: + ```javascript -export default { +import { createLazyGenerator } from '../../utils/generators.mjs'; + +export default createLazyGenerator({ name: 'my-generator', + dependsOn: 'metadata', // This generator requires metadata output - async generate(input, worker) { - // input contains the output from 'metadata' generator - }, -}; + // ... other metadata +}); +``` + +In `generate.mjs`: + +```javascript +export async function generate(input, worker) { + // input contains the output from 'metadata' generator +} ``` ### Dependency Chain Example ```javascript // Step 1: Parse markdown to AST -export default { +// src/generators/ast/index.mjs +export default createLazyGenerator({ name: 'ast', dependsOn: undefined, // No dependency // Processes raw markdown files -}; +}); // Step 2: Extract metadata from AST -export default { +// src/generators/metadata/index.mjs +export default createLazyGenerator({ name: 'metadata', dependsOn: 'ast', // Depends on AST // Processes AST output -}; +}); // Step 3: Generate HTML from metadata -export default { +// src/generators/html-generator/index.mjs +export default createLazyGenerator({ name: 'html-generator', dependsOn: 'metadata', // Depends on metadata // Processes metadata output -}; +}); ``` ### Multiple Consumers @@ -360,47 +449,63 @@ The framework ensures `metadata` runs once and its output is cached for all cons ### Writing Output Files +In `generate.mjs`: + ```javascript -const config = getConfig('my-generator'); +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; -if (!config.output) { - // Return data without writing - return result; -} +import getConfig from '../../utils/configuration/index.mjs'; + +export async function generate(input, worker) { + const config = getConfig('my-format'); -// Ensure directory exists -await mkdir(config.output, { recursive: true }); + if (!config.output) { + // Return data without writing + return result; + } -// Write single file -await writeFile(join(config.output, 'output.txt'), content, 'utf-8'); + // Ensure directory exists + await mkdir(config.output, { recursive: true }); -// Write multiple files -for (const item of items) { - await writeFile( - join(config.output, `${item.name}.txt`), - item.content, - 'utf-8' - ); -} + // Write single file + await writeFile(join(config.output, 'output.txt'), content, 'utf-8'); + + // Write multiple files + for (const item of items) { + await writeFile( + join(config.output, `${item.name}.txt`), + item.content, + 'utf-8' + ); + } -return result; + return result; +} ``` ### Copying Assets ```javascript -const config = getConfig('my-generator'); - -if (config.output) { - // Copy asset directory - await cp( - new URL('./assets', import.meta.url), - join(config.output, 'assets'), - { recursive: true } - ); -} +import { cp } from 'node:fs/promises'; +import { join } from 'node:path'; + +import getConfig from '../../utils/configuration/index.mjs'; -return result; +export async function generate(input, worker) { + const config = getConfig('my-format'); + + if (config.output) { + // Copy asset directory + await cp( + new URL('./assets', import.meta.url), + join(config.output, 'assets'), + { recursive: true } + ); + } + + return result; +} ``` ### Output Structure diff --git a/src/generators.mjs b/src/generators.mjs index 81e90526..451c39b5 100644 --- a/src/generators.mjs +++ b/src/generators.mjs @@ -53,12 +53,8 @@ const createGenerator = () => { return; } - const { dependsOn } = allGenerators[generatorName]; - - // Lazy-load the generator implementation - const { generate, processChunk } = await import( - `./generators/${generatorName}/generate.mjs` - ); + const { dependsOn, generate, hasParallelProcessor } = + allGenerators[generatorName]; // Schedule dependency first if (dependsOn && !(dependsOn in cachedGenerators)) { @@ -67,7 +63,7 @@ const createGenerator = () => { generatorsLogger.debug(`Scheduling "${generatorName}"`, { dependsOn: dependsOn || 'none', - streaming: Boolean(processChunk), + streaming: hasParallelProcessor, }); // Schedule the generator @@ -77,7 +73,7 @@ const createGenerator = () => { generatorsLogger.debug(`Starting "${generatorName}"`); // Create parallel worker for streaming generators - const worker = processChunk + const worker = hasParallelProcessor ? createParallelWorker(generatorName, pool, configuration) : Promise.resolve(null); diff --git a/src/generators/addon-verify/generate.mjs b/src/generators/addon-verify/generate.mjs index 074c7211..10f45c95 100644 --- a/src/generators/addon-verify/generate.mjs +++ b/src/generators/addon-verify/generate.mjs @@ -17,7 +17,7 @@ import getConfig from '../../utils/configuration/index.mjs'; /** * Generates a file list from code blocks. * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function generate(input) { const config = getConfig('addon-verify'); diff --git a/src/generators/addon-verify/index.mjs b/src/generators/addon-verify/index.mjs index 6df1b576..fab78599 100644 --- a/src/generators/addon-verify/index.mjs +++ b/src/generators/addon-verify/index.mjs @@ -1,5 +1,7 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator generates a file list from code blocks extracted from * `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime @@ -7,7 +9,7 @@ * * @type {import('./types').Generator} */ -export default { +export default await createLazyGenerator({ name: 'addon-verify', version: '1.0.0', @@ -16,4 +18,4 @@ export default { 'Generates a file list from code blocks extracted from `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime validations', dependsOn: 'metadata', -}; +}); diff --git a/src/generators/addon-verify/types.d.ts b/src/generators/addon-verify/types.d.ts index 25af16cb..44520a3d 100644 --- a/src/generators/addon-verify/types.d.ts +++ b/src/generators/addon-verify/types.d.ts @@ -1,5 +1,4 @@ -export type Generator = GeneratorMetadata<{}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + {}, Generate, Promise>> >; diff --git a/src/generators/api-links/generate.mjs b/src/generators/api-links/generate.mjs index 5f27de3b..fca10a01 100644 --- a/src/generators/api-links/generate.mjs +++ b/src/generators/api-links/generate.mjs @@ -7,12 +7,15 @@ import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs'; import { extractExports } from './utils/extractExports.mjs'; import { findDefinitions } from './utils/findDefinitions.mjs'; import getConfig from '../../utils/configuration/index.mjs'; -import { GH_BLOB_URL, populate } from '../../utils/configuration/templates.mjs'; +import { + GITHUB_BLOB_URL, + populate, +} from '../../utils/configuration/templates.mjs'; /** * Generates the `apilinks.json` file. * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function generate(input) { const config = getConfig('api-links'); @@ -39,7 +42,7 @@ export async function generate(input) { checkIndirectReferences(program, exports, nameToLineNumberMap); - const fullGitUrl = `${populate(GH_BLOB_URL, config)}lib/${baseName}.js`; + const fullGitUrl = `${populate(GITHUB_BLOB_URL, config)}lib/${baseName}.js`; // Add the exports we found in this program to our output Object.keys(nameToLineNumberMap).forEach(key => { diff --git a/src/generators/api-links/index.mjs b/src/generators/api-links/index.mjs index 9252be95..0c159dda 100644 --- a/src/generators/api-links/index.mjs +++ b/src/generators/api-links/index.mjs @@ -1,5 +1,7 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator is responsible for mapping publicly accessible functions in * Node.js to their source locations in the Node.js repository. @@ -10,7 +12,7 @@ * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'api-links', version: '1.0.0', @@ -21,4 +23,4 @@ export default { // Unlike the rest of the generators, this utilizes Javascript sources being // passed into the input field rather than Markdown. dependsOn: 'ast-js', -}; +}); diff --git a/src/generators/api-links/types.d.ts b/src/generators/api-links/types.d.ts index e0d8124d..48a75b65 100644 --- a/src/generators/api-links/types.d.ts +++ b/src/generators/api-links/types.d.ts @@ -4,8 +4,7 @@ export interface ProgramExports { indirects: Record; } -export type Generator = GeneratorMetadata<{}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + {}, Generate>> >; diff --git a/src/generators/ast-js/generate.mjs b/src/generators/ast-js/generate.mjs index 25f688ca..4cbb9f68 100644 --- a/src/generators/ast-js/generate.mjs +++ b/src/generators/ast-js/generate.mjs @@ -12,7 +12,7 @@ import getConfig from '../../utils/configuration/index.mjs'; * Process a chunk of JavaScript files in a worker thread. * Parses JS source files into AST representations. * - * @type {import('./types').Implementation['processChunk']} + * @type {import('./types').Generator['processChunk']} */ export async function processChunk(inputSlice, itemIndices) { const filePaths = itemIndices.map(idx => inputSlice[idx]); @@ -39,7 +39,7 @@ export async function processChunk(inputSlice, itemIndices) { /** * Generates a JavaScript AST from the input files. * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function* generate(_, worker) { const config = getConfig('ast-js'); diff --git a/src/generators/ast-js/index.mjs b/src/generators/ast-js/index.mjs index 6a771ccc..06652167 100644 --- a/src/generators/ast-js/index.mjs +++ b/src/generators/ast-js/index.mjs @@ -1,5 +1,7 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator parses Javascript sources passed into the generator's input * field. This is separate from the Markdown parsing step since it's not as @@ -10,10 +12,12 @@ * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'ast-js', version: '1.0.0', description: 'Parses Javascript source files passed into the input.', -}; + + hasParallelProcessor: true, +}); diff --git a/src/generators/ast-js/types.d.ts b/src/generators/ast-js/types.d.ts index 2e992a22..70358fc9 100644 --- a/src/generators/ast-js/types.d.ts +++ b/src/generators/ast-js/types.d.ts @@ -1,6 +1,5 @@ -export type Generator = GeneratorMetadata<{}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + {}, Generate>, ProcessChunk >; diff --git a/src/generators/ast/generate.mjs b/src/generators/ast/generate.mjs index b4da54e0..d5574019 100644 --- a/src/generators/ast/generate.mjs +++ b/src/generators/ast/generate.mjs @@ -18,7 +18,7 @@ const remarkProcessor = getRemark(); * Process a chunk of markdown files in a worker thread. * Loads and parses markdown files into AST representations. * - * @type {import('./types').Implementation['processChunk']} + * @type {import('./types').Generator['processChunk']} */ export async function processChunk(inputSlice, itemIndices) { const filePaths = itemIndices.map(idx => inputSlice[idx]); @@ -42,7 +42,7 @@ export async function processChunk(inputSlice, itemIndices) { /** * Generates AST trees from markdown input files. * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function* generate(_, worker) { const { ast: config } = getConfig(); diff --git a/src/generators/ast/index.mjs b/src/generators/ast/index.mjs index 6d3fbfdc..4c34b637 100644 --- a/src/generators/ast/index.mjs +++ b/src/generators/ast/index.mjs @@ -1,15 +1,19 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator parses Markdown API doc files into AST trees. * It parallelizes the parsing across worker threads for better performance. * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'ast', version: '1.0.0', description: 'Parses Markdown API doc files into AST trees', -}; + + hasParallelProcessor: true, +}); diff --git a/src/generators/ast/types.d.ts b/src/generators/ast/types.d.ts index 03cba205..256ce4e7 100644 --- a/src/generators/ast/types.d.ts +++ b/src/generators/ast/types.d.ts @@ -1,8 +1,7 @@ import type { Root } from 'mdast'; -export type Generator = GeneratorMetadata<{}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + {}, Generate>>, ProcessChunk> >; diff --git a/src/generators/json-simple/generate.mjs b/src/generators/json-simple/generate.mjs index 8e9188e0..48be74e1 100644 --- a/src/generators/json-simple/generate.mjs +++ b/src/generators/json-simple/generate.mjs @@ -11,7 +11,7 @@ import createQueries from '../../utils/queries/index.mjs'; /** * Generates the simplified JSON version of the API docs * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function generate(input) { const config = getConfig('json-simple'); diff --git a/src/generators/json-simple/index.mjs b/src/generators/json-simple/index.mjs index b0fa3341..9cd62537 100644 --- a/src/generators/json-simple/index.mjs +++ b/src/generators/json-simple/index.mjs @@ -1,5 +1,7 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator generates a simplified JSON version of the API docs and returns it as a string * this is not meant to be used for the final API docs, but for debugging and testing purposes @@ -9,7 +11,7 @@ * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'json-simple', version: '1.0.0', @@ -18,4 +20,4 @@ export default { 'Generates the simple JSON version of the API docs, and returns it as a string', dependsOn: 'metadata', -}; +}); diff --git a/src/generators/json-simple/types.d.ts b/src/generators/json-simple/types.d.ts index c1e53df6..72960787 100644 --- a/src/generators/json-simple/types.d.ts +++ b/src/generators/json-simple/types.d.ts @@ -1,5 +1,4 @@ -export type Generator = GeneratorMetadata<{}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + {}, Generate, Promise>> >; diff --git a/src/generators/jsx-ast/generate.mjs b/src/generators/jsx-ast/generate.mjs index 4e18ff74..ea55af76 100644 --- a/src/generators/jsx-ast/generate.mjs +++ b/src/generators/jsx-ast/generate.mjs @@ -14,7 +14,7 @@ const remarkRecma = getRemarkRecma(); * Each item is a SlicedModuleInput containing the head node * and all entries for that module - no need to recompute grouping. * - * @type {import('./types').Implementation['processChunk']} + * @type {import('./types').Generator['processChunk']} */ export async function processChunk(slicedInput, itemIndices, docPages) { const results = []; @@ -40,7 +40,7 @@ export async function processChunk(slicedInput, itemIndices, docPages) { /** * Generates a JSX AST * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function* generate(input, worker) { const config = getConfig('jsx-ast'); diff --git a/src/generators/jsx-ast/index.mjs b/src/generators/jsx-ast/index.mjs index 435f98e8..da61cb56 100644 --- a/src/generators/jsx-ast/index.mjs +++ b/src/generators/jsx-ast/index.mjs @@ -1,11 +1,13 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * Generator for converting MDAST to JSX AST. * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'jsx-ast', version: '1.0.0', @@ -17,4 +19,6 @@ export default { defaultConfiguration: { ref: 'main', }, -}; + + hasParallelProcessor: true, +}); diff --git a/src/generators/jsx-ast/types.d.ts b/src/generators/jsx-ast/types.d.ts index 4bb81b5d..9103bea1 100644 --- a/src/generators/jsx-ast/types.d.ts +++ b/src/generators/jsx-ast/types.d.ts @@ -1,8 +1,7 @@ import type { JSXContent } from './utils/buildContent.mjs'; -export type Generator = GeneratorMetadata<{}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + {}, Generate, AsyncGenerator>, ProcessChunk< { head: ApiDocMetadataEntry; entries: Array }, diff --git a/src/generators/jsx-ast/utils/buildBarProps.mjs b/src/generators/jsx-ast/utils/buildBarProps.mjs index 3e7c7b67..2dd8c8b4 100644 --- a/src/generators/jsx-ast/utils/buildBarProps.mjs +++ b/src/generators/jsx-ast/utils/buildBarProps.mjs @@ -6,7 +6,7 @@ import { visit } from 'unist-util-visit'; import { getFullName } from './buildSignature.mjs'; import getConfig from '../../../utils/configuration/index.mjs'; import { - GH_EDIT_URL, + GITHUB_EDIT_URL, populate, } from '../../../utils/configuration/templates.mjs'; import { @@ -103,7 +103,7 @@ export const buildMetaBarProps = (head, entries) => { ['JSON', `${head.api}.json`], ['MD', `${head.api}.md`], ], - editThisPage: `${populate(GH_EDIT_URL, config)}${head.api}.md`, + editThisPage: `${populate(GITHUB_EDIT_URL, config)}${head.api}.md`, }; }; diff --git a/src/generators/jsx-ast/utils/buildContent.mjs b/src/generators/jsx-ast/utils/buildContent.mjs index cd9736ab..363d1829 100644 --- a/src/generators/jsx-ast/utils/buildContent.mjs +++ b/src/generators/jsx-ast/utils/buildContent.mjs @@ -24,7 +24,7 @@ import { import insertSignature, { getFullName } from './buildSignature.mjs'; import getConfig from '../../../utils/configuration/index.mjs'; import { - GH_BLOB_URL, + GITHUB_BLOB_URL, populate, } from '../../../utils/configuration/templates.mjs'; @@ -83,7 +83,7 @@ export const createSourceLink = sourceLink => { createElement( 'a', { - href: `${populate(GH_BLOB_URL, config)}${sourceLink}`, + href: `${populate(GITHUB_BLOB_URL, config)}${sourceLink}`, target: '_blank', }, [ diff --git a/src/generators/legacy-html-all/generate.mjs b/src/generators/legacy-html-all/generate.mjs index 7d85bfd3..b9cc3a88 100644 --- a/src/generators/legacy-html-all/generate.mjs +++ b/src/generators/legacy-html-all/generate.mjs @@ -12,7 +12,7 @@ import tableOfContents from '../legacy-html/utils/tableOfContents.mjs'; /** * Generates the `all.html` file from the `legacy-html` generator * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function generate(input) { const config = getConfig('legacy-html-all'); diff --git a/src/generators/legacy-html-all/index.mjs b/src/generators/legacy-html-all/index.mjs index ce364e78..2edea628 100644 --- a/src/generators/legacy-html-all/index.mjs +++ b/src/generators/legacy-html-all/index.mjs @@ -1,5 +1,6 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; import legacyHtml from '../legacy-html/index.mjs'; /** @@ -11,7 +12,7 @@ import legacyHtml from '../legacy-html/index.mjs'; * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'legacy-html-all', version: '1.0.0', @@ -24,4 +25,4 @@ export default { defaultConfiguration: { templatePath: legacyHtml.defaultConfiguration.templatePath, }, -}; +}); diff --git a/src/generators/legacy-html-all/types.d.ts b/src/generators/legacy-html-all/types.d.ts index 5d69700a..e005d1ff 100644 --- a/src/generators/legacy-html-all/types.d.ts +++ b/src/generators/legacy-html-all/types.d.ts @@ -8,10 +8,9 @@ export interface TemplateValues { content: string; } -export type Generator = GeneratorMetadata<{ - templatePath: string; -}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + { + templatePath: string; + }, Generate, Promise> >; diff --git a/src/generators/legacy-html/generate.mjs b/src/generators/legacy-html/generate.mjs index 210e045f..6888612d 100644 --- a/src/generators/legacy-html/generate.mjs +++ b/src/generators/legacy-html/generate.mjs @@ -28,7 +28,7 @@ const remarkRehypeProcessor = getRemarkRehypeWithShiki(); * Each item is pre-grouped {head, nodes, headNodes} - no need to * recompute groupNodesByModule for every chunk. * - * @type {import('./types').Implementation['processChunk']} + * @type {import('./types').Generator['processChunk']} */ export async function processChunk(slicedInput, itemIndices, navigation) { const results = []; @@ -72,7 +72,7 @@ export async function processChunk(slicedInput, itemIndices, navigation) { /** * Generates the legacy version of the API docs in HTML * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function* generate(input, worker) { const config = getConfig('legacy-html'); diff --git a/src/generators/legacy-html/index.mjs b/src/generators/legacy-html/index.mjs index d5b3c56d..f87910a0 100644 --- a/src/generators/legacy-html/index.mjs +++ b/src/generators/legacy-html/index.mjs @@ -2,6 +2,8 @@ import { join } from 'node:path'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * * This generator generates the legacy HTML pages of the legacy API docs @@ -12,7 +14,7 @@ import { join } from 'node:path'; * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'legacy-html', version: '1.0.0', @@ -27,4 +29,6 @@ export default { additionalPathsToCopy: [join(import.meta.dirname, 'assets')], ref: 'main', }, -}; + + hasParallelProcessor: true, +}); diff --git a/src/generators/legacy-html/types.d.ts b/src/generators/legacy-html/types.d.ts index c7732aae..e0222dfa 100644 --- a/src/generators/legacy-html/types.d.ts +++ b/src/generators/legacy-html/types.d.ts @@ -7,12 +7,11 @@ export interface TemplateValues { content: string; } -export type Generator = GeneratorMetadata<{ - templatePath: string; - additionalPathsToCopy: Array; -}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + { + templatePath: string; + additionalPathsToCopy: Array; + }, Generate, AsyncGenerator>, ProcessChunk< { diff --git a/src/generators/legacy-html/utils/buildContent.mjs b/src/generators/legacy-html/utils/buildContent.mjs index 8ab0c7ef..af22e8e3 100644 --- a/src/generators/legacy-html/utils/buildContent.mjs +++ b/src/generators/legacy-html/utils/buildContent.mjs @@ -7,7 +7,7 @@ import { SKIP, visit } from 'unist-util-visit'; import buildExtraContent from './buildExtraContent.mjs'; import getConfig from '../../../utils/configuration/index.mjs'; import { - GH_BLOB_URL, + GITHUB_BLOB_URL, populate, } from '../../../utils/configuration/templates.mjs'; import createQueries from '../../../utils/queries/index.mjs'; @@ -116,7 +116,7 @@ const buildMetadataElement = (node, remark) => { // We use a `span` element to display the source link as a clickable link to the source within Node.js if (typeof node.source_link === 'string') { // Creates the source link URL with the base URL and the source link - const sourceLink = `${populate(GH_BLOB_URL, config)}${node.source_link}`; + const sourceLink = `${populate(GITHUB_BLOB_URL, config)}${node.source_link}`; // Creates the source link element with the source link and the source link text const sourceLinkElement = createElement('span', [ diff --git a/src/generators/legacy-html/utils/replaceTemplateValues.mjs b/src/generators/legacy-html/utils/replaceTemplateValues.mjs index c2e39f09..460592d3 100644 --- a/src/generators/legacy-html/utils/replaceTemplateValues.mjs +++ b/src/generators/legacy-html/utils/replaceTemplateValues.mjs @@ -8,7 +8,7 @@ import { } from './buildDropdowns.mjs'; import tableOfContents from './tableOfContents.mjs'; import { - GH_EDIT_URL, + GITHUB_EDIT_URL, populate, } from '../../../utils/configuration/templates.mjs'; @@ -39,6 +39,8 @@ export const replaceTemplateValues = ( .replace('__ALTDOCS__', buildVersions(api, added, config.changelog)) .replace( '__EDIT_ON_GITHUB__', - skipGitHub ? '' : buildGitHub(`${populate(GH_EDIT_URL, config)}${api}.md`) + skipGitHub + ? '' + : buildGitHub(`${populate(GITHUB_EDIT_URL, config)}${api}.md`) ); }; diff --git a/src/generators/legacy-json-all/generate.mjs b/src/generators/legacy-json-all/generate.mjs index 4deb7e68..bc04f9aa 100644 --- a/src/generators/legacy-json-all/generate.mjs +++ b/src/generators/legacy-json-all/generate.mjs @@ -9,7 +9,7 @@ import { legacyToJSON } from '../../utils/generators.mjs'; /** * Generates the legacy JSON `all.json` file. * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function generate(input) { const config = getConfig('legacy-json-all'); diff --git a/src/generators/legacy-json-all/index.mjs b/src/generators/legacy-json-all/index.mjs index cce57ea0..dcfe0fab 100644 --- a/src/generators/legacy-json-all/index.mjs +++ b/src/generators/legacy-json-all/index.mjs @@ -1,12 +1,14 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator consolidates data from the `legacy-json` generator into a single * JSON file (`all.json`). * * @type {import('./types.d.ts').Generator} */ -export default { +export default createLazyGenerator({ name: 'legacy-json-all', version: '1.0.0', @@ -19,4 +21,4 @@ export default { defaultConfiguration: { minify: false, }, -}; +}); diff --git a/src/generators/legacy-json-all/types.d.ts b/src/generators/legacy-json-all/types.d.ts index 77cf7365..d95ffd14 100644 --- a/src/generators/legacy-json-all/types.d.ts +++ b/src/generators/legacy-json-all/types.d.ts @@ -13,8 +13,7 @@ export interface Output { methods: Array; } -export type Generator = GeneratorMetadata<{}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + {}, Generate, Promise> >; diff --git a/src/generators/legacy-json/generate.mjs b/src/generators/legacy-json/generate.mjs index f41daf3d..4e5972d6 100644 --- a/src/generators/legacy-json/generate.mjs +++ b/src/generators/legacy-json/generate.mjs @@ -16,7 +16,7 @@ const buildSection = createSectionBuilder(); * Each item is pre-grouped {head, nodes} - no need to * recompute groupNodesByModule for every chunk. * - * @type {import('./types').Implementation['processChunk']} + * @type {import('./types').Generator['processChunk']} */ export async function processChunk(slicedInput, itemIndices) { const results = []; @@ -33,7 +33,7 @@ export async function processChunk(slicedInput, itemIndices) { /** * Generates a legacy JSON file. * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function* generate(input, worker) { const config = getConfig('legacy-json'); diff --git a/src/generators/legacy-json/index.mjs b/src/generators/legacy-json/index.mjs index 009542e8..a2b5f5d9 100644 --- a/src/generators/legacy-json/index.mjs +++ b/src/generators/legacy-json/index.mjs @@ -1,5 +1,7 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator is responsible for generating the legacy JSON files for the * legacy API docs for retro-compatibility. It is to be replaced while we work @@ -11,7 +13,7 @@ * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'legacy-json', version: '1.0.0', @@ -24,4 +26,6 @@ export default { ref: 'main', minify: false, }, -}; + + hasParallelProcessor: true, +}); diff --git a/src/generators/legacy-json/types.d.ts b/src/generators/legacy-json/types.d.ts index 0c207253..16649acc 100644 --- a/src/generators/legacy-json/types.d.ts +++ b/src/generators/legacy-json/types.d.ts @@ -279,9 +279,8 @@ export interface ParameterList { options?: ParameterList; } -export type Generator = GeneratorMetadata<{}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + {}, Generate, AsyncGenerator
>, ProcessChunk< { head: ApiDocMetadataEntry; nodes: Array }, diff --git a/src/generators/llms-txt/generate.mjs b/src/generators/llms-txt/generate.mjs index 76e44cf0..01f4a506 100644 --- a/src/generators/llms-txt/generate.mjs +++ b/src/generators/llms-txt/generate.mjs @@ -9,7 +9,7 @@ import getConfig from '../../utils/configuration/index.mjs'; /** * Generates a llms.txt file * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function generate(input) { const config = getConfig('llms-txt'); diff --git a/src/generators/llms-txt/index.mjs b/src/generators/llms-txt/index.mjs index f94a1bbf..cce5f6ed 100644 --- a/src/generators/llms-txt/index.mjs +++ b/src/generators/llms-txt/index.mjs @@ -2,13 +2,15 @@ import { join } from 'node:path'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator generates a llms.txt file to provide information to LLMs at * inference time * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'llms-txt', version: '1.0.0', @@ -21,4 +23,4 @@ export default { defaultConfiguration: { templatePath: join(import.meta.dirname, 'template.txt'), }, -}; +}); diff --git a/src/generators/llms-txt/types.d.ts b/src/generators/llms-txt/types.d.ts index 80b58191..693195fe 100644 --- a/src/generators/llms-txt/types.d.ts +++ b/src/generators/llms-txt/types.d.ts @@ -1,7 +1,6 @@ -export type Generator = GeneratorMetadata<{ - templatePath: string; -}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + { + templatePath: string; + }, Generate, Promise> >; diff --git a/src/generators/man-page/generate.mjs b/src/generators/man-page/generate.mjs index 3f8aae65..d7947f62 100644 --- a/src/generators/man-page/generate.mjs +++ b/src/generators/man-page/generate.mjs @@ -27,7 +27,7 @@ function extractMandoc(components, start, end, convert) { /** * Generates the Node.js man-page * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function generate(input) { const config = getConfig('man-page'); diff --git a/src/generators/man-page/index.mjs b/src/generators/man-page/index.mjs index 56de3c7e..1128720e 100644 --- a/src/generators/man-page/index.mjs +++ b/src/generators/man-page/index.mjs @@ -2,13 +2,15 @@ import { join } from 'node:path'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator generates a man page version of the CLI.md file. * See https://man.openbsd.org/mdoc.7 for the formatting. * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'man-page', version: '1.0.0', @@ -23,4 +25,4 @@ export default { envVarsHeaderSlug: 'environment-variables-1', templatePath: join(import.meta.dirname, 'template.1'), }, -}; +}); diff --git a/src/generators/man-page/types.d.ts b/src/generators/man-page/types.d.ts index f4193773..a6e26f29 100644 --- a/src/generators/man-page/types.d.ts +++ b/src/generators/man-page/types.d.ts @@ -1,10 +1,9 @@ -export type Generator = GeneratorMetadata<{ - fileName: string; - cliOptionsHeaderSlug: string; - envVarsHeaderSlug: string; - templatePath: string; -}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + { + fileName: string; + cliOptionsHeaderSlug: string; + envVarsHeaderSlug: string; + templatePath: string; + }, Generate, Promise> >; diff --git a/src/generators/metadata/generate.mjs b/src/generators/metadata/generate.mjs index 7b943793..d3548141 100644 --- a/src/generators/metadata/generate.mjs +++ b/src/generators/metadata/generate.mjs @@ -8,7 +8,7 @@ import { importFromURL } from '../../utils/url.mjs'; * Process a chunk of API doc files in a worker thread. * Called by chunk-worker.mjs for parallel processing. * - * @type {import('./types').Implementation['processChunk']} + * @type {import('./types').Generator['processChunk']} */ export async function processChunk(fullInput, itemIndices, typeMap) { const results = []; @@ -23,7 +23,7 @@ export async function processChunk(fullInput, itemIndices, typeMap) { /** * Generates a flattened list of metadata entries from API docs. * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function* generate(inputs, worker) { const { metadata: config } = getConfig(); diff --git a/src/generators/metadata/index.mjs b/src/generators/metadata/index.mjs index acffb2c2..6615343f 100644 --- a/src/generators/metadata/index.mjs +++ b/src/generators/metadata/index.mjs @@ -1,11 +1,13 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator generates a flattened list of metadata entries from a API doc * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'metadata', version: '1.0.0', @@ -17,4 +19,6 @@ export default { defaultConfiguration: { typeMap: import.meta.resolve('./typeMap.json'), }, -}; + + hasParallelProcessor: true, +}); diff --git a/src/generators/metadata/types.d.ts b/src/generators/metadata/types.d.ts index 2c0919c0..403170db 100644 --- a/src/generators/metadata/types.d.ts +++ b/src/generators/metadata/types.d.ts @@ -1,10 +1,9 @@ import type { Root } from 'mdast'; -export type Generator = GeneratorMetadata<{ - typeMap: string | URL; -}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + { + typeMap: string | URL; + }, Generate, AsyncGenerator>, ProcessChunk> >; diff --git a/src/generators/orama-db/generate.mjs b/src/generators/orama-db/generate.mjs index 6149515e..4e3e0527 100644 --- a/src/generators/orama-db/generate.mjs +++ b/src/generators/orama-db/generate.mjs @@ -13,7 +13,7 @@ import { transformNodeToString } from '../../utils/unist.mjs'; /** * Generates the Orama database. * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function generate(input) { const config = getConfig('orama-db'); diff --git a/src/generators/orama-db/index.mjs b/src/generators/orama-db/index.mjs index 70bb6695..bb6c2755 100644 --- a/src/generators/orama-db/index.mjs +++ b/src/generators/orama-db/index.mjs @@ -1,12 +1,14 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator is responsible for generating the Orama database for the * API docs. It is based on the legacy-json generator. * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'orama-db', version: '1.0.0', @@ -14,4 +16,4 @@ export default { description: 'Generates the Orama database for the API docs.', dependsOn: 'metadata', -}; +}); diff --git a/src/generators/orama-db/types.d.ts b/src/generators/orama-db/types.d.ts index b5f285e4..6fe89926 100644 --- a/src/generators/orama-db/types.d.ts +++ b/src/generators/orama-db/types.d.ts @@ -23,8 +23,7 @@ export interface OramaDbEntry { */ export type OramaDb = Orama; -export type Generator = GeneratorMetadata<{}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + {}, Generate, Promise> >; diff --git a/src/generators/sitemap/generate.mjs b/src/generators/sitemap/generate.mjs index c163067c..20e01e62 100644 --- a/src/generators/sitemap/generate.mjs +++ b/src/generators/sitemap/generate.mjs @@ -9,7 +9,7 @@ import getConfig from '../../utils/configuration/index.mjs'; /** * Generates a sitemap.xml file * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function generate(entries) { const { sitemap: config } = getConfig(); diff --git a/src/generators/sitemap/index.mjs b/src/generators/sitemap/index.mjs index e3aa4335..383c3b70 100644 --- a/src/generators/sitemap/index.mjs +++ b/src/generators/sitemap/index.mjs @@ -1,11 +1,13 @@ 'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * This generator generates a sitemap.xml file for search engine optimization * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'sitemap', version: '1.0.0', @@ -13,4 +15,4 @@ export default { description: 'Generates a sitemap.xml file for search engine optimization', dependsOn: 'metadata', -}; +}); diff --git a/src/generators/sitemap/types.d.ts b/src/generators/sitemap/types.d.ts index ba849797..d9cb9c2f 100644 --- a/src/generators/sitemap/types.d.ts +++ b/src/generators/sitemap/types.d.ts @@ -12,8 +12,7 @@ export interface SitemapEntry { priority?: string; } -export type Generator = GeneratorMetadata<{}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + {}, Generate, Promise> >; diff --git a/src/generators/types.d.ts b/src/generators/types.d.ts index af93204d..f2fc03d7 100644 --- a/src/generators/types.d.ts +++ b/src/generators/types.d.ts @@ -7,9 +7,6 @@ declare global { // All generators including internal ones (metadata, jsx-ast, ast-js) export type AllGenerators = typeof allGenerators; - // The resolved type of a generator (now synchronous, not lazy) - export type ResolvedGenerator = AllGenerators[K]; - /** * ParallelWorker interface for distributing work across Node.js worker threads. * Streams results as chunks complete, enabling pipeline parallelism. @@ -60,11 +57,11 @@ declare global { dependencies: D ) => Promise; - /** - * Generator metadata - loaded synchronously from each generator's index.mjs. - * Contains only descriptive metadata and default configuration. - */ - export type GeneratorMetadata = { + export type GeneratorMetadata< + C extends any, + G extends Generate, + P extends ProcessChunk | undefined = undefined, + > = { readonly defaultConfiguration: C; // The name of the Generator. Must match the Key in AllGenerators @@ -74,6 +71,8 @@ declare global { description: string; + hasParallelProcessor: boolean; + /** * The immediate generator that this generator depends on. * For example, the `html` generator depends on the `react` generator. @@ -97,16 +96,7 @@ declare global { * passes the ASTs for any JavaScript files given in the input. */ dependsOn: keyof AllGenerators | undefined; - }; - /** - * Generator implementation - loaded dynamically from each generator's generate.mjs. - * Contains the generate function and optional processChunk for parallel processing. - */ - export type GeneratorImpl< - G extends Generate, - P extends ProcessChunk | undefined = undefined, - > = { /** * Generators are abstract and the different generators have different sort of inputs and outputs. * For example, a MDX generator would take the raw AST and output MDX with React Components; diff --git a/src/generators/web/generate.mjs b/src/generators/web/generate.mjs index 0c43914c..1679a5a8 100644 --- a/src/generators/web/generate.mjs +++ b/src/generators/web/generate.mjs @@ -11,7 +11,7 @@ import getConfig from '../../utils/configuration/index.mjs'; /** * Main generation function that processes JSX AST entries into web bundles. * - * @type {import('./types').Implementation['generate']} + * @type {import('./types').Generator['generate']} */ export async function generate(input) { const config = getConfig('web'); diff --git a/src/generators/web/index.mjs b/src/generators/web/index.mjs index 9860c7d2..00b72d20 100644 --- a/src/generators/web/index.mjs +++ b/src/generators/web/index.mjs @@ -2,6 +2,8 @@ import { join } from 'node:path'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + /** * Web generator - transforms JSX AST entries into complete web bundles. * @@ -15,7 +17,7 @@ import { join } from 'node:path'; * * @type {import('./types').Generator} */ -export default { +export default createLazyGenerator({ name: 'web', version: '1.0.0', @@ -31,4 +33,4 @@ export default { '#config/Logo': '@node-core/ui-components/Common/NodejsLogo', }, }, -}; +}); diff --git a/src/generators/web/types.d.ts b/src/generators/web/types.d.ts index e0669944..5feed60e 100644 --- a/src/generators/web/types.d.ts +++ b/src/generators/web/types.d.ts @@ -1,11 +1,10 @@ import type { JSXContent } from '../jsx-ast/utils/buildContent.mjs'; -export type Generator = GeneratorMetadata<{ - templatePath: string; - title: string; - imports: Record; -}>; - -export type Implementation = GeneratorImpl< +export type Generator = GeneratorMetadata< + { + templatePath: string; + title: string; + imports: Record; + }, Generate, AsyncGenerator<{ html: string; css: string }>> >; diff --git a/src/generators/web/utils/bundle.mjs b/src/generators/web/utils/bundle.mjs index 874cb6a2..69780372 100644 --- a/src/generators/web/utils/bundle.mjs +++ b/src/generators/web/utils/bundle.mjs @@ -14,14 +14,6 @@ const DOC_KIT_NODE_MODULES = join( '../../../../node_modules' ); -// Modules expected to be provided at runtime in the server environment, and thus excluded from the bundle. -const SERVER_MODULES = [ - 'preact', - 'preact-render-to-string', - '@node-core/ui-components', - '@node-core/rehype-shiki', -]; - /** * Asynchronously bundles JavaScript source code (and its CSS imports), * targeting either browser (client) or server (Node.js) environments. @@ -40,8 +32,6 @@ export default async function bundleCode(codeMap, { server = false } = {}) { // Experimental features: import maps for client, none for server experimental: { chunkImportMap: !server, - - lazyBarrel: true, }, checks: { @@ -69,7 +59,9 @@ export default async function bundleCode(codeMap, { server = false } = {}) { // External dependencies to exclude from bundling. // These are expected to be available at runtime in the server environment. // This reduces bundle size and avoids bundling shared server libs. - external: server ? SERVER_MODULES : [], + external: server + ? ['preact', 'preact-render-to-string', '@node-core/ui-components'] + : [], transform: { // Inject global compile-time constants that will be replaced in code. diff --git a/src/threading/__tests__/parallel.test.mjs b/src/threading/__tests__/parallel.test.mjs index 758c3d8e..41b04c6f 100644 --- a/src/threading/__tests__/parallel.test.mjs +++ b/src/threading/__tests__/parallel.test.mjs @@ -41,7 +41,7 @@ async function collectChunks(generator) { describe('createParallelWorker', () => { it('should create a ParallelWorker with stream method', async () => { const pool = createWorkerPool(2); - const worker = await createParallelWorker('metadata', pool, { threads: 2 }); + const worker = createParallelWorker('metadata', pool, { threads: 2 }); ok(worker); strictEqual(typeof worker.stream, 'function'); @@ -51,7 +51,7 @@ describe('createParallelWorker', () => { it('should handle empty items array', async () => { const pool = createWorkerPool(2); - const worker = await createParallelWorker('ast-js', pool, { + const worker = createParallelWorker('ast-js', pool, { threads: 2, chunkSize: 10, }); @@ -65,7 +65,7 @@ describe('createParallelWorker', () => { it('should distribute items to multiple worker threads', async () => { const pool = createWorkerPool(4); - const worker = await createParallelWorker('metadata', pool, { + const worker = createParallelWorker('metadata', pool, { threads: 4, chunkSize: 1, }); @@ -104,7 +104,7 @@ describe('createParallelWorker', () => { it('should yield results as chunks complete', async () => { const pool = createWorkerPool(2); - const worker = await createParallelWorker('metadata', pool, { + const worker = createParallelWorker('metadata', pool, { threads: 2, chunkSize: 1, }); @@ -131,7 +131,7 @@ describe('createParallelWorker', () => { it('should work with single thread and items', async () => { const pool = createWorkerPool(2); - const worker = await createParallelWorker('metadata', pool, { + const worker = createParallelWorker('metadata', pool, { threads: 2, chunkSize: 5, }); @@ -155,7 +155,7 @@ describe('createParallelWorker', () => { it('should use sliceInput for metadata generator', async () => { const pool = createWorkerPool(2); - const worker = await createParallelWorker('metadata', pool, { + const worker = createParallelWorker('metadata', pool, { threads: 2, chunkSize: 1, }); diff --git a/src/threading/chunk-worker.mjs b/src/threading/chunk-worker.mjs index 8e0ddf8f..08e363ac 100644 --- a/src/threading/chunk-worker.mjs +++ b/src/threading/chunk-worker.mjs @@ -1,3 +1,4 @@ +import { allGenerators } from '../generators/index.mjs'; import { setConfig } from '../utils/configuration/index.mjs'; /** @@ -16,9 +17,7 @@ export default async ({ }) => { await setConfig(configuration); - const { processChunk } = await import( - `../generators/${generatorName}/generate.mjs` - ); + const generator = allGenerators[generatorName]; - return processChunk(input, itemIndices, extra); + return generator.processChunk(input, itemIndices, extra); }; diff --git a/src/threading/parallel.mjs b/src/threading/parallel.mjs index 5b083a05..6627f0f0 100644 --- a/src/threading/parallel.mjs +++ b/src/threading/parallel.mjs @@ -1,5 +1,6 @@ 'use strict'; +import { allGenerators } from '../generators/index.mjs'; import logger from '../logger/index.mjs'; const parallelLogger = logger.child('parallel'); @@ -62,16 +63,14 @@ const createTask = ( * @param {import('../utils/configuration/types').Configuration} configuration - Generator options * @returns {ParallelWorker} */ -export default async function createParallelWorker( +export default function createParallelWorker( generatorName, pool, configuration ) { const { threads, chunkSize } = configuration; - const { processChunk } = await import( - `../generators/${generatorName}/generate.mjs` - ); + const generator = allGenerators[generatorName]; return { /** @@ -101,9 +100,9 @@ export default async function createParallelWorker( const pending = new Set( chunks.map(indices => { if (runInOneGo) { - const promise = processChunk(fullInput, indices, extra).then( - result => ({ promise, result }) - ); + const promise = generator + .processChunk(fullInput, indices, extra) + .then(result => ({ promise, result })); return promise; } diff --git a/src/utils/configuration/index.mjs b/src/utils/configuration/index.mjs index 3a465ee2..fd3d210d 100644 --- a/src/utils/configuration/index.mjs +++ b/src/utils/configuration/index.mjs @@ -14,30 +14,30 @@ import { importFromURL } from '../url.mjs'; /** * Get's the default configuration */ -export const getDefaultConfig = lazy(async () => { - const defaults = /** @type {import('./types').Configuration} */ ({ - global: { - version: process.version, - minify: true, - repository: 'nodejs/node', - ref: 'HEAD', - baseURL: 'https://nodejs.org/docs', - changelog: populate(CHANGELOG_URL, { +export const getDefaultConfig = lazy(() => + Object.keys(allGenerators).reduce( + (acc, k) => { + acc[k] = allGenerators[k].defaultConfiguration ?? {}; + return acc; + }, + /** @type {import('./types').Configuration} */ ({ + global: { + version: process.version, + minify: true, repository: 'nodejs/node', ref: 'HEAD', - }), - }, - - threads: cpus().length, - chunkSize: 10, - }); - - for (const k of Object.keys(allGenerators)) { - defaults[k] = allGenerators[k].defaultConfiguration ?? {}; - } - - return defaults; -}); + baseURL: 'https://nodejs.org/docs', + changelog: populate(CHANGELOG_URL, { + repository: 'nodejs/node', + ref: 'HEAD', + }), + }, + + threads: cpus().length, + chunkSize: 10, + }) + ) +); /** * Loads a configuration file from a URL or file path. @@ -114,7 +114,7 @@ export const createRunConfiguration = async options => { const merged = deepMerge( config, createConfigFromCLIOptions(options), - await getDefaultConfig() + getDefaultConfig() ); // These need to be coerced diff --git a/src/utils/configuration/templates.mjs b/src/utils/configuration/templates.mjs index 778c610f..802e227b 100644 --- a/src/utils/configuration/templates.mjs +++ b/src/utils/configuration/templates.mjs @@ -3,11 +3,11 @@ export const CHANGELOG_URL = 'https://raw.githubusercontent.com/{repository}/{ref}/CHANGELOG.md'; // This is the Base URL for viewing a file within GitHub UI -export const GH_BLOB_URL = 'https://github.com/{repository}/blob/{ref}/'; +export const GITHUB_BLOB_URL = 'https://github.com/{repository}/blob/{ref}/'; // This is the API docs base URL for editing a file on GitHub UI // TODO(@avivkeller): specify /doc/api in config -export const GH_EDIT_URL = +export const GITHUB_EDIT_URL = 'https://github.com/{repository}/edit/{ref}/doc/api/'; /** diff --git a/src/utils/configuration/types.d.ts b/src/utils/configuration/types.d.ts index 18deb134..853ad969 100644 --- a/src/utils/configuration/types.d.ts +++ b/src/utils/configuration/types.d.ts @@ -16,7 +16,7 @@ export type Configuration = { chunkSize: number; } & { [K in keyof AllGenerators]: GlobalConfiguration & - ResolvedGenerator['defaultConfiguration']; + AllGenerators[K]['defaultConfiguration']; }; export type GlobalConfiguration = { diff --git a/src/utils/generators.mjs b/src/utils/generators.mjs index a857fa53..6b8d2884 100644 --- a/src/utils/generators.mjs +++ b/src/utils/generators.mjs @@ -2,6 +2,8 @@ import { coerce, major } from 'semver'; +import { lazy } from './misc.mjs'; + /** * Groups all the API metadata nodes by module (`api` property) so that we can process each different file * based on the module it belongs to. @@ -161,3 +163,30 @@ export const buildApiDocURL = (entry, baseURL, useHtml = false) => { return URL.parse(path, baseURL); }; + +/** + * Creates a generator with the provided metadata. + * @template T + * @param {T} metadata - The metadata object + * @returns {Promise} The metadata object with generator methods + */ +export const createLazyGenerator = metadata => { + const generator = lazy( + () => import(`../generators/${metadata.name}/generate.mjs`) + ); + return { + ...metadata, + /** + * Processes a chunk using the lazily-loaded generator. + * @param {...any} args - Arguments to pass to the processChunk method + * @returns {Promise} Result from the generator's processChunk method + */ + processChunk: async (...args) => (await generator()).processChunk(...args), + /** + * Generates output using the lazily-loaded generator. + * @param {...any} args - Arguments to pass to the generate method + * @returns {Promise} Result from the generator's generate method + */ + generate: async (...args) => (await generator()).generate(...args), + }; +};