diff --git a/.gitignore b/.gitignore index cf61f01..483d879 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ out *.generated.* /.cache /pages/api +/pages/loaders/ +/pages/plugins/ diff --git a/components/Footer/index.jsx b/components/Footer/index.jsx index 16f9ace..7f6d787 100644 --- a/components/Footer/index.jsx +++ b/components/Footer/index.jsx @@ -2,7 +2,7 @@ import GitHubIcon from '@node-core/ui-components/Icons/Social/GitHub'; import LinkedInIcon from '@node-core/ui-components/Icons/Social/LinkedIn'; import DiscordIcon from '@node-core/ui-components/Icons/Social/Discord'; import XIcon from '@node-core/ui-components/Icons/Social/X'; -import { footer } from '#theme/site' with { type: 'json' }; +import { footer } from '#theme/site'; import Logo from '#theme/Logo'; import styles from './index.module.css'; diff --git a/components/NavBar.jsx b/components/NavBar.jsx index fd75c3e..ab10da8 100644 --- a/components/NavBar.jsx +++ b/components/NavBar.jsx @@ -5,7 +5,7 @@ import GitHubIcon from '@node-core/ui-components/Icons/Social/GitHub'; import SearchBox from '@node-core/doc-kit/src/generators/web/ui/components/SearchBox'; import { useTheme } from '@node-core/doc-kit/src/generators/web/ui/hooks/useTheme.mjs'; -import { navbar } from '#theme/site' with { type: 'json' }; +import { navbar } from '#theme/site'; import Logo from '#theme/Logo'; /** diff --git a/components/SideBar.jsx b/components/SideBar.jsx index 2b5b181..b33cd05 100644 --- a/components/SideBar.jsx +++ b/components/SideBar.jsx @@ -1,5 +1,5 @@ import SideBar from '@node-core/ui-components/Containers/Sidebar'; -import { sidebar } from '#theme/local/site' with { type: 'json' }; +import { sidebar } from '#theme/local/site'; /** @param {string} url */ const redirect = url => (window.location.href = url); @@ -8,15 +8,24 @@ const PrefetchLink = props => ; const pathnameFor = path => path.replace(/\/index$/, '') || '/'; +const groupsFor = path => { + const segment = path.split('/').filter(Boolean)[0]; + const matched = sidebar.filter(g => g.groupName.toLowerCase() === segment); + return matched.length > 0 ? matched : sidebar; +}; + /** * Sidebar component for MDX documentation with page navigation. */ -export default ({ metadata }) => ( - -); +export default ({ metadata }) => { + const path = pathnameFor(metadata.path); + return ( + + ); +}; diff --git a/package.json b/package.json index 58a6015..4d266ba 100644 --- a/package.json +++ b/package.json @@ -2,8 +2,11 @@ "scripts": { "prep": "node scripts/prepare/index.mjs", "build:md": "node scripts/markdown/index.mjs", + "build:md:loaders": "node scripts/fetch-readmes.mjs --loaders", + "build:md:plugins": "node scripts/fetch-readmes.mjs --plugins", + "build:md:readmes": "node scripts/fetch-readmes.mjs", "build:html": "node scripts/html/index.mjs", - "build": "npm run prep && npm run build:md && npm run build:html", + "build": "npm run prep && npm run build:md && npm run build:md:readmes && npm run build:html", "lint": "eslint .", "lint:fix": "eslint --fix .", "format": "prettier --write .", diff --git a/pages/site.mjs b/pages/site.mjs new file mode 100644 index 0000000..22c8075 --- /dev/null +++ b/pages/site.mjs @@ -0,0 +1,7 @@ +import base from './site.json' with { type: 'json' }; +import loadersSite from './loaders/site.json' with { type: 'json' }; +import pluginsSite from './plugins/site.json' with { type: 'json' }; + +export const { navbar, footer } = base; + +export const sidebar = [...loadersSite.sidebar, ...pluginsSite.sidebar]; diff --git a/scripts/fetch-readmes.mjs b/scripts/fetch-readmes.mjs new file mode 100644 index 0000000..ac424c8 --- /dev/null +++ b/scripts/fetch-readmes.mjs @@ -0,0 +1,196 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const { GH_TOKEN } = process.env; + +const BASE_HEADERS = { + ...(GH_TOKEN && { Authorization: `Bearer ${GH_TOKEN}` }), + 'X-GitHub-Api-Version': '2022-11-28', +}; + +const parseNextLink = linkHeader => { + if (!linkHeader) return null; + const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/); + return match ? match[1] : null; +}; + +const discoverRepos = async () => { + const loaders = []; + const plugins = []; + let url = + 'https://api.github.com/orgs/webpack/repos?per_page=100&type=public'; + + while (url) { + const res = await fetch(url, { headers: BASE_HEADERS }); + if (!res.ok) + throw new Error( + `Failed to list org repos: ${res.status} ${res.statusText}` + ); + + const repos = await res.json(); + for (const repo of repos) { + if (repo.archived) continue; + if (repo.name.endsWith('-loader')) { + loaders.push(repo.full_name); + } else if (repo.name.endsWith('-plugin')) { + plugins.push(repo.full_name); + } + } + + url = parseNextLink(res.headers.get('link')); + } + + return { loaders, plugins }; +}; + +const stripLeadingDiv = content => + content.replace(/^\s*\n*/i, ''); + +// Remove badge lines - lines consisting only of [![...][ref]][ref] or [![...](url)](url) links +const stripBadges = content => + content + .replace( + /^(\[!\[[^\]]*\](?:\[[^\]]*\]|\([^)]*\))\]\s*(?:\[[^\]]*\]|\([^)]*\))\s*)+$/gm, + '' + ) + .replace(/\n{3,}/g, '\n\n'); + +// TODO: remove this allowlist once Shiki silently skips unknown languages instead of build errors. +const SUPPORTED_LANGS = new Set([ + 'bash', + 'c', + 'c++', + 'cjs', + 'coffee', + 'coffeescript', + 'console', + 'cpp', + 'diff', + 'docker', + 'dockerfile', + 'glsl', + 'gql', + 'graphql', + 'http', + 'ini', + 'java', + 'javascript', + 'js', + 'json', + 'jsx', + 'mjs', + 'powershell', + 'ps', + 'ps1', + 'regex', + 'regexp', + 'sh', + 'shell', + 'shellscript', + 'shellsession', + 'sql', + 'ts', + 'tsx', + 'typescript', + 'xml', + 'yaml', + 'yml', + 'zsh', +]); + +const sanitizeCodeFences = content => + content.replace(/^```([a-zA-Z0-9_+-]+)\b/gm, (match, lang) => + SUPPORTED_LANGS.has(lang.toLowerCase()) ? match : '```' + ); + +// remark-gfm does not support GitHub alert syntax (> [!TYPE]); rewrite to bold label inside the blockquote. +const GFM_ALERT_LABELS = { + NOTE: 'Note', + TIP: 'Tip', + IMPORTANT: 'Important', + WARNING: 'Warning', + CAUTION: 'Caution', +}; +const GFM_ALERT_RE = + /^([ \t]*>[ \t]*)\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\][ \t]*$/gim; + +const transformGfmAlerts = content => + content.replace( + GFM_ALERT_RE, + (_, prefix, type) => `${prefix}**${GFM_ALERT_LABELS[type]}:**` + ); + +const processContent = content => + transformGfmAlerts(sanitizeCodeFences(stripBadges(stripLeadingDiv(content)))); + +const fetchReadme = async fullName => { + const url = `https://raw.githubusercontent.com/${fullName}/HEAD/README.md`; + const res = await fetch(url); + return res.ok + ? { ok: true, text: await res.text() } + : { ok: false, status: res.status }; +}; + +const processRepos = async (repos, { groupName, basePath, outputDir }) => { + mkdirSync(outputDir, { recursive: true }); + const repoName = r => r.split('/')[1]; + console.log( + `Discovered ${groupName.toLowerCase()}: ${repos.map(repoName).join(', ')}` + ); + + const fetched = []; + for (const fullName of repos) { + const name = repoName(fullName); + const result = await fetchReadme(fullName); + if (!result.ok) { + console.log(`Failed: ${name} — ${result.status}`); + continue; + } + const content = processContent(result.text); + writeFileSync(join(outputDir, `${name}.md`), content, 'utf8'); + fetched.push(name); + console.log(`Fetched: ${name}`); + } + + const siteJson = { + sidebar: [ + { + groupName, + items: fetched + .sort() + .map(name => ({ link: `${basePath}/${name}`, label: name })), + }, + ], + }; + writeFileSync( + join(outputDir, 'site.json'), + JSON.stringify(siteJson, null, 2) + '\n', + 'utf8' + ); + console.log( + `Written: ${outputDir}/site.json (${fetched.length} ${groupName.toLowerCase()})` + ); +}; + +const args = process.argv.slice(2); +const runLoaders = args.includes('--loaders') || args.length === 0; +const runPlugins = args.includes('--plugins') || args.length === 0; + +const root = join(import.meta.dirname, '..'); +const { loaders, plugins } = await discoverRepos(); + +if (runLoaders) { + await processRepos(loaders, { + groupName: 'Loaders', + basePath: '/loaders', + outputDir: join(root, 'pages/loaders'), + }); +} + +if (runPlugins) { + await processRepos(plugins, { + groupName: 'Plugins', + basePath: '/plugins', + outputDir: join(root, 'pages/plugins'), + }); +} diff --git a/scripts/html/doc-kit.config.mjs b/scripts/html/doc-kit.config.mjs index 1a366f9..8e64edd 100644 --- a/scripts/html/doc-kit.config.mjs +++ b/scripts/html/doc-kit.config.mjs @@ -40,10 +40,12 @@ export default { useAbsoluteURLs: true, remoteConfigUrl: null, imports: { - '#theme/local/site': join(ROOT, inputDir, 'site.json'), + '#theme/local/site': VERSION + ? join(inputDir, 'site.json') + : join(ROOT, 'pages/site.mjs'), '#theme/Sidebar': join(ROOT, 'components/SideBar.jsx'), - '#theme/site': join(ROOT, 'pages/site.json'), + '#theme/site': join(ROOT, 'pages/site.mjs'), '#theme/Layout': join(ROOT, 'components/Layout.jsx'), '#theme/Navigation': join(ROOT, 'components/NavBar.jsx'), '#theme/Footer': join(ROOT, 'components/Footer/index.jsx'),