diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 693f8b12ef..cdac3713c1 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -39,7 +39,7 @@ npm run docs # resolve product docs into nuxt/content/docs, no build npm run build # production build ``` -> When working on the handbook, docs, or other migrated sections, `npm run dev:nuxt` is sufficient. `npm start` is only needed when also touching 11ty-served pages. +> When working on the handbook, docs, or other migrated sections, `npm run dev:nuxt` is sufficient. `npm start` is only needed when also touching 11ty-served pages. For product docs, add `npm run dev:docs` beside it: that watcher re-syncs each edited page, and without it a docs edit only appears after a restart. `npm run dev` and `npm start` already include it. > > **Local docs development:** a checkout of `flowfuse/flowfuse` sitting next to this repo (`../flowfuse`) is picked up automatically, with no configuration. Full resolution order, which every build logs: `FLOWFUSE_DOCS_LOCAL` (explicit path, and a path that does not exist is an error), then a sibling checkout, then a clone of `FLOWFUSE_DOCS_REF` (default `main`) — this is what Netlify production deploys use. CI relies on the sibling rule: `FlowFuse/flowfuse`'s `Publish Documentation` workflow checks itself out next to the website so a docs PR is validated against its own changes. diff --git a/README.md b/README.md index 7a0d10e7c4..063e86005e 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,8 @@ Nothing needs configuring for that to happen. Every build resolves the docs in t `npm run docs` runs that resolution on its own, without a full build, writing `nuxt/content/docs` and `nuxt/public/docs`. Both are generated, and neither is committed on `main`. +`npm run dev` and `npm start` also watch the resolved docs and re-sync each file as it changes, so an edit appears without restarting. `npm run dev:nuxt` on its own does not include that watcher; run `npm run dev:docs` beside it if you want one. + ## llms.txt `/llms.txt` (and `/llms-full.txt`) are generated by the [`nuxt-llms`](https://github.com/nuxtlabs/nuxt-llms) module, configured in `nuxt/nuxt.config.ts` under the `llms` key. Sections are built from `@nuxt/content` collections (`docs`, `blog`, `changelog`, `ebooks`, `whitepapers`) plus a small hardcoded list of standalone Nuxt routes (pricing, integrations, etc.) that aren't backed by a collection. diff --git a/nuxt/lib/docs-sync.mjs b/nuxt/lib/docs-sync.mjs index e8ca1b37a4..766eceb612 100644 --- a/nuxt/lib/docs-sync.mjs +++ b/nuxt/lib/docs-sync.mjs @@ -3,7 +3,7 @@ import { execFileSync } from 'node:child_process' import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs' -import { join, relative } from 'node:path' +import { basename, dirname, join, relative } from 'node:path' import { tmpdir } from 'node:os' import { processMarkdown } from './docs-markdown.mjs' @@ -96,43 +96,103 @@ function gitOutput (cwd, args) { } } -function copyDocsDir (srcDir, repoRoot, contentDir, publicDir, version) { - mkdirSync(contentDir, { recursive: true }) - mkdirSync(publicDir, { recursive: true }) +/** + * Where one source file lands: markdown becomes a page under content/, a README becomes + * its section index, and anything else is an asset served from public/. + */ +function destinationFor (relPath, contentDocsDir, publicDocsDir) { + const name = basename(relPath) + const dir = dirname(relPath) + const prefix = dir === '.' ? '' : dir + + return name.endsWith('.md') + ? join(contentDocsDir, prefix, name === 'README.md' ? 'index.md' : name) + : join(publicDocsDir, prefix, name) +} - for (const entry of readdirSync(srcDir, { withFileTypes: true })) { - if (entry.name.startsWith('.')) continue +/** + * Copy one file out of the docs tree. Split out of copyDocsDir so a dev-server edit can + * sync just the file that changed through exactly the same code as a full build. + */ +function writeDocsFile ({ docsDir, sourceRoot, contentDocsDir, publicDocsDir, relPath, version }) { + const srcPath = join(docsDir, relPath) + const destPath = destinationFor(relPath, contentDocsDir, publicDocsDir) - const srcPath = join(srcDir, entry.name) - const destName = entry.name === 'README.md' ? 'index.md' : entry.name + mkdirSync(dirname(destPath), { recursive: true }) - if (entry.isDirectory()) { - copyDocsDir(srcPath, repoRoot, join(contentDir, entry.name), join(publicDir, entry.name), version) - } else if (entry.name.endsWith('.md')) { - const relFromRepo = relative(repoRoot, srcPath) - const originalPath = relative(join(repoRoot, 'docs'), srcPath) + if (!relPath.endsWith('.md')) { + cpSync(srcPath, destPath) + return + } - // Argument array, not a shell string: relFromRepo comes from filenames in the - // source repo, so quoting it into a shell command would be an injection path. - const updated = gitOutput(repoRoot, ['log', '-1', '--pretty=format:%ci', '--', relFromRepo]) + // Argument array, not a shell string: the path comes from filenames in the source + // repo, so quoting it into a shell command would be an injection path. + const updated = gitOutput(sourceRoot, ['log', '-1', '--pretty=format:%ci', '--', relative(sourceRoot, srcPath)]) - const raw = readFileSync(srcPath, 'utf8') - writeFileSync(join(contentDir, destName), processMarkdown(raw, originalPath, updated, version), 'utf8') + const raw = readFileSync(srcPath, 'utf8') + writeFileSync(destPath, processMarkdown(raw, relPath, updated, version), 'utf8') +} + +function copyDocsDir ({ docsDir, sourceRoot, contentDocsDir, publicDocsDir, version, relDir = '' }) { + mkdirSync(join(contentDocsDir, relDir), { recursive: true }) + mkdirSync(join(publicDocsDir, relDir), { recursive: true }) + + for (const entry of readdirSync(join(docsDir, relDir), { withFileTypes: true })) { + if (entry.name.startsWith('.')) continue + + const relPath = join(relDir, entry.name) + const args = { docsDir, sourceRoot, contentDocsDir, publicDocsDir, version } + + if (entry.isDirectory()) { + copyDocsDir({ ...args, relDir: relPath }) } else { - cpSync(srcPath, join(publicDir, entry.name)) + writeDocsFile({ ...args, relPath }) } } } -function writeDocs ({ docsDir, sourceRoot, contentDocsDir, publicDocsDir, kind, ref }) { - let version = '' +function readVersion (sourceRoot) { try { - version = JSON.parse(readFileSync(join(sourceRoot, 'package.json'), 'utf8')).version || '' - } catch { /* not fatal */ } + return JSON.parse(readFileSync(join(sourceRoot, 'package.json'), 'utf8')).version || '' + } catch { + return '' // not fatal + } +} + +/** + * Sync a single file from the docs tree, or remove its output if the file has gone. + * + * The dev watcher calls this once per change. `syncDocs()` rebuilds the whole tree, which + * is what a build wants and what a running dev server cannot survive: deleting and + * recreating all 130-odd pages on every save makes @nuxt/content re-index the entire + * collection at once and exhaust its heap. + */ +export function syncDocsPath ({ docsDir, nuxtRoot, relPath }) { + const sourceRoot = join(docsDir, '..') + const contentDocsDir = join(nuxtRoot, 'content', 'docs') + const publicDocsDir = join(nuxtRoot, 'public', 'docs') + + if (!existsSync(join(docsDir, relPath))) { + rmSync(destinationFor(relPath, contentDocsDir, publicDocsDir), { force: true }) + return + } + + writeDocsFile({ + docsDir, + sourceRoot, + contentDocsDir, + publicDocsDir, + relPath, + version: readVersion(sourceRoot), + }) +} + +function writeDocs ({ docsDir, sourceRoot, contentDocsDir, publicDocsDir, kind, ref }) { + const version = readVersion(sourceRoot) rmSync(contentDocsDir, { recursive: true, force: true }) rmSync(publicDocsDir, { recursive: true, force: true }) - copyDocsDir(docsDir, sourceRoot, contentDocsDir, publicDocsDir, version) + copyDocsDir({ docsDir, sourceRoot, contentDocsDir, publicDocsDir, version }) const manifest = { source: kind, diff --git a/nuxt/lib/docs-sync.test.mjs b/nuxt/lib/docs-sync.test.mjs index ad9c267e7d..014c7692bb 100644 --- a/nuxt/lib/docs-sync.test.mjs +++ b/nuxt/lib/docs-sync.test.mjs @@ -1,7 +1,10 @@ import { test } from 'node:test' import assert from 'node:assert/strict' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' -import { resolveSource } from './docs-sync.mjs' +import { resolveSource, syncDocsPath } from './docs-sync.mjs' const repoRoot = '/repo/website' @@ -52,3 +55,92 @@ test('cloning falls back to main and honours an explicit ref', () => { assert.deepEqual(resolve({}), { kind: 'clone', ref: 'main' }) assert.equal(resolve({ FLOWFUSE_DOCS_REF: 'maintenance' }).ref, 'maintenance') }) + +// A flowfuse checkout next to a website checkout, as the dev watcher sees it. Outside a +// git repo, so the `git log` that dates a page returns nothing and the output is stable. +function fixture (t) { + const root = mkdtempSync(join(tmpdir(), 'docs-sync-')) + t.after(() => rmSync(root, { recursive: true, force: true })) + + const docsDir = join(root, 'flowfuse', 'docs') + const nuxtRoot = join(root, 'website', 'nuxt') + + mkdirSync(join(docsDir, 'cloud'), { recursive: true }) + mkdirSync(join(nuxtRoot, 'content', 'docs'), { recursive: true }) + mkdirSync(join(nuxtRoot, 'public', 'docs'), { recursive: true }) + writeFileSync(join(root, 'flowfuse', 'package.json'), JSON.stringify({ version: '2.34.0' })) + + return { + docsDir, + nuxtRoot, + content: (...parts) => join(nuxtRoot, 'content', 'docs', ...parts), + public: (...parts) => join(nuxtRoot, 'public', 'docs', ...parts), + } +} + +test('an edited page is written with its provenance frontmatter', (t) => { + const fx = fixture(t) + writeFileSync(join(fx.docsDir, 'cloud', 'billing.md'), '# Billing\n') + + syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'cloud/billing.md' }) + + const page = readFileSync(fx.content('cloud', 'billing.md'), 'utf8') + assert.match(page, /^---\noriginalPath: cloud\/billing\.md\n/) + assert.match(page, /^version: 2\.34\.0$/m) + assert.match(page, /# Billing/) +}) + +test('a README becomes the index page of its section', (t) => { + const fx = fixture(t) + writeFileSync(join(fx.docsDir, 'cloud', 'README.md'), '# Cloud\n') + + syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'cloud/README.md' }) + + assert.ok(existsSync(fx.content('cloud', 'index.md'))) + assert.ok(!existsSync(fx.content('cloud', 'README.md'))) +}) + +test('a non-markdown file is copied to the public tree unchanged', (t) => { + const fx = fixture(t) + writeFileSync(join(fx.docsDir, 'cloud', 'diagram.svg'), '') + + syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'cloud/diagram.svg' }) + + assert.equal(readFileSync(fx.public('cloud', 'diagram.svg'), 'utf8'), '') + assert.ok(!existsSync(fx.content('cloud', 'diagram.svg'))) +}) + +test('a deleted page is removed rather than left stale', (t) => { + const fx = fixture(t) + writeFileSync(fx.content('gone.md'), 'stale\n') + + syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'gone.md' }) + + assert.ok(!existsSync(fx.content('gone.md'))) +}) + +test('a deleted asset is removed from the public tree', (t) => { + const fx = fixture(t) + mkdirSync(fx.public('cloud'), { recursive: true }) + writeFileSync(fx.public('cloud', 'gone.png'), 'stale') + + syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'cloud/gone.png' }) + + assert.ok(!existsSync(fx.public('cloud', 'gone.png'))) +}) + +// The reason this function exists. Re-running the whole sync on every keystroke deletes and +// recreates all 130-odd pages, and @nuxt/content's dev watcher runs out of heap re-indexing +// them. One edit has to cost one write. +test('syncing one page leaves every other page untouched', (t) => { + const fx = fixture(t) + writeFileSync(join(fx.docsDir, 'first.md'), '# First\n') + writeFileSync(join(fx.docsDir, 'second.md'), '# Second\n') + writeFileSync(fx.content('first.md'), 'previously synced first\n') + writeFileSync(fx.content('second.md'), 'previously synced second\n') + + syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'first.md' }) + + assert.match(readFileSync(fx.content('first.md'), 'utf8'), /# First/) + assert.equal(readFileSync(fx.content('second.md'), 'utf8'), 'previously synced second\n') +}) diff --git a/package-lock.json b/package-lock.json index cea98af5c2..a6b2868e99 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,6 +38,7 @@ "@takumi-rs/core": "1.8.4", "@types/markdown-it-attrs": "^4.1.3", "@types/markdown-it-footnote": "^3.0.4", + "chokidar": "^3.6.0", "concurrently": "^9.0.0", "del-cli": "^5.0.0", "dotenv-cli": "^7.4.4", diff --git a/package.json b/package.json index 515e4a26d6..cc55c82c47 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ ], "scripts": { "test": "node --test nuxt/server/lib/*.test.mjs nuxt/lib/*.test.mjs", - "dev": "concurrently \"npm run dev:eleventy\" \"npm run dev:postcss\" \"npm run dev:postcss-nuxt\" \"dotenv -- npm run dev --workspace=nuxt\"", + "dev": "concurrently \"npm run dev:eleventy\" \"npm run dev:docs\" \"npm run dev:postcss\" \"npm run dev:postcss-nuxt\" \"dotenv -- npm run dev --workspace=nuxt\"", "start": "npm-run-all2 clean:dev build:js blueprints --parallel dev:*", "build:js": "terser -c -m -o _site/js/cc.min.js node_modules/vanilla-cookieconsent/dist/cookieconsent.umd.js src/js/cookieconsent-config.js && cp node_modules/@flowfuse/flow-renderer/index.min.js _site/js/flowrenderer.min.js", "build": "dotenv -v NODE_ENV=production -- npm-run-all2 clean build:js --parallel prod:*", @@ -19,6 +19,7 @@ "clean:dev": "dotenv -- npx del-cli '_site/!(img)' 'src/blueprints/**/!(*submit.njk)' && dotenv -- npx mkdirp '_site/js/flows'", "clean": "dotenv -- npx del-cli '_site/!(img)' && dotenv -- mkdir -p '_site/js'", "dev:blueprints": "node scripts/watch_blueprints.js", + "dev:docs": "node scripts/watch_docs.mjs", "dev:netlify": "npx netlify dev -c \"dotenv -- npx @11ty/eleventy --serve --quiet --incremental\"", "dev:postcss": "dotenv -v TAILWIND_MODE=watch -- npx postcss ./src/css/style.css -o ./_site/css/style.css --config ./postcss.config.js -w", "dev:postcss-nuxt": "dotenv -v TAILWIND_MODE=watch -- npx postcss ./src/css/style.css -o ./nuxt/public/css/style.css --config ./postcss.config.js -w", @@ -50,6 +51,7 @@ "@tailwindcss/typography": "^0.5.14", "@types/markdown-it-attrs": "^4.1.3", "@types/markdown-it-footnote": "^3.0.4", + "chokidar": "^3.6.0", "concurrently": "^9.0.0", "del-cli": "^5.0.0", "dotenv-cli": "^7.4.4", diff --git a/scripts/watch_docs.mjs b/scripts/watch_docs.mjs new file mode 100644 index 0000000000..8ddbd8cd4b --- /dev/null +++ b/scripts/watch_docs.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +// Keeps nuxt/content/docs in step with a local flowfuse checkout while the dev server runs. +// nuxt/modules/docs-source.ts syncs once during setup and never again, so without this a +// docs edit only shows up after a restart. +// +// One edit syncs one file. Re-running the whole sync instead would delete and recreate all +// 130-odd pages on every save, and @nuxt/content re-indexing the entire collection that way +// exhausts the dev server's heap. + +import { basename, dirname, join, relative } from 'node:path' +import { fileURLToPath } from 'node:url' + +import chokidar from 'chokidar' + +import { resolveSource, syncDocsPath } from '../nuxt/lib/docs-sync.mjs' + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..') +const nuxtRoot = join(repoRoot, 'nuxt') + +// The same precedence the build uses, rather than a second hardcoded path that could drift +// from it: FLOWFUSE_DOCS_LOCAL, then a sibling checkout, then a clone. +const source = resolveSource({ repoRoot }) + +if (source.kind === 'clone') { + console.log(`Docs resolve to a clone of ${source.ref}, so there is nothing local to watch`) + process.exit(0) +} + +const { docsDir } = source + +const watcher = chokidar.watch(docsDir, { + // The Nuxt module has already synced by the time this starts. + ignoreInitial: true, + ignored: (path) => basename(path).startsWith('.'), + // Editors write a file in more than one step, so wait for it to settle rather than + // publish a half-written page. + awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 50 }, + // Host file events do not cross the macOS podman VM, which is why compose.yaml sets + // these for every other watcher in this repo. + usePolling: Boolean(process.env.CHOKIDAR_USEPOLLING), + interval: Number(process.env.CHOKIDAR_INTERVAL) || 100, +}) + +// No extension allowlist: the build copies every non-markdown file into public/docs, so +// watching everything is what keeps the dev tree matching a build. +const verbs = { add: 'Added', change: 'Synced', unlink: 'Removed' } + +for (const [event, verb] of Object.entries(verbs)) { + watcher.on(event, (path) => { + const relPath = relative(docsDir, path) + try { + syncDocsPath({ docsDir, nuxtRoot, relPath }) + console.log(`${verb} ${relPath}`) + } catch (err) { + // One bad file must not take the watcher down; the next save retries it. + console.error(`Could not sync ${relPath}: ${err.message}`) + } + }) +} + +watcher.on('ready', () => console.log(`Watching ${docsDir} for docs changes`))