Add a docs watcher for the dev server

.claude/CLAUDE.md

@@ -39,7 +39,7 @@ npm run docs           # resolve product docs into nuxt/content/docs, no build
 npm run build          # production build
 ```
 
-> When working on the handbook, docs, or other migrated sections, `npm run dev:nuxt` is sufficient. `npm start` is only needed when also touching 11ty-served pages.
+> When working on the handbook, docs, or other migrated sections, `npm run dev:nuxt` is sufficient. `npm start` is only needed when also touching 11ty-served pages. For product docs, add `npm run dev:docs` beside it: that watcher re-syncs each edited page, and without it a docs edit only appears after a restart. `npm run dev` and `npm start` already include it.
 >
 > **Local docs development:** a checkout of `flowfuse/flowfuse` sitting next to this repo (`../flowfuse`) is picked up automatically, with no configuration. Full resolution order, which every build logs: `FLOWFUSE_DOCS_LOCAL` (explicit path, and a path that does not exist is an error), then a sibling checkout, then a clone of `FLOWFUSE_DOCS_REF` (default `main`) — this is what Netlify production deploys use. CI relies on the sibling rule: `FlowFuse/flowfuse`'s `Publish Documentation` workflow checks itself out next to the website so a docs PR is validated against its own changes.
 

README.md

@@ -105,6 +105,8 @@ Nothing needs configuring for that to happen. Every build resolves the docs in t
 
 `npm run docs` runs that resolution on its own, without a full build, writing `nuxt/content/docs` and `nuxt/public/docs`. Both are generated, and neither is committed on `main`.
 
+`npm run dev` and `npm start` also watch the resolved docs and re-sync each file as it changes, so an edit appears without restarting. `npm run dev:nuxt` on its own does not include that watcher; run `npm run dev:docs` beside it if you want one.
+
 ## llms.txt
 
 `/llms.txt` (and `/llms-full.txt`) are generated by the [`nuxt-llms`](https://github.com/nuxtlabs/nuxt-llms) module, configured in `nuxt/nuxt.config.ts` under the `llms` key. Sections are built from `@nuxt/content` collections (`docs`, `blog`, `changelog`, `ebooks`, `whitepapers`) plus a small hardcoded list of standalone Nuxt routes (pricing, integrations, etc.) that aren't backed by a collection.

nuxt/lib/docs-sync.mjs

@@ -3,7 +3,7 @@
 
 import { execFileSync } from 'node:child_process'
 import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'
-import { join, relative } from 'node:path'
+import { basename, dirname, join, relative } from 'node:path'
 import { tmpdir } from 'node:os'
 
 import { processMarkdown } from './docs-markdown.mjs'
@@ -96,43 +96,103 @@ function gitOutput (cwd, args) {
     }
 }
 
-function copyDocsDir (srcDir, repoRoot, contentDir, publicDir, version) {
-    mkdirSync(contentDir, { recursive: true })
-    mkdirSync(publicDir, { recursive: true })
+/**
+ * Where one source file lands: markdown becomes a page under content/, a README becomes
+ * its section index, and anything else is an asset served from public/.
+ */
+function destinationFor (relPath, contentDocsDir, publicDocsDir) {
+    const name = basename(relPath)
+    const dir = dirname(relPath)
+    const prefix = dir === '.' ? '' : dir
+
+    return name.endsWith('.md')
+        ? join(contentDocsDir, prefix, name === 'README.md' ? 'index.md' : name)
+        : join(publicDocsDir, prefix, name)
+}
 
-    for (const entry of readdirSync(srcDir, { withFileTypes: true })) {
-        if (entry.name.startsWith('.')) continue
+/**
+ * Copy one file out of the docs tree. Split out of copyDocsDir so a dev-server edit can
+ * sync just the file that changed through exactly the same code as a full build.
+ */
+function writeDocsFile ({ docsDir, sourceRoot, contentDocsDir, publicDocsDir, relPath, version }) {
+    const srcPath = join(docsDir, relPath)
+    const destPath = destinationFor(relPath, contentDocsDir, publicDocsDir)
 
-        const srcPath = join(srcDir, entry.name)
-        const destName = entry.name === 'README.md' ? 'index.md' : entry.name
+    mkdirSync(dirname(destPath), { recursive: true })
 
-        if (entry.isDirectory()) {
-            copyDocsDir(srcPath, repoRoot, join(contentDir, entry.name), join(publicDir, entry.name), version)
-        } else if (entry.name.endsWith('.md')) {
-            const relFromRepo = relative(repoRoot, srcPath)
-            const originalPath = relative(join(repoRoot, 'docs'), srcPath)
+    if (!relPath.endsWith('.md')) {
+        cpSync(srcPath, destPath)
+        return
+    }
 
-            // Argument array, not a shell string: relFromRepo comes from filenames in the
-            // source repo, so quoting it into a shell command would be an injection path.
-            const updated = gitOutput(repoRoot, ['log', '-1', '--pretty=format:%ci', '--', relFromRepo])
+    // Argument array, not a shell string: the path comes from filenames in the source
+    // repo, so quoting it into a shell command would be an injection path.
+    const updated = gitOutput(sourceRoot, ['log', '-1', '--pretty=format:%ci', '--', relative(sourceRoot, srcPath)])
 
-            const raw = readFileSync(srcPath, 'utf8')
-            writeFileSync(join(contentDir, destName), processMarkdown(raw, originalPath, updated, version), 'utf8')
+    const raw = readFileSync(srcPath, 'utf8')
+    writeFileSync(destPath, processMarkdown(raw, relPath, updated, version), 'utf8')
+}
+
+function copyDocsDir ({ docsDir, sourceRoot, contentDocsDir, publicDocsDir, version, relDir = '' }) {
+    mkdirSync(join(contentDocsDir, relDir), { recursive: true })
+    mkdirSync(join(publicDocsDir, relDir), { recursive: true })
+
+    for (const entry of readdirSync(join(docsDir, relDir), { withFileTypes: true })) {
+        if (entry.name.startsWith('.')) continue
+
+        const relPath = join(relDir, entry.name)
+        const args = { docsDir, sourceRoot, contentDocsDir, publicDocsDir, version }
+
+        if (entry.isDirectory()) {
+            copyDocsDir({ ...args, relDir: relPath })
         } else {
-            cpSync(srcPath, join(publicDir, entry.name))
+            writeDocsFile({ ...args, relPath })
         }
     }
 }
 
-function writeDocs ({ docsDir, sourceRoot, contentDocsDir, publicDocsDir, kind, ref }) {
-    let version = ''
+function readVersion (sourceRoot) {
     try {
-        version = JSON.parse(readFileSync(join(sourceRoot, 'package.json'), 'utf8')).version || ''
-    } catch { /* not fatal */ }
+        return JSON.parse(readFileSync(join(sourceRoot, 'package.json'), 'utf8')).version || ''
+    } catch {
+        return '' // not fatal
+    }
+}
+
+/**
+ * Sync a single file from the docs tree, or remove its output if the file has gone.
+ *
+ * The dev watcher calls this once per change. `syncDocs()` rebuilds the whole tree, which
+ * is what a build wants and what a running dev server cannot survive: deleting and
+ * recreating all 130-odd pages on every save makes @nuxt/content re-index the entire
+ * collection at once and exhaust its heap.
+ */
+export function syncDocsPath ({ docsDir, nuxtRoot, relPath }) {
+    const sourceRoot = join(docsDir, '..')
+    const contentDocsDir = join(nuxtRoot, 'content', 'docs')
+    const publicDocsDir = join(nuxtRoot, 'public', 'docs')
+
+    if (!existsSync(join(docsDir, relPath))) {
+        rmSync(destinationFor(relPath, contentDocsDir, publicDocsDir), { force: true })
+        return
+    }
+
+    writeDocsFile({
+        docsDir,
+        sourceRoot,
+        contentDocsDir,
+        publicDocsDir,
+        relPath,
+        version: readVersion(sourceRoot),
+    })
+}
+
+function writeDocs ({ docsDir, sourceRoot, contentDocsDir, publicDocsDir, kind, ref }) {
+    const version = readVersion(sourceRoot)
 
     rmSync(contentDocsDir, { recursive: true, force: true })
     rmSync(publicDocsDir, { recursive: true, force: true })
-    copyDocsDir(docsDir, sourceRoot, contentDocsDir, publicDocsDir, version)
+    copyDocsDir({ docsDir, sourceRoot, contentDocsDir, publicDocsDir, version })
 
     const manifest = {
         source: kind,

nuxt/lib/docs-sync.test.mjs

@@ -1,7 +1,10 @@
 import { test } from 'node:test'
 import assert from 'node:assert/strict'
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
 
-import { resolveSource } from './docs-sync.mjs'
+import { resolveSource, syncDocsPath } from './docs-sync.mjs'
 
 const repoRoot = '/repo/website'
 
@@ -52,3 +55,92 @@ test('cloning falls back to main and honours an explicit ref', () => {
     assert.deepEqual(resolve({}), { kind: 'clone', ref: 'main' })
     assert.equal(resolve({ FLOWFUSE_DOCS_REF: 'maintenance' }).ref, 'maintenance')
 })
+
+// A flowfuse checkout next to a website checkout, as the dev watcher sees it. Outside a
+// git repo, so the `git log` that dates a page returns nothing and the output is stable.
+function fixture (t) {
+    const root = mkdtempSync(join(tmpdir(), 'docs-sync-'))
+    t.after(() => rmSync(root, { recursive: true, force: true }))
+
+    const docsDir = join(root, 'flowfuse', 'docs')
+    const nuxtRoot = join(root, 'website', 'nuxt')
+
+    mkdirSync(join(docsDir, 'cloud'), { recursive: true })
+    mkdirSync(join(nuxtRoot, 'content', 'docs'), { recursive: true })
+    mkdirSync(join(nuxtRoot, 'public', 'docs'), { recursive: true })
+    writeFileSync(join(root, 'flowfuse', 'package.json'), JSON.stringify({ version: '2.34.0' }))
+
+    return {
+        docsDir,
+        nuxtRoot,
+        content: (...parts) => join(nuxtRoot, 'content', 'docs', ...parts),
+        public: (...parts) => join(nuxtRoot, 'public', 'docs', ...parts),
+    }
+}
+
+test('an edited page is written with its provenance frontmatter', (t) => {
+    const fx = fixture(t)
+    writeFileSync(join(fx.docsDir, 'cloud', 'billing.md'), '# Billing\n')
+
+    syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'cloud/billing.md' })
+
+    const page = readFileSync(fx.content('cloud', 'billing.md'), 'utf8')
+    assert.match(page, /^---\noriginalPath: cloud\/billing\.md\n/)
+    assert.match(page, /^version: 2\.34\.0$/m)
+    assert.match(page, /# Billing/)
+})
+
+test('a README becomes the index page of its section', (t) => {
+    const fx = fixture(t)
+    writeFileSync(join(fx.docsDir, 'cloud', 'README.md'), '# Cloud\n')
+
+    syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'cloud/README.md' })
+
+    assert.ok(existsSync(fx.content('cloud', 'index.md')))
+    assert.ok(!existsSync(fx.content('cloud', 'README.md')))
+})
+
+test('a non-markdown file is copied to the public tree unchanged', (t) => {
+    const fx = fixture(t)
+    writeFileSync(join(fx.docsDir, 'cloud', 'diagram.svg'), '<svg/>')
+
+    syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'cloud/diagram.svg' })
+
+    assert.equal(readFileSync(fx.public('cloud', 'diagram.svg'), 'utf8'), '<svg/>')
+    assert.ok(!existsSync(fx.content('cloud', 'diagram.svg')))
+})
+
+test('a deleted page is removed rather than left stale', (t) => {
+    const fx = fixture(t)
+    writeFileSync(fx.content('gone.md'), 'stale\n')
+
+    syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'gone.md' })
+
+    assert.ok(!existsSync(fx.content('gone.md')))
+})
+
+test('a deleted asset is removed from the public tree', (t) => {
+    const fx = fixture(t)
+    mkdirSync(fx.public('cloud'), { recursive: true })
+    writeFileSync(fx.public('cloud', 'gone.png'), 'stale')
+
+    syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'cloud/gone.png' })
+
+    assert.ok(!existsSync(fx.public('cloud', 'gone.png')))
+})
+
+// The reason this function exists. Re-running the whole sync on every keystroke deletes and
+// recreates all 130-odd pages, and @nuxt/content's dev watcher runs out of heap re-indexing
+// them. One edit has to cost one write.
+test('syncing one page leaves every other page untouched', (t) => {
+    const fx = fixture(t)
+    writeFileSync(join(fx.docsDir, 'first.md'), '# First\n')
+    writeFileSync(join(fx.docsDir, 'second.md'), '# Second\n')
+    writeFileSync(fx.content('first.md'), 'previously synced first\n')
+    writeFileSync(fx.content('second.md'), 'previously synced second\n')
+
+    syncDocsPath({ docsDir: fx.docsDir, nuxtRoot: fx.nuxtRoot, relPath: 'first.md' })
+
+    assert.match(readFileSync(fx.content('first.md'), 'utf8'), /# First/)
+    assert.equal(readFileSync(fx.content('second.md'), 'utf8'), 'previously synced second\n')
+})

package.json

@@ -11,14 +11,15 @@
     ],
     "scripts": {
         "test": "node --test nuxt/server/lib/*.test.mjs nuxt/lib/*.test.mjs",
-        "dev": "concurrently \"npm run dev:eleventy\" \"npm run dev:postcss\" \"npm run dev:postcss-nuxt\" \"dotenv -- npm run dev --workspace=nuxt\"",
+        "dev": "concurrently \"npm run dev:eleventy\" \"npm run dev:docs\" \"npm run dev:postcss\" \"npm run dev:postcss-nuxt\" \"dotenv -- npm run dev --workspace=nuxt\"",
         "start": "npm-run-all2 clean:dev build:js blueprints --parallel dev:*",
         "build:js": "terser -c -m -o _site/js/cc.min.js node_modules/vanilla-cookieconsent/dist/cookieconsent.umd.js src/js/cookieconsent-config.js && cp node_modules/@flowfuse/flow-renderer/index.min.js _site/js/flowrenderer.min.js",
         "build": "dotenv -v NODE_ENV=production -- npm-run-all2 clean build:js --parallel prod:*",
         "build:skip-images": "dotenv -v SKIP_IMAGES=true -- npm run build",
         "clean:dev": "dotenv -- npx del-cli '_site/!(img)' 'src/blueprints/**/!(*submit.njk)' && dotenv -- npx mkdirp '_site/js/flows'",
         "clean": "dotenv -- npx del-cli '_site/!(img)' && dotenv -- mkdir -p '_site/js'",
         "dev:blueprints": "node scripts/watch_blueprints.js",
+        "dev:docs": "node scripts/watch_docs.mjs",
         "dev:netlify": "npx netlify dev -c \"dotenv -- npx @11ty/eleventy --serve --quiet --incremental\"",
         "dev:postcss": "dotenv -v TAILWIND_MODE=watch -- npx postcss ./src/css/style.css -o ./_site/css/style.css --config ./postcss.config.js -w",
         "dev:postcss-nuxt": "dotenv -v TAILWIND_MODE=watch -- npx postcss ./src/css/style.css -o ./nuxt/public/css/style.css --config ./postcss.config.js -w",
@@ -50,6 +51,7 @@
         "@tailwindcss/typography": "^0.5.14",
         "@types/markdown-it-attrs": "^4.1.3",
         "@types/markdown-it-footnote": "^3.0.4",
+        "chokidar": "^3.6.0",
         "concurrently": "^9.0.0",
         "del-cli": "^5.0.0",
         "dotenv-cli": "^7.4.4",

scripts/watch_docs.mjs

@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+// Keeps nuxt/content/docs in step with a local flowfuse checkout while the dev server runs.
+// nuxt/modules/docs-source.ts syncs once during setup and never again, so without this a
+// docs edit only shows up after a restart.
+//
+// One edit syncs one file. Re-running the whole sync instead would delete and recreate all
+// 130-odd pages on every save, and @nuxt/content re-indexing the entire collection that way
+// exhausts the dev server's heap.
+
+import { basename, dirname, join, relative } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import chokidar from 'chokidar'
+
+import { resolveSource, syncDocsPath } from '../nuxt/lib/docs-sync.mjs'
+
+const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..')
+const nuxtRoot = join(repoRoot, 'nuxt')
+
+// The same precedence the build uses, rather than a second hardcoded path that could drift
+// from it: FLOWFUSE_DOCS_LOCAL, then a sibling checkout, then a clone.
+const source = resolveSource({ repoRoot })
+
+if (source.kind === 'clone') {
+    console.log(`Docs resolve to a clone of ${source.ref}, so there is nothing local to watch`)
+    process.exit(0)
+}
+
+const { docsDir } = source
+
+const watcher = chokidar.watch(docsDir, {
+    // The Nuxt module has already synced by the time this starts.
+    ignoreInitial: true,
+    ignored: (path) => basename(path).startsWith('.'),
+    // Editors write a file in more than one step, so wait for it to settle rather than
+    // publish a half-written page.
+    awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 50 },
+    // Host file events do not cross the macOS podman VM, which is why compose.yaml sets
+    // these for every other watcher in this repo.
+    usePolling: Boolean(process.env.CHOKIDAR_USEPOLLING),
+    interval: Number(process.env.CHOKIDAR_INTERVAL) || 100,
+})
+
+// No extension allowlist: the build copies every non-markdown file into public/docs, so
+// watching everything is what keeps the dev tree matching a build.
+const verbs = { add: 'Added', change: 'Synced', unlink: 'Removed' }
+
+for (const [event, verb] of Object.entries(verbs)) {
+    watcher.on(event, (path) => {
+        const relPath = relative(docsDir, path)
+        try {
+            syncDocsPath({ docsDir, nuxtRoot, relPath })
+            console.log(`${verb} ${relPath}`)
+        } catch (err) {
+            // One bad file must not take the watcher down; the next save retries it.
+            console.error(`Could not sync ${relPath}: ${err.message}`)
+        }
+    })
+}
+
+watcher.on('ready', () => console.log(`Watching ${docsDir} for docs changes`))