From bd22a2bc0d99bd37adda32dedf7da7143ca9f3d3 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Sun, 13 Sep 2026 19:09:31 -0700 Subject: [PATCH 1/8] Detect conflicting output paths across build steps (#288 foundation) --- docs/assets/README.md | 30 +- docs/migrations/v12-migration.md | 9 + index.js | 401 ++++++++---- lib/build-copy/index.js | 30 +- lib/build-esbuild/index.js | 134 +++- lib/build-esbuild/output-conflicts.js | 70 ++ lib/build-pages/index.js | 230 ++++--- lib/build-pages/page-builders/page-writer.js | 1 + .../page-builders/template-builder.js | 18 +- lib/build-static/index.js | 27 +- lib/builder.js | 124 +++- lib/domstack-manifest/hooks.js | 14 +- lib/helpers/domstack-error.js | 2 +- lib/helpers/staged-copy.js | 61 ++ lib/output-registry.js | 174 +++++ test-cases/output-conflicts/index.test.js | 614 ++++++++++++++++++ 16 files changed, 1656 insertions(+), 283 deletions(-) create mode 100644 lib/build-esbuild/output-conflicts.js create mode 100644 lib/helpers/staged-copy.js create mode 100644 lib/output-registry.js create mode 100644 test-cases/output-conflicts/index.test.js diff --git a/docs/assets/README.md b/docs/assets/README.md index 24fb878c..f0f2d522 100644 --- a/docs/assets/README.md +++ b/docs/assets/README.md @@ -14,6 +14,31 @@ Build-tool configuration and site-wide variables are documented separately in [S [[toc]] +## Output conflicts + +Each output file must have one producer across pages, generated pages, templates, static files, copied directories, esbuild bundles, service workers, and generated metadata. +This includes page `workers.json` files, the optional `domstack-manifest.json`, and files written through a manifest hook's `writeFile` helper. +Two templates cannot emit the same path, and a single template cannot repeat an output in an array or async iterator. +Repeated identical reporting records within one batch are deduplicated; they are not additional writes. +File-versus-directory conflicts such as `feed` and `feed/index.xml` are also rejected. +Output separators and dot segments are normalized, and case aliases are checked using the destination filesystem's case behavior. + +One-shot builds claim outputs before writing them, so a conflicting second producer cannot overwrite the first. +Earlier successful build steps may remain in the destination after a later failure; the build is not rolled back. +Manifest hooks can read files they just wrote through `writeFile` from their supplied `dest`. +Watch rebuilds retain ownership for untouched producers, revalidate page outputs before promotion, and release obsolete paths after successful replacement or removal. +In a successfully started watch session, a failed conflict check retains the previous successful outputs and ownership, so fixing the source can recover without restarting watch mode. +Initial copy or esbuild failures abort startup and require starting watch again; initial page failures are logged and can recover within the session. +Full watch rebuilds replace the complete ownership map rather than accumulating historical paths. + +Page phases and full watch builds use unique stages on the destination filesystem, and copied sources are isolated before publication. +Staging requires additional disk space. +Publication is not an atomic filesystem transaction: an I/O failure during the final copy can still leave partially updated files. +The registry covers DOMStack-managed writers, not arbitrary filesystem writes performed directly by user code or esbuild plugins. +Manifest hooks should use their supplied `writeFile` helper to participate in conflict detection. +Identifiable esbuild entry collisions use the same conflict error; native plugin or shared-chunk collisions that cannot be attributed to two sources retain esbuild's diagnostic. +Concurrent builds use independent stages, but separate DOMStack instances should not publish different sites to the same destination concurrently. + ## Static assets All static assets in the `src` directory are copied 1:1 to the destination directory using [cpx2](https://github.com/bcomnes/cpx2). @@ -30,8 +55,9 @@ Place a file in a directory whose structure encodes its desired destination path To copy multiple directories, repeat the flag: `domstack --copy oldsite --copy archived-docs`. > [!WARNING] -> DOMStack does not detect conflicts between copied directories and other build output. -If multiple inputs produce the same destination path, the result is undefined. +> DOMStack rejects conflicting output paths with `DOM_STACK_ERROR_OUTPUT_CONFLICT`. +The error identifies the destination-relative path and both producers, including files from different `--copy` directories. +Rename or exclude one input instead of relying on copy order to select a winner. Copy folders must live **outside** of the `dest` directory. Copy directories can be in the src directory allowing for nested builds. diff --git a/docs/migrations/v12-migration.md b/docs/migrations/v12-migration.md index a4e26239..991accf0 100644 --- a/docs/migrations/v12-migration.md +++ b/docs/migrations/v12-migration.md @@ -20,6 +20,15 @@ Then apply the v12 changes below. --- +## Conflicting output paths now fail + +DOMStack rejects duplicate output paths across pages, generated pages, templates, esbuild, static assets, and `--copy` directories with `DOM_STACK_ERROR_OUTPUT_CONFLICT`. +The error names the destination-relative output and both producers. +Previously, copied files and templates could silently overwrite other outputs depending on write order. +Rename or exclude the conflicting input; there is no implicit last-writer-wins override. +This also applies to file-versus-directory conflicts and case aliases on case-insensitive destination filesystems. +See [Output conflicts](../assets/#output-conflicts) for staging, watch recovery, and custom-writer limitations. + ## Runtime requirements DOMStack v12 supports Node.js 22.18+ within the 22.x release line, and Node.js 24 or newer: diff --git a/index.js b/index.js index 35074735..7c0ad7e1 100644 --- a/index.js +++ b/index.js @@ -12,6 +12,8 @@ * @import { DomstackManifestRecord } from './lib/domstack-manifest/index.js' * @import { WatchDependencyState } from './lib/build-pages/watch-dependencies.js' * @import { WatchSnapshot, WatchEvent, WatchPlan } from './lib/watch-plan.js' + * @import { OutputClaim } from './lib/output-registry.js' + * @import { PageBuildStepResult } from './lib/build-pages/index.js' * @typedef {{ dispose: () => Promise }} DisposableBuildContext * @typedef {{ pageFilePath: string, sourcePageFilePath?: string | undefined, pagesFilePath?: string | undefined, layoutNames: string[], outputs?: DomstackManifestRecord[] | undefined }} WatchedPageReport * @typedef {object} WatchSession @@ -22,10 +24,11 @@ */ import { once } from 'events' import assert from 'node:assert' -import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { cp, copyFile, mkdir, mkdtemp, readFile, realpath, rm, rmdir, stat } from 'node:fs/promises' import { tmpdir } from 'node:os' +import { createHash } from 'node:crypto' import chokidar from 'chokidar' -import { basename, join, relative, resolve } from 'node:path' +import { basename, dirname, join, relative, resolve } from 'node:path' // @ts-expect-error import makeArray from 'make-array' import ignore from 'ignore' @@ -34,7 +37,7 @@ import { inspect } from 'util' import { createServer } from '@domstack/sync' import { find } from '@11ty/dependency-tree-typescript' -import { assertInsideDest } from './lib/helpers/path.js' +import { assertInsideDest, toPosix } from './lib/helpers/path.js' import { getCopyGlob } from './lib/build-static/index.js' import { getCopyDirs } from './lib/build-copy/index.js' import { isProcessedFile, globalBundleAssets, pageBundleAssets, layoutBundleAssets } from './lib/file-conventions.js' @@ -43,9 +46,9 @@ import { buildEsbuildWatch } from './lib/build-esbuild/index.js' import { buildPages } from './lib/build-pages/index.js' import { identifyPages } from './lib/identify-pages.js' import { classifyWatchEvent, planWatchEvent, planBundleChange } from './lib/watch-plan.js' -import { ensureDest } from './lib/helpers/ensure-dest.js' import { DomStackAggregateError } from './lib/helpers/domstack-aggregate-error.js' import { createDomStackLogger } from './lib/logger.js' +import { OutputRegistry, isCaseInsensitiveDest } from './lib/output-registry.js' export { PageData } from './lib/build-pages/page-data.js' export { @@ -75,6 +78,8 @@ export class DomStack { /** @type {Readonly} */ opts /** @type {FSWatcher?} */ #watcher = null /** @type {ReturnType[]} */ #cpxWatchers = [] + /** @type {string[]} */ #cpxWatchStages = [] + /** @type {Map Promise>} */ #pendingCopyUpdates = new Map() /** @type {BsInstance?} */ #syncServer = null /** @type {DisposableBuildContext?} */ #esbuildContext = null /** @type {SiteData?} */ #siteData = null @@ -101,16 +106,19 @@ export class DomStack { #globalDataDepPaths = new Set() /** @type {Set} absolute filepaths of esbuild entry points */ #esbuildEntryPoints = new Set() - /** @type {Set} destination-relative outputs from the last successful page builds */ - #pageOutputRelnames = new Set() - /** @type {Map>} *.pages.* filepath → owned destination-relative outputs */ - #pagesFileOutputMap = new Map() + /** @type {Map>} *.pages.* filepath → layouts used by its generated pages */ #pagesFileLayoutMap = new Map() /** @type {WatchDependencyState | null} subscriptions and fingerprints from the last successful page build */ #watchDependencies = null /** @type {boolean} Failed builds may leave the previous routing state incomplete. */ #pageBuildFailed = false + /** @type {OutputClaim[]} Output ownership from the latest successful watch build. */ + #outputClaims = [] + #outputLock = Promise.resolve() + #caseInsensitive = false + /** @type {string | null} Unpublished initial watch outputs, retained for recovery. */ + #initialStage = null // One session owns the resources above until shutdown finishes. // Normal path: absent → starting → watching → stopping → absent. @@ -231,6 +239,8 @@ export class DomStack { } session.state = 'watching' + for (const update of this.#pendingCopyUpdates.values()) this.#enqueueBuild(session, update) + this.#pendingCopyUpdates.clear() const enqueue = (/** @type {() => Promise} */ fn) => { this.#enqueueBuild(session, fn) } @@ -260,12 +270,28 @@ export class DomStack { throw new DomStackAggregateError(siteData.errors, 'Page walk finished but there were errors.', siteData) } - await ensureDest(this.#dest, siteData) + await mkdir(this.#dest, { recursive: true }) + this.#initialStage = await mkdtemp(join(resolve(this.#dest), '.domstack-stage-')) + + this.#caseInsensitive = await isCaseInsensitiveDest(this.#dest) + // The watchers' initial inventories are the only copy scan at startup. + const copyDirs = getCopyDirs(this.opts.copy ?? []) + const copyStartup = await Promise.allSettled([ + ...(this.opts.static === false ? [] : [this.#startCopyWatcher(getCopyGlob(this.#src), signal, 'static', this.opts.ignore ?? [])]), + ...copyDirs.map((copyDir, index) => this.#startCopyWatcher(copyDir, signal, 'copy', [], `copy-root:${index}:`)), + ]) + const copyErrors = copyStartup.filter(result => result.status === 'rejected').map(result => result.reason) + if (copyErrors.length) throw new AggregateError(copyErrors, 'Copy watch startup failed') + await this.#drainPendingCopyUpdates() // Start esbuild in watch mode (stable filenames, no hash) let esbuildContext try { - const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { logger: this.#logger }) + const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { + logger: this.#logger, + writeDest: () => this.#initialStage ?? this.#dest, + promoteOutputs: (outputs, phase, write) => this.#promoteEsbuildOutputs(outputs, phase, write), + }) esbuildContext = context } catch (err) { throw new Error('Error starting esbuild watch context', { cause: err }) @@ -273,13 +299,18 @@ export class DomStack { this.#esbuildContext = esbuildContext this.#siteData = siteData + await this.#drainPendingCopyUpdates() // Build pages (initial full build) let report try { - const pageBuildResults = await buildPages(this.#src, this.#dest, siteData, { + const pageBuildResults = await buildPages(this.#src, this.#initialStage ?? this.#dest, siteData, { ...this.opts, trackWatchDependencies: true, + previousOutputClaims: this.#outputClaims, + caseInsensitive: this.#caseInsensitive, + promoteOutputs: (report, write) => this.#promotePageOutputs(report, write), }) + this.#remapInitialPageReport(pageBuildResults) if (pageBuildResults.errors.length > 0) { throw new DomStackAggregateError(pageBuildResults.errors, 'Page build finished but there were errors.', { siteData, @@ -291,12 +322,13 @@ export class DomStack { siteData, pageBuildResults, } - this.#pageOutputRelnames = getPageOutputRelnames(pageBuildResults.outputs) - this.#pagesFileOutputMap = getPagesFileOutputMap(pageBuildResults.report.pages) + this.#pagesFileLayoutMap = getPagesFileLayoutMap(pageBuildResults.report.pages) this.#updatePageLayoutNames(pageBuildResults.report.pages, true) this.#pageBuildFailed = false this.#watchDependencies = pageBuildResults.report.watchDependencies ?? null + delete pageBuildResults.report.newClaims + delete pageBuildResults.report.replacedOwnerIds delete pageBuildResults.report.watchDependencies delete pageBuildResults.report.rebuiltPagesFilePaths buildLogger(report, this.#logger) @@ -312,13 +344,9 @@ export class DomStack { await this.#rebuildMaps(siteData) // Copy readiness is cancellable: cpx2 invalidates pending scans on close. - const copyDirs = getCopyDirs(this.opts.copy ?? []) - const copyStartup = await Promise.allSettled([ - this.#startCopyWatcher(getCopyGlob(this.#src), signal, this.opts.ignore ?? []), - ...copyDirs.map(copyDir => this.#startCopyWatcher(copyDir, signal)), - ]) - const copyErrors = copyStartup.filter(result => result.status === 'rejected').map(result => result.reason) - if (copyErrors.length) throw new AggregateError(copyErrors, 'Copy watch startup failed') + await this.#drainPendingCopyUpdates() + if (!signal.aborted && !this.#pageBuildFailed) await this.#publishInitialStage() + await this.#drainPendingCopyUpdates() // ── Chokidar watcher ───────────────────────────────────────────────── const ig = ignore().add(this.opts.ignore ?? []) @@ -359,20 +387,57 @@ export class DomStack { /** * @param {string} source * @param {AbortSignal} signal + * @param {'static' | 'copy'} kind * @param {string[]} [ignores] + * @param {string} [ownerPrefix] - Matches buildCopy's configured root identity. */ - async #startCopyWatcher (source, signal, ignores = []) { - const watcher = cpxWatch(source, this.#dest, { ignore: ignores }) + async #startCopyWatcher (source, signal, kind, ignores = [], ownerPrefix = '') { + const stageDest = await mkdtemp(join(this.#dest, '.domstack-copy-watch-')) + this.#cpxWatchStages.push(stageDest) + const watcher = cpxWatch(source, stageDest, { ignore: ignores }) this.#cpxWatchers.push(watcher) + // Isolate each source before cpx writes, including case and file/dir aliases. + // Retain cpx's logical mapping; only its private physical destination changes. + const toDestination = watcher.toDestination + const logicalOutputs = new Map() + watcher.toDestination = sourcePath => { + const path = join(stageDest, createHash('sha256').update(sourcePath).digest('hex')) + logicalOutputs.set(path, toPosix(relative(stageDest, toDestination(sourcePath)))) + return path + } + const ownerByOutput = new Map() let ready = false let initialCopies = 0 watcher.on('copy', (/** @type{{ srcPath: string, dstPath: string }} */e) => { + const outputRelname = logicalOutputs.get(e.dstPath) + const sourceRelname = toPosix(relative(this.#src, resolve(e.srcPath))) + const owner = { id: `${ownerPrefix}${kind}:${sourceRelname}`, type: kind, path: sourceRelname } + ownerByOutput.set(e.dstPath, owner) if (!ready) initialCopies++ this.#logger.debug(`Copy ${e.srcPath} to ${e.dstPath}`) - if (ready) this.#logger.info(`Static asset updated: ${e.srcPath}`) + if (this.#watchSession) { + const update = async () => { + try { + await stat(e.srcPath) + await this.#promoteCopyOutput(e.dstPath, outputRelname, owner) + } catch (error) { + if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error + await this.#removeCopyOutput(outputRelname, owner) + } + } + if (!ready || this.#watchSession.state === 'starting') this.#pendingCopyUpdates.set(e.dstPath, update) + else this.#enqueueBuild(this.#watchSession, update) + } }) watcher.on('remove', (/** @type{{ path: string }} */e) => { - this.#logger.info(`Remove ${e.path}`) + const outputRelname = logicalOutputs.get(e.path) + const owner = ownerByOutput.get(e.path) + ownerByOutput.delete(e.path) + if (owner && this.#watchSession) { + const update = () => this.#removeCopyOutput(outputRelname, owner) + if (!ready || this.#watchSession.state === 'starting') this.#pendingCopyUpdates.set(e.path, update) + else this.#enqueueBuild(this.#watchSession, update) + } }) watcher.on('watch-error', (/** @type{Error} */err) => { this.#logger.error(`Copy error: ${err.message}`) @@ -381,23 +446,133 @@ export class DomStack { // cpx2 reports startup failure as "watch-error", not EventEmitter's "error". // A closed session may never emit readiness, so cancellation must also settle // this wait. This does not drain file operations already started by cpx2. - const { promise, resolve, reject } = Promise.withResolvers() - const onAbort = () => resolve(undefined) - watcher.once('watch-ready', resolve) + const { promise, resolve: resolveReady, reject } = Promise.withResolvers() + const onAbort = () => resolveReady(undefined) + watcher.once('watch-ready', resolveReady) watcher.once('watch-error', reject) signal.addEventListener('abort', onAbort, { once: true }) try { if (signal.aborted) return await promise + if (signal.aborted) return ready = true - if (!signal.aborted) this.#logger.info(`Static asset watcher ready (${initialCopies} initial copy operations)`) + this.#logger.info(`Static asset watcher ready (${initialCopies} initial copy operations)`) } finally { - watcher.off('watch-ready', resolve) + watcher.off('watch-ready', resolveReady) watcher.off('watch-error', reject) signal.removeEventListener('abort', onAbort) } } + async #drainPendingCopyUpdates () { + while (this.#pendingCopyUpdates.size > 0) { + const updates = [...this.#pendingCopyUpdates.values()] + this.#pendingCopyUpdates.clear() + for (const update of updates) await update() + } + } + + /** + * Serialize ownership checks, promotion, and commits independently of the + * watch build queue: esbuild callbacks also run during queued context startup. + * @param {() => OutputClaim[]} nextClaims + * @param {() => Promise} write + */ + #commitOutputs (nextClaims, write) { + const transaction = this.#outputLock.then(async () => { + const next = nextClaims() + const paths = new Set(next.map(claim => claim.outputRelname)) + for (const claim of this.#outputClaims) { + if (paths.has(claim.outputRelname)) continue + const writeDest = this.#initialStage ?? this.#dest + const target = resolve(writeDest, claim.outputRelname) + assertInsideDest(writeDest, target) + await rm(target, { force: true }) + // Only remove empty directories; never recursively delete unowned files. + for (let dir = dirname(target); dir !== resolve(writeDest); dir = dirname(dir)) { + try { await rmdir(dir) } catch { break } + } + } + await write() + this.#outputClaims = next + }) + this.#outputLock = transaction.catch(() => {}) + return transaction + } + + /** Publish initial watch outputs only once every initial producer succeeds. */ + async #publishInitialStage () { + const publish = this.#outputLock.then(async () => { + if (!this.#initialStage) return + await mkdir(this.#dest, { recursive: true }) + await cp(this.#initialStage, await realpath(this.#dest), { recursive: true, force: true }) + await rm(this.#initialStage, { recursive: true, force: true }) + this.#initialStage = null + }) + this.#outputLock = publish.catch(() => {}) + await publish + } + + /** @param {PageBuildStepResult} result */ + #remapInitialPageReport (result) { + if (!this.#initialStage) return + for (const output of result.outputs) output.filepath = resolve(this.#dest, output.outputRelname) + for (const page of result.report.pages) page.pageFilePath = resolve(this.#dest, relative(this.#initialStage, page.pageFilePath)) + } + + /** + * @param {DomstackManifestRecord[]} outputs + * @param {'browser' | 'service-worker'} phase + * @param {() => Promise} write + */ + #promoteEsbuildOutputs (outputs, phase, write) { + return this.#commitOutputs(() => replaceEsbuildClaims(this.#outputClaims, outputs, phase, this.#caseInsensitive), write) + } + + /** + * Revalidate the worker's selected producers against current ownership, not + * its possibly stale pre-render snapshot. + * @param {PageBuildStepResult} report + * @param {() => Promise} write + */ + #promotePageOutputs (report, write) { + return this.#commitOutputs(() => { + const replaced = new Set(report.report.replacedOwnerIds ?? []) + const registry = new OutputRegistry(this.#outputClaims, { replaceOwnerIds: replaced, caseInsensitive: this.#caseInsensitive }) + for (const claim of report.report.newClaims ?? []) registry.claim(claim.outputRelname, claim.owner) + return registry.snapshot() + }, write) + } + + /** + * @param {string} stagedPath + * @param {string} outputRelname + * @param {{ id: string, type: string, path: string }} owner + */ + async #promoteCopyOutput (stagedPath, outputRelname, owner) { + await this.#commitOutputs(() => { + const registry = new OutputRegistry(this.#outputClaims, { replaceOwnerIds: [owner.id], caseInsensitive: this.#caseInsensitive }) + registry.claim(outputRelname, owner) + return registry.snapshot() + }, async () => { + const writeDest = this.#initialStage ?? this.#dest + const target = resolve(writeDest, outputRelname) + assertInsideDest(writeDest, target) + await mkdir(dirname(target), { recursive: true }) + await copyFile(stagedPath, target) + }) + this.#logger.info(`Static asset updated: ${owner.path}`) + } + + /** + * @param {string} outputRelname + * @param {{ id: string, type: string, path: string }} owner + */ + async #removeCopyOutput (outputRelname, owner) { + await this.#commitOutputs(() => this.#outputClaims.filter(claim => claim.owner.id !== owner.id || claim.outputRelname !== outputRelname), async () => {}) + this.#logger.info(`Remove ${outputRelname}`) + } + async #startWatchServer () { this.#syncServer = await createServer({ server: this.#dest, @@ -419,22 +594,35 @@ export class DomStack { this.#esbuildContext = null } - const siteData = await identifyPages(this.#src, this.opts) - - if (siteData.errors.length > 0) { - this.#logger.error(`identifyPages errors: -${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) - return + try { + const results = await builder(this.#src, this.#dest, { ...this.opts, domstackManifest: false }, { + watch: true, + caseInsensitive: this.#caseInsensitive, + promoteOutputs: (claims, write) => this.#commitOutputs(() => claims, write), + }) + if (this.#initialStage) { + await rm(this.#initialStage, { recursive: true, force: true }) + this.#initialStage = null + } + const { siteData, pageBuildResults } = results + this.#siteData = siteData + if (pageBuildResults) { + this.#updatePageLayoutNames(pageBuildResults.report.pages, true) + this.#pagesFileLayoutMap = getPagesFileLayoutMap(pageBuildResults.report.pages) + this.#watchDependencies = pageBuildResults.report.watchDependencies ?? null + } + this.#pageBuildFailed = false + const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { + logger: this.#logger, + promoteOutputs: (outputs, phase, write) => this.#promoteEsbuildOutputs(outputs, phase, write), + }) + this.#esbuildContext = context + await this.#rebuildMaps(siteData) + buildLogger(results, this.#logger) + } catch (error) { + this.#pageBuildFailed = true + throw error } - - await ensureDest(this.#dest, siteData) - - const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { logger: this.#logger }) - this.#esbuildContext = context - this.#siteData = siteData - - await this.#runPageBuild(siteData) - await this.#rebuildMaps(siteData) } /** @returns {WatchSnapshot | undefined} */ @@ -463,6 +651,10 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) async #handleWatchEvent (changedPath, type) { const snapshot = this.#watchSnapshot() if (!snapshot) return + if (!this.#esbuildContext) { + await this.#fullRebuild() + return + } const event = classifyWatchEvent(type, changedPath) await this.#executeWatchPlan(planWatchEvent(snapshot, event), event) } @@ -498,12 +690,16 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) this.#logger.error(`identifyPages errors:\n${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) return } - await ensureDest(this.#dest, siteData) + await mkdir(this.#dest, { recursive: true }) if (this.#esbuildContext) { await this.#esbuildContext.dispose() this.#esbuildContext = null } - const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { logger: this.#logger }) + const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { + logger: this.#logger, + writeDest: () => this.#initialStage ?? this.#dest, + promoteOutputs: (outputs, phase, write) => this.#promoteEsbuildOutputs(outputs, phase, write), + }) this.#esbuildContext = context this.#siteData = siteData const snapshot = this.#watchSnapshot() @@ -527,32 +723,37 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) // Retry the complete page phase after a failure: neither subscriptions nor // layout routing from a failed build can safely drive an incremental retry. if (this.#pageBuildFailed) pageFilterPaths = templateFilterPaths = pagesFileFilterPaths = null + try { - const pageBuildResults = await buildPages(this.#src, this.#dest, siteData, { + const pageBuildResults = await buildPages(this.#src, this.#initialStage ?? this.#dest, siteData, { ...this.opts, ...(pageFilterPaths ? { pageFilterPaths } : {}), ...(templateFilterPaths ? { templateFilterPaths } : {}), ...(pagesFileFilterPaths ? { pagesFileFilterPaths } : {}), previousWatchDependencies: this.#watchDependencies, trackWatchDependencies: true, + previousOutputClaims: this.#outputClaims, + caseInsensitive: this.#caseInsensitive, + promoteOutputs: (report, write) => this.#promotePageOutputs(report, write), }) + this.#remapInitialPageReport(pageBuildResults) if (pageBuildResults.errors.length > 0) { throw new DomStackAggregateError(pageBuildResults.errors, 'Page build finished but there were errors.', { siteData, pageBuildResults, }) } + await this.#publishInitialStage() const isFiltered = pageFilterPaths !== null || templateFilterPaths !== null || pagesFileFilterPaths !== null this.#updatePageLayoutNames(pageBuildResults.report.pages, !isFiltered) if (!isFiltered) { - await this.#removeObsoletePageOutputs(pageBuildResults.outputs) - this.#pagesFileOutputMap = getPagesFileOutputMap(pageBuildResults.report.pages) this.#pagesFileLayoutMap = getPagesFileLayoutMap(pageBuildResults.report.pages) } else if ((pageBuildResults.report.rebuiltPagesFilePaths?.length ?? 0) > 0) { - await this.#removeObsoleteGeneratedPageOutputs(pageBuildResults.report.rebuiltPagesFilePaths ?? [], pageBuildResults.report.pages) updatePagesFileLayoutMap(this.#pagesFileLayoutMap, pageBuildResults.report.rebuiltPagesFilePaths ?? [], pageBuildResults.report.pages) } this.#watchDependencies = pageBuildResults.report.watchDependencies ?? this.#watchDependencies + delete pageBuildResults.report.newClaims + delete pageBuildResults.report.replacedOwnerIds delete pageBuildResults.report.watchDependencies delete pageBuildResults.report.rebuiltPagesFilePaths await this.#rebuildMaps(siteData) @@ -569,64 +770,6 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) } } - /** - * Remove page files that were emitted by the previous successful full build - * but are no longer claimed by the current page or template build. - * - * @param {DomstackManifestRecord[]} outputs - */ - async #removeObsoletePageOutputs (outputs) { - const currentOutputRelnames = new Set(outputs.map(output => output.outputRelname)) - const currentPageOutputRelnames = getPageOutputRelnames(outputs) - const dest = resolve(this.#dest) - - await Promise.all(Array.from(this.#pageOutputRelnames, async outputRelname => { - if (currentOutputRelnames.has(outputRelname)) return - const filepath = resolve(dest, outputRelname) - assertInsideDest(dest, filepath) - if (filepath === dest) throw new Error('Refusing to remove the build destination') - await rm(filepath, { force: true }) - })) - - this.#pageOutputRelnames = currentPageOutputRelnames - } - - /** - * Remove outputs no longer emitted by the selected generated-pages owners. - * Ownership state changes only after a successful targeted build. - * - * @param {string[]} pagesFileFilterPaths - * @param {WatchedPageReport[]} pageReports - */ - async #removeObsoleteGeneratedPageOutputs (pagesFileFilterPaths, pageReports) { - const currentByOwner = getPagesFileOutputMap(pageReports) - const dest = resolve(this.#dest) - - for (const pagesFilePath of pagesFileFilterPaths) { - const previousOutputs = this.#pagesFileOutputMap.get(pagesFilePath) ?? new Set() - const currentOutputs = currentByOwner.get(pagesFilePath) ?? new Set() - - await Promise.all(Array.from(previousOutputs, async outputRelname => { - if (currentOutputs.has(outputRelname)) return - const filepath = resolve(dest, outputRelname) - assertInsideDest(dest, filepath) - if (filepath === dest) throw new Error('Refusing to remove the build destination') - await rm(filepath, { force: true }) - })) - - for (const outputRelname of previousOutputs) this.#pageOutputRelnames.delete(outputRelname) - for (const report of pageReports) { - if (report.pagesFilePath !== pagesFilePath) continue - for (const output of report.outputs ?? []) { - if (output.kind === 'page') this.#pageOutputRelnames.add(output.outputRelname) - } - } - - if (currentOutputs.size > 0) this.#pagesFileOutputMap.set(pagesFilePath, currentOutputs) - else this.#pagesFileOutputMap.delete(pagesFilePath) - } - } - /** * @param {WatchSession} session * @param {() => Promise} fn @@ -825,11 +968,17 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) Promise.resolve().then(() => this.#esbuildContext?.dispose()), Promise.resolve().then(() => this.#syncServer?.exit()), ])) + await this.#outputLock + results.push(...await Promise.allSettled([...this.#cpxWatchStages, ...(this.#initialStage ? [this.#initialStage] : [])].map(stage => rm(stage, { recursive: true, force: true })))) + this.#initialStage = null this.#watcher = null this.#cpxWatchers = [] + this.#cpxWatchStages = [] + this.#pendingCopyUpdates.clear() this.#esbuildContext = null this.#syncServer = null this.#siteData = null + this.#outputClaims = [] this.#buildLock = Promise.resolve() this.#watchSession = null const errors = results.filter(result => result.status === 'rejected').map(result => result.reason) @@ -842,37 +991,23 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) */ async settled () { await this.#buildLock + await this.#outputLock } } /** + * @param {OutputClaim[]} claims * @param {DomstackManifestRecord[]} outputs - * @returns {Set} + * @param {'browser' | 'service-worker'} phase + * @param {boolean} caseInsensitive + * @returns {OutputClaim[]} */ -function getPageOutputRelnames (outputs) { - return new Set(outputs - .filter(output => output.kind === 'page') - .map(output => output.outputRelname)) -} - -/** - * Group generated-page outputs by their owning *.pages.* filepath. - * - * @param {WatchedPageReport[]} pageReports - * @returns {Map>} - */ -function getPagesFileOutputMap (pageReports) { - /** @type {Map>} */ - const outputsByOwner = new Map() - - for (const report of pageReports) { - if (!report.pagesFilePath) continue - const outputs = outputsByOwner.get(report.pagesFilePath) ?? new Set() - for (const output of report.outputs ?? []) outputs.add(output.outputRelname) - outputsByOwner.set(report.pagesFilePath, outputs) - } - - return outputsByOwner +function replaceEsbuildClaims (claims, outputs, phase, caseInsensitive) { + const prefix = `esbuild:${phase}:` + const replaceOwnerIds = claims.filter(claim => claim.owner.id.startsWith(prefix)).map(claim => claim.owner.id) + const registry = new OutputRegistry(claims, { replaceOwnerIds, caseInsensitive }) + registry.claimRecords(outputs, prefix) + return registry.snapshot() } /** diff --git a/lib/build-copy/index.js b/lib/build-copy/index.js index a213eca0..874f1175 100644 --- a/lib/build-copy/index.js +++ b/lib/build-copy/index.js @@ -1,10 +1,11 @@ /** - * @import { BuildStepResult, BuildStep } from '../builder.js' + * @import { BuildStepResult, BuildStep, DomStackOpts } from '../builder.js' */ -import { copy } from 'cpx2' +/** @import { copy } from 'cpx2' */ import { join } from 'node:path' -import { createCopiedDomstackManifestRecords } from '../helpers/cpx2-report.js' +import { stagedCopy } from '../helpers/staged-copy.js' +import { OutputRegistry } from '../output-registry.js' /** * @typedef {Record>>} CopyBuilderReport @@ -24,9 +25,13 @@ export function getCopyDirs (copy = []) { /** * run CPX2 on src folder * - * @type {CopyBuildStep} + * @param {string} src + * @param {string} dest + * @param {unknown} _siteData + * @param {DomStackOpts | null} [opts] + * @param {OutputRegistry} [registry] */ -export async function buildCopy (src, dest, _siteData, opts) { +export async function buildCopy (src, dest, _siteData, opts, registry = new OutputRegistry()) { /** @type {CopyBuildStepResult} */ const results = { type: 'copy', @@ -38,8 +43,10 @@ export async function buildCopy (src, dest, _siteData, opts) { const copyDirs = getCopyDirs(opts?.copy) - const copyTasks = copyDirs.map((copyDir) => { - return copy(copyDir, dest) + // Each configured root is a producer, even when roots overlap or repeat. + // Keep this prefix identical to the live watch inventory's mapping identity. + const copyTasks = copyDirs.map((copyDir, index) => { + return stagedCopy(copyDir, src, dest, 'copy', registry, [], `copy-root:${index}:`) }) const settled = await Promise.allSettled(copyTasks) @@ -51,13 +58,8 @@ export async function buildCopy (src, dest, _siteData, opts) { } else { const copyDir = copyDirs[index] if (!copyDir) continue - results.report[copyDir] = result.value - results.outputs.push(...createCopiedDomstackManifestRecords({ - src, - dest, - report: result.value, - kind: 'copy', - })) + results.report[copyDir] = result.value.report + results.outputs.push(...result.value.outputs) } } return results diff --git a/lib/build-esbuild/index.js b/lib/build-esbuild/index.js index 7dacd392..c705f9ef 100644 --- a/lib/build-esbuild/index.js +++ b/lib/build-esbuild/index.js @@ -4,9 +4,11 @@ * @import { Logger as PinoLogger } from 'pino' */ -import { writeFile } from 'fs/promises' -import { join, relative, basename, resolve, extname } from 'path' +import { mkdir, writeFile } from 'fs/promises' +import { join, relative, basename, dirname, resolve, extname } from 'path' import esbuild from 'esbuild' +import { OutputRegistry } from '../output-registry.js' +import { rethrowEsbuildOutputConflict, validateEsbuildEntryOutputs } from './output-conflicts.js' import { globalBundleAssets, pageBundleAssets, layoutBundleAssets } from '../file-conventions.js' import { resolveVars } from '../build-pages/resolve-vars.js' import { @@ -215,7 +217,7 @@ async function createBrowserBuildOpts (src, dest, siteData, opts, modeOpts = {}) /** @type {EsbuildLogLevel} */ logLevel: 'silent', bundle: true, - write: true, + write: false, /** @type {EsbuildFormat} */ format: 'esm', splitting: true, @@ -261,6 +263,10 @@ async function createBrowserBuildOpts (src, dest, siteData, opts, modeOpts = {}) return { ...extendedBuildOpts, + // DomStack must own the write boundary and inventory even when settings + // override these options. opts.metafile only controls publishing the JSON. + write: false, + metafile: true, define: preserveDomstackDefines(extendedBuildOpts.define, domstackDefines), } } @@ -287,6 +293,19 @@ function preserveDomstackDefines (define, domstackDefines) { return mergedDefine } +/** + * @param {esbuild.BuildResult} result + * @param {string} [dest] + * @param {string} [writeDest] + */ +async function writeEsbuildOutputFiles (result, dest, writeDest) { + for (const output of result.outputFiles ?? []) { + const target = dest && writeDest ? resolve(writeDest, relative(dest, output.path)) : output.path + await mkdir(dirname(target), { recursive: true }) + await writeFile(target, output.contents) + } +} + /** * @param {object} params * @param {string} params.dest @@ -323,23 +342,33 @@ function emptyEsbuildReport () { /** * Build all of the bundles using esbuild. * - * @type {EsBuildStep} + * @param {string} src + * @param {string} dest + * @param {SiteData} siteData + * @param {DomStackOpts | null} opts + * @param {OutputRegistry} [registry] + * @param {boolean} [watch] + * @param {string} [publicDest] - Compile paths relative to the published destination, even when staging writes. + * @returns {Promise} */ -export async function buildEsbuild (src, dest, siteData, opts) { +export async function buildEsbuild (src, dest, siteData, opts, registry = new OutputRegistry(), watch = false, publicDest = dest) { try { - const extendedBuildOpts = await createBrowserBuildOpts(src, dest, siteData, opts, { watch: false }) + const extendedBuildOpts = await createBrowserBuildOpts(src, publicDest, siteData, opts, { watch }) - const buildResults = await esbuild.build(extendedBuildOpts) + const buildResults = await buildControlled(extendedBuildOpts) - await writeMetafile({ dest, result: buildResults, shouldWrite: opts?.metafile !== false }) - const outputMap = applyBuildOutputMap({ dest, result: buildResults, siteData, src }) + const outputMap = applyBuildOutputMap({ dest: publicDest, result: buildResults, siteData, src }) const outputs = createEsbuildOutputRecords({ src, - dest, + dest: publicDest, siteData, buildResults, includeMetafileRecord: opts?.metafile !== false, }) + registry.claimRecords(outputs, 'esbuild:browser:') + for (const output of outputs) output.filepath = resolve(dest, output.outputRelname) + await writeEsbuildOutputFiles(buildResults, publicDest, dest) + await writeMetafile({ dest, result: buildResults, shouldWrite: opts?.metafile !== false }) return { type: 'esbuild', @@ -376,9 +405,10 @@ export async function buildEsbuild (src, dest, siteData, opts) { * @param {SiteData} siteData * @param {EsbuildBuildOptions | undefined} browserBuildOpts * @param {ServiceWorkerBuildDefines} [defines] + * @param {OutputRegistry} [registry] * @returns {Promise} */ -export async function buildServiceWorkerEsbuild (src, dest, siteData, browserBuildOpts, defines = {}) { +export async function buildServiceWorkerEsbuild (src, dest, siteData, browserBuildOpts, defines = {}, registry = new OutputRegistry()) { if (!siteData.serviceWorker) { return { type: 'esbuild', @@ -399,14 +429,26 @@ export async function buildServiceWorkerEsbuild (src, dest, siteData, browserBui serviceWorker: siteData.serviceWorker, src, }) - const serviceWorkerBuildResults = await esbuild.build(serviceWorkerBuildOpts) - const outputMap = applyBuildOutputMap({ dest, result: serviceWorkerBuildResults, siteData, src }) + const serviceWorkerBuildResults = await buildControlled(serviceWorkerBuildOpts) + + const publicDest = serviceWorkerBuildOpts.outdir ?? dest + const outputMap = applyBuildOutputMap({ dest: publicDest, result: serviceWorkerBuildResults, siteData, src }) + const outputs = createEsbuildOutputRecords({ + src, + dest: publicDest, + siteData, + buildResults: serviceWorkerBuildResults, + includeMetafileRecord: false, + }) + registry.claimRecords(outputs, 'esbuild:service-worker:') + for (const output of outputs) output.filepath = resolve(dest, output.outputRelname) + await writeEsbuildOutputFiles(serviceWorkerBuildResults, publicDest, dest) return { type: 'esbuild', errors: serviceWorkerBuildResults.errors, warnings: serviceWorkerBuildResults.warnings, - outputs: [], + outputs, report: { buildResults: serviceWorkerBuildResults, buildOpts: serviceWorkerBuildOpts, @@ -503,8 +545,8 @@ function createDomstackDefines ({ opts, siteData, watch }) { * @param {string} dest * @param {SiteData} siteData * @param {DomStackOpts} opts - * @param {{ onEnd?: (result: esbuild.BuildResult) => void, logger?: PinoLogger }} [watchOpts] - * @returns {Promise<{ context: DisposableBuildContext, outputMap: OutputMap, buildResults: esbuild.BuildResult, buildOpts: EsbuildBuildOptions }>} + * @param {{ onEnd?: (result: esbuild.BuildResult) => void, logger?: PinoLogger, writeDest?: () => string, promoteOutputs?: (outputs: DomstackManifestRecord[], phase: 'browser' | 'service-worker', write: () => Promise) => Promise }} [watchOpts] + * @returns {Promise<{ context: DisposableBuildContext, outputMap: OutputMap, outputs: DomstackManifestRecord[], buildResults: esbuild.BuildResult, buildOpts: EsbuildBuildOptions }>} */ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = {}) { const logger = watchOpts.logger ?? opts.logger ?? createDomStackLogger() @@ -514,8 +556,12 @@ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = dest, label: 'JS/CSS', logger, - onEnd: watchOpts.onEnd, + ...(watchOpts.onEnd ? { onEnd: watchOpts.onEnd } : {}), + promote: (result, write) => watchOpts.promoteOutputs + ? watchOpts.promoteOutputs(createEsbuildOutputRecords({ src, dest, siteData, buildResults: result, includeMetafileRecord: opts?.metafile !== false }), 'browser', write) + : write(), shouldWriteMetafile: opts?.metafile !== false, + ...(watchOpts.writeDest ? { writeDest: watchOpts.writeDest } : {}), }) const initialResult = browserWatch.initialResult @@ -524,6 +570,13 @@ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = const contexts = [browserWatch.context] try { const outputMap = applyBuildOutputMap({ dest, result: initialResult, siteData, src }) + const outputs = createEsbuildOutputRecords({ + src, + dest, + siteData, + buildResults: initialResult, + includeMetafileRecord: opts?.metafile !== false, + }) if (siteData.serviceWorker) { // Keep service-worker-only defines and no-policy watch cleanup behavior out of browser bundles. @@ -538,7 +591,11 @@ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = dest, label: 'Service worker', logger, + promote: (result, write) => watchOpts.promoteOutputs + ? watchOpts.promoteOutputs(createEsbuildOutputRecords({ src, dest, siteData, buildResults: result, includeMetafileRecord: false }), 'service-worker', write) + : write(), shouldWriteMetafile: false, + ...(watchOpts.writeDest ? { writeDest: watchOpts.writeDest } : {}), }) contexts.push(serviceWorkerWatch.context) applyBuildOutputMap({ @@ -547,11 +604,20 @@ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = siteData, src, }) + outputs.push(...createEsbuildOutputRecords({ + src, + dest, + siteData, + buildResults: serviceWorkerWatch.initialResult, + includeMetafileRecord: false, + })) } + if (!siteData.serviceWorker) await watchOpts.promoteOutputs?.([], 'service-worker', async () => {}) return { context: createDisposableBuildContext(contexts), outputMap, + outputs, buildResults: initialResult, buildOpts: extendedBuildOpts, } @@ -570,10 +636,13 @@ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = * @param {string} params.label * @param {PinoLogger} params.logger * @param {(result: esbuild.BuildResult) => void | Promise} [params.onEnd] + * @param {(result: esbuild.BuildResult, write: () => Promise) => Promise} [params.promote] * @param {boolean} params.shouldWriteMetafile + * @param {() => string} [params.writeDest] * @returns {Promise<{ context: esbuild.BuildContext, initialResult: esbuild.BuildResult }>} */ -async function createWatchBuild ({ buildOpts, dest, label, logger, onEnd, shouldWriteMetafile }) { +async function createWatchBuild ({ buildOpts, dest, label, logger, onEnd, promote, shouldWriteMetafile, writeDest }) { + validateEsbuildEntryOutputs(buildOpts) const initial = Promise.withResolvers() // Attach a rejection handler before watch() can deliver a failing initial build. initial.promise.catch(() => {}) @@ -589,6 +658,7 @@ async function createWatchBuild ({ buildOpts, dest, label, logger, onEnd, should isInitialBuild = false try { if (result.errors.length > 0) { + rethrowEsbuildOutputConflict(result.errors, buildOpts) const failure = Object.assign(new Error(`${label} build failed`), { errors: result.errors.map(serializeEsbuildMessage), warnings: result.warnings.map(serializeEsbuildMessage), @@ -602,7 +672,13 @@ async function createWatchBuild ({ buildOpts, dest, label, logger, onEnd, should if (result.warnings.length) { logger.warn({ warnings: result.warnings.map(serializeEsbuildMessage) }, `${label} build warnings`) } - await writeMetafile({ dest, result, shouldWrite: shouldWriteMetafile }) + const write = async () => { + const target = writeDest?.() ?? dest + await writeEsbuildOutputFiles(result, dest, target) + await writeMetafile({ dest: target, result, shouldWrite: shouldWriteMetafile }) + } + if (promote) await promote(result, write) + else await write() if (first) logger.debug(`${label} initial build complete`) else logger.info(`${label} rebuild complete`) } @@ -633,6 +709,18 @@ async function createWatchBuild ({ buildOpts, dest, label, logger, onEnd, should } } +/** @param {esbuild.BuildOptions} opts */ +async function buildControlled (opts) { + validateEsbuildEntryOutputs(opts) + try { + return await esbuild.build(opts) + } catch (error) { + const failure = /** @type {esbuild.BuildFailure} */ (error) + rethrowEsbuildOutputConflict(failure.errors ?? [], opts) + throw error + } +} + /** * @param {esbuild.BuildContext[]} contexts * @returns {DisposableBuildContext} @@ -688,8 +776,12 @@ export function createEsbuildOutputRecords ({ src, dest, siteData, buildResults, filepath, outputRelname, kind, - entryPoint: outputMeta.entryPoint, - sourceRelname: outputMeta.entryPoint ? toPosix(relative(src, resolve(outputMeta.entryPoint))) : undefined, + ...(outputMeta.entryPoint + ? { + entryPoint: outputMeta.entryPoint, + sourceRelname: toPosix(relative(src, resolve(outputMeta.entryPoint))), + } + : {}), })) } diff --git a/lib/build-esbuild/output-conflicts.js b/lib/build-esbuild/output-conflicts.js new file mode 100644 index 00000000..b7328fcc --- /dev/null +++ b/lib/build-esbuild/output-conflicts.js @@ -0,0 +1,70 @@ +/** @import { BuildOptions, Message } from 'esbuild' */ +import { basename, dirname, extname, join, relative, resolve } from 'node:path' +import { OutputRegistry } from '../output-registry.js' +import { toPosix } from '../helpers/path.js' + +/** Entry output patterns, using esbuild's configured base, aliases and extensions. + * @param {BuildOptions} opts + */ +function entryOutputs (opts) { + const entries = Array.isArray(opts.entryPoints) + ? opts.entryPoints.map(entry => typeof entry === 'string' ? { in: entry, out: undefined } : entry) + : Object.entries(opts.entryPoints ?? {}).map(([out, input]) => ({ in: input, out })) + const cwd = opts.absWorkingDir ?? process.cwd() + const base = resolve(cwd, opts.outbase ?? '.') + return entries.map(entry => { + const extension = extname(entry.in) + const outputExtension = extension === '.css' ? '.css' : '.js' + const name = basename(entry.in, extension) + const dir = toPosix(relative(base, dirname(resolve(cwd, entry.in)))) + const pattern = (entry.out ?? (opts.entryNames ?? '[dir]/[name]') + .replaceAll('[dir]', dir || '.') + .replaceAll('[name]', name) + .replaceAll('[ext]', (opts.outExtension?.[outputExtension] ?? outputExtension).slice(1))) + (opts.outExtension?.[outputExtension] ?? outputExtension) + return { pattern: toPosix(join(pattern)), source: entry.in } + }) +} + +/** Reject statically identifiable entry collisions, including identical contents + * that esbuild would silently coalesce. Hashed names are checked on native errors. + * @param {BuildOptions} opts + */ +export function validateEsbuildEntryOutputs (opts) { + if (opts.outfile || !opts.outbase) return + const registry = new OutputRegistry() + const seen = new Set() + for (const { pattern, source } of entryOutputs(opts)) { + if (pattern.includes('[hash]') || pattern.startsWith('../')) continue + const key = JSON.stringify([pattern, source]) + if (seen.has(key)) continue + seen.add(key) + registry.claim(pattern, { id: source, type: 'esbuild', path: source }) + } +} + +/** Attach domain conflict diagnostics when the native collision can be traced to + * two configured entries. Unidentifiable plugin/chunk errors stay native rather + * than inventing producer attribution or rerunning user plugins. + * @param {Message[]} errors + * @param {BuildOptions} opts + */ +export function rethrowEsbuildOutputConflict (errors, opts) { + if (!opts.outdir || !opts.outbase) return + for (const error of errors) { + const match = /Two output files share the same path but have different contents: (.+)$/.exec(error.text) + if (!match?.[1]) continue + const path = toPosix(relative(resolve(opts.absWorkingDir ?? process.cwd(), opts.outdir), resolve(opts.absWorkingDir ?? process.cwd(), match[1]))) + const candidates = entryOutputs(opts).filter(({ pattern }) => { + const patterns = [pattern] + const jsExtension = opts.outExtension?.['.js'] ?? '.js' + if (pattern.endsWith(jsExtension)) patterns.push(pattern.slice(0, -jsExtension.length) + (opts.outExtension?.['.css'] ?? '.css')) + return patterns.some(pattern => { + const regex = pattern.split('[hash]').map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('[^/]+') + return new RegExp(`^${regex}(?:\\.map)?$`).test(path) + }) + }) + if (new Set(candidates.map(entry => entry.source)).size < 2) continue + const registry = new OutputRegistry() + for (const entry of candidates) registry.claim(path, { id: entry.source, type: 'esbuild', path: entry.source }) + } +} diff --git a/lib/build-pages/index.js b/lib/build-pages/index.js index e89392c3..b7f71a7a 100644 --- a/lib/build-pages/index.js +++ b/lib/build-pages/index.js @@ -6,10 +6,12 @@ * @import { ResolvedLayout } from './page-data.js' * @import { DomstackManifestRecord } from '../domstack-manifest/index.js' * @import { WatchDependencyState, WatchConsumer, WatchDependencyTracker } from './watch-dependencies.js' + * @import { OutputClaim, OutputOwner } from '../output-registry.js' */ import { Worker } from 'worker_threads' -import { basename, dirname, isAbsolute, join, normalize, resolve } from 'path' +import { basename, dirname, isAbsolute, join, normalize, relative, resolve } from 'path' +import { copyFile, mkdir, mkdtemp, rm } from 'node:fs/promises' import pMap from 'p-map' import { cpus } from 'os' import { keyBy } from '../helpers/key-by.js' @@ -23,6 +25,8 @@ import { DomStackDataError, DomStackOutputConflictError } from '../helpers/domst import { isAsyncIterable, isPlainObject } from '../helpers/type-guards.js' import { createSubscribedData, resolveDataDeps } from './data-deps.js' import { WatchDependencyTracker as WatchDependencyTrackerClass } from './watch-dependencies.js' +import { OutputRegistry, isCaseInsensitiveDest } from '../output-registry.js' +import { assertInsideDest } from '../helpers/path.js' const MAX_CONCURRENCY = Math.min(cpus().length, 24) @@ -44,6 +48,8 @@ const __dirname = import.meta.dirname * @property {TemplateReport[]} templates * @property {WatchDependencyState | undefined} [watchDependencies] * @property {string[] | undefined} [rebuiltPagesFilePaths] + * @property {OutputClaim[] | undefined} [newClaims] - Claims produced by the selected owners. + * @property {string[]} [replacedOwnerIds] */ /** @@ -89,6 +95,10 @@ const __dirname = import.meta.dirname * @property {boolean | undefined} [buildDrafts] - Include generated page definitions marked as drafts. * @property {WatchDependencyState | null | undefined} [previousWatchDependencies] - Dependency state from the previous successful watch build. * @property {boolean | undefined} [trackWatchDependencies] - Collect subscriptions for incremental watch builds. + * @property {OutputClaim[] | null | undefined} [previousOutputClaims] - Output ownership from the latest successful build. + + * @property {boolean} [caseInsensitive] + * @property {(report: PageBuildStepResult, write: () => Promise) => Promise} [promoteOutputs] */ /** @@ -317,7 +327,7 @@ function normalizeGeneratedOutputPart (value, { field, allowEmpty = false }) { if (value.split(/[\\/]+/).includes('..')) throw new Error(`Generated page ${field} must not contain ".." segments: ${value}`) if (/[\\/]$/.test(value)) throw new Error(`Generated page ${field} must name a file: ${value}`) - const normalized = normalize(value) + const normalized = normalize(value.replaceAll('\\', '/')) if (!allowEmpty && normalized === '.') throw new Error(`Generated page ${field} must not be empty`) return normalized === '.' ? '' : normalized } @@ -364,31 +374,12 @@ function generatedDefinitionToPageInfo ({ definition, pagesFile, index }) { * @param {Set | null} params.pagesFileFilterSet * @param {boolean | undefined} params.buildDrafts * @param {WatchDependencyTracker} params.watchDependencyTracker + * @param {OutputRegistry} params.outputRegistry * @returns {Promise} */ -async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, pagesFileFilterSet, buildDrafts, watchDependencyTracker }) { +async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, pagesFileFilterSet, buildDrafts, watchDependencyTracker, outputRegistry }) { /** @type {PageInfo[]} */ const generatedPageInfos = [] - /** @type {Map} */ - const pageOutputClaims = new Map() - - for (const pageInfo of siteData.pages) { - pageOutputClaims.set(resolve(pageInfo.outputRelname), { - type: 'page', - path: pageInfo.pageFile.relname, - }) - } - - // Unselected factories keep their outputs. Reserve those paths without - // rerunning the owners, so a targeted build cannot silently overwrite them. - if (pagesFileFilterSet) { - const ownerRelnames = new Map((siteData.pagesFiles ?? []).map(({ pagesFile }) => [pagesFile.filepath, pagesFile.relname])) - for (const consumer of Object.values(watchDependencyTracker.state.consumers)) { - if (consumer.type === 'page' && consumer.ownerPath && !pagesFileFilterSet.has(consumer.ownerPath)) { - pageOutputClaims.set(resolve(consumer.key), { type: 'page', path: ownerRelnames.get(consumer.ownerPath) ?? consumer.key }) - } - } - } for (const pagesFile of siteData.pagesFiles ?? []) { if (pagesFileFilterSet && !pagesFileFilterSet.has(pagesFile.pagesFile.filepath)) continue @@ -425,24 +416,7 @@ async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, p const generatedPageInfo = generatedDefinitionToPageInfo({ definition, pagesFile, index }) if (generatedPageInfo.draft && !buildDrafts) continue - const outputKey = resolve(generatedPageInfo.outputRelname) - const existingClaim = pageOutputClaims.get(outputKey) - const generatedClaim = { - type: /** @type {const} */ ('page'), - path: generatedPageInfo.pageFile.relname, - } - if (existingClaim) { - throw new DomStackOutputConflictError( - `Output path conflict: ${generatedPageInfo.outputRelname} is produced by both ${existingClaim.path} and ${generatedClaim.path}.`, - { - outputPath: generatedPageInfo.outputRelname, - a: existingClaim, - b: generatedClaim, - } - ) - } - - pageOutputClaims.set(outputKey, generatedClaim) + outputRegistry.claim(generatedPageInfo.outputRelname, pageOutputOwner(generatedPageInfo)) generatedPageInfos.push(generatedPageInfo) } } catch (err) { @@ -462,50 +436,89 @@ async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, p * * @type {PageBuildStep} */ -export function buildPages (src, dest, siteData, opts) { - // Only page-build filters cross the worker boundary. General build options - // can contain functions (manifest hooks and predicates) or logger instances, - // neither of which can be structured-cloned. - /** @type {BuildPagesFilterOptions} */ - const workerOpts = { - pageFilterPaths: opts?.pageFilterPaths, - templateFilterPaths: opts?.templateFilterPaths, - pagesFileFilterPaths: opts?.pagesFileFilterPaths, - buildDrafts: opts?.buildDrafts, - previousWatchDependencies: opts?.previousWatchDependencies, - trackWatchDependencies: opts?.trackWatchDependencies, - } +export async function buildPages (src, dest, siteData, opts) { + await mkdir(dest, { recursive: true }) + const stageDest = await mkdtemp(join(dest, '.domstack-pages-')) + try { + // Only page-build filters cross the worker boundary. General build options + // can contain functions (manifest hooks and predicates) or logger instances, + // neither of which can be structured-cloned. + /** @type {BuildPagesFilterOptions} */ + const workerOpts = { + pageFilterPaths: opts?.pageFilterPaths, + templateFilterPaths: opts?.templateFilterPaths, + pagesFileFilterPaths: opts?.pagesFileFilterPaths, + buildDrafts: opts?.buildDrafts, + previousWatchDependencies: opts?.previousWatchDependencies, + trackWatchDependencies: opts?.trackWatchDependencies, + previousOutputClaims: opts?.previousOutputClaims, + caseInsensitive: opts?.caseInsensitive ?? await isCaseInsensitiveDest(dest), + } + + const buildReport = await new Promise((resolve, reject) => { + const worker = new Worker(join(__dirname, 'worker.js'), { + workerData: { src, dest: stageDest, siteData, opts: workerOpts }, + }) + + worker.once('message', message => { + /** @type { WorkerBuildStepResult } */ + const workerReport = message + + /** @type {PageBuildStepResult} */ + const report = { + type: workerReport.type, + report: workerReport.report, + outputs: workerReport.outputs, + errors: [], + warnings: workerReport.warnings ?? [], + } - return new Promise((resolve, reject) => { - const worker = new Worker(join(__dirname, 'worker.js'), { - workerData: { src, dest, siteData, opts: workerOpts }, + if (workerReport.errors.length > 0) { + report.errors = workerReport.errors.map(({ error, errorData = {} }) => { + return restoreWorkerError(error, errorData) + }) + } + resolve(report) + }) + worker.once('error', reject) + worker.once('exit', (code) => { + if (code !== 0) reject(new Error(`Worker stopped with exit code ${code}`)) + }) }) - worker.once('message', message => { - /** @type { WorkerBuildStepResult } */ - const workerReport = message - - /** @type {PageBuildStepResult} */ - const buildReport = { - type: workerReport.type, - report: workerReport.report, - outputs: workerReport.outputs, - errors: [], - warnings: workerReport.warnings ?? [], + try { + if (buildReport.errors.length === 0) { + const write = () => promotePageOutputs(dest, buildReport) + if (opts?.promoteOutputs) await opts.promoteOutputs(buildReport, write) + else await write() } + return buildReport + } finally { + for (const output of buildReport.outputs) output.filepath = resolve(dest, output.outputRelname) + for (const page of buildReport.report.pages) page.pageFilePath = resolve(dest, relative(stageDest, page.pageFilePath)) + } + } finally { + await rm(stageDest, { recursive: true, force: true }) + } +} - if (workerReport.errors.length > 0) { - buildReport.errors = workerReport.errors.map(({ error, errorData = {} }) => { - return restoreWorkerError(error, errorData) - }) - } - resolve(buildReport) - }) - worker.once('error', reject) - worker.once('exit', (code) => { - if (code !== 0) { reject(new Error(`Worker stopped with exit code ${code}`)) } - }) - }) +/** + * Promote a successful page phase only after every output has rendered and all + * claims have been validated. Render/conflict failures do not alter the real dest; + * an I/O failure during promotion can leave partially updated files. + * + * @param {string} dest + * @param {PageBuildStepResult} buildReport + */ +async function promotePageOutputs (dest, buildReport) { + const resolvedDest = resolve(dest) + for (const output of buildReport.outputs) { + const target = resolve(resolvedDest, output.outputRelname) + assertInsideDest(resolvedDest, target) + await mkdir(dirname(target), { recursive: true }) + await copyFile(output.filepath, target) + output.filepath = target + } } /** @@ -543,6 +556,10 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { enabled: opts?.trackWatchDependencies === true, } ) + const outputRegistry = new OutputRegistry(opts?.previousOutputClaims ?? [], { + + caseInsensitive: opts?.caseInsensitive ?? false, + }) // Note: markdown-it settings are now passed directly to builders through builderOptions @@ -650,8 +667,29 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { }) } + // Select once, after global-data invalidation has expanded the filters. Include + // deleted producers and factories that now emit no files, not just render results. + const replacedOwnerIds = new Set([ + ...(opts?.previousOutputClaims ?? []).filter(({ owner }) => { + if (owner.id.startsWith('page:')) return !pageFilterSet || pageFilterSet.has(owner.id.slice(5)) + if (owner.id.startsWith('template:')) return !templateFilterSet || templateFilterSet.has(owner.id.slice(9)) + if (owner.id.startsWith('pages-file:')) return !pagesFileFilterSet || pagesFileFilterSet.has(owner.id.slice(11)) + return false + }).map(({ owner }) => owner.id), + ...siteData.pages.filter(page => !pageFilterSet || pageFilterSet.has(page.pageFile.filepath)).map(page => pageOutputOwner(page).id), + ...siteData.templates.filter(template => !templateFilterSet || templateFilterSet.has(template.templateFile.filepath)).map(template => `template:${template.templateFile.filepath}`), + ...(siteData.pagesFiles ?? []).filter(file => !pagesFileFilterSet || pagesFileFilterSet.has(file.pagesFile.filepath)).map(file => `pages-file:${file.pagesFile.filepath}`), + ]) + outputRegistry.releaseOwnerIds(replacedOwnerIds) + result.report.replacedOwnerIds = [...replacedOwnerIds] + let generatedPageInfos = /** @type {PageInfo[]} */ ([]) try { + for (const page of siteData.pages) { + if (replacedOwnerIds.has(pageOutputOwner(page).id) || !opts?.previousOutputClaims) { + outputRegistry.claim(page.outputRelname, pageOutputOwner(page)) + } + } generatedPageInfos = await resolveGeneratedPageInfos({ siteData, factoryVars: globalVars, @@ -659,6 +697,7 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { pagesFileFilterSet, buildDrafts: opts?.buildDrafts, watchDependencyTracker, + outputRegistry, }) } catch (err) { const pagesFile = /** @type {PagesFileInfo | undefined} */ (err instanceof Error && 'pagesFile' in err ? err.pagesFile : undefined) @@ -710,6 +749,19 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { : (siteData.pagesFiles ?? []).map(pagesFile => pagesFile.pagesFile.filepath) } + try { + for (const page of pagesToWrite) { + const owner = pageOutputOwner(page.pageInfo) + + if (page.pageInfo.workers && Object.values(page.pageInfo.workers).some(worker => worker.outputRelname)) { + outputRegistry.claim(join(page.pageInfo.path, 'workers.json'), owner) + } + } + } catch (err) { + result.errors.push(serializeBuildError(err)) + return result + } + await Promise.all([ pMap(pagesToWrite, async (page) => { try { @@ -739,6 +791,7 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { globalData, template, watchDependencyTracker, + outputRegistry, }) result.report.templates.push(buildResult.report) @@ -749,6 +802,12 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { }, { concurrency: dividedConcurrency[1] }), ]) + result.report.newClaims = outputRegistry.snapshot() + .filter(claim => replacedOwnerIds.has(claim.owner.id)) + .map(claim => claim.owner.id.startsWith('pages-file:') + // Persist the factory's source identity, not its transient definition index. + ? { ...claim, owner: { ...claim.owner, path: claim.owner.path.replace(/#\d+$/, '') } } + : claim) if (opts?.trackWatchDependencies) { watchDependencyTracker.pruneGeneratedPages( new Set(generatedPages.map(page => page.pageInfo.outputRelname)), @@ -759,6 +818,17 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { return result } +/** + * @param {PageInfo} pageInfo + * @returns {OutputOwner} + */ +function pageOutputOwner (pageInfo) { + const pagesFile = pageInfo.generated?.pagesFile.pagesFile + return pagesFile + ? { id: `pages-file:${pagesFile.filepath}`, type: 'page', path: pageInfo.pageFile.relname } + : { id: `page:${pageInfo.pageFile.filepath}`, type: 'page', path: pageInfo.pageFile.relname } +} + /** * Add invalidated consumers to the mutable filters for a targeted build. * diff --git a/lib/build-pages/page-builders/page-writer.js b/lib/build-pages/page-builders/page-writer.js index 4461e2e0..982e252b 100644 --- a/lib/build-pages/page-builders/page-writer.js +++ b/lib/build-pages/page-builders/page-writer.js @@ -156,6 +156,7 @@ export async function pageWriter ({ filepath: workersFilePath, outputRelname: join(page.pageInfo.path, 'workers.json'), kind: 'worker-manifest', + sourceRelname: page.pageInfo.pageFile.relname, pagePath: page.pageInfo.path, pageUrl: page.pageInfo.url, })) diff --git a/lib/build-pages/page-builders/template-builder.js b/lib/build-pages/page-builders/template-builder.js index f5d91efb..dc24fbbb 100644 --- a/lib/build-pages/page-builders/template-builder.js +++ b/lib/build-pages/page-builders/template-builder.js @@ -4,6 +4,7 @@ * @import { WatchDependencyTracker } from '../watch-dependencies.js' */ +import { OutputRegistry } from '../../output-registry.js' import { dirname, join, relative, resolve } from 'node:path' import { writeFile, mkdir } from 'fs/promises' import { createDomstackManifestRecord } from '../../domstack-manifest/index.js' @@ -91,6 +92,7 @@ function isTemplateOutputOverrideArray (value) { * @param {Record} params.globalData - Values returned by global.data. * @param {TemplateInfo} params.template - The TemplateInfo of the template. * @param {WatchDependencyTracker} params.watchDependencyTracker - Declarative watch dependency state. + * @param {OutputRegistry} [params.outputRegistry] - Shared output ownership for this page phase. * @returns {Promise<{ report: TemplateReport, outputs: DomstackManifestRecord[] }>} */ export async function templateBuilder ({ @@ -99,6 +101,7 @@ export async function templateBuilder ({ globalData, template, watchDependencyTracker, + outputRegistry = new OutputRegistry(), }) { const importResults = await import(template.templateFile.filepath) if (!importResults.default || typeof importResults.default !== 'function') { @@ -144,6 +147,7 @@ export async function templateBuilder ({ content: templateResults, template, outputRecords, + outputRegistry, }) } else if (isTemplateOutputOverrideArray(templateResults)) { type = 'array' @@ -155,6 +159,7 @@ export async function templateBuilder ({ content: templateResult.content, template, outputRecords, + outputRegistry, }) } } else if (isTemplateOutputOverride(templateResults)) { @@ -166,6 +171,7 @@ export async function templateBuilder ({ content: templateResults.content, template, outputRecords, + outputRegistry, }) } else if (isAsyncIterable(templateResults)) { type = 'async-iterator' @@ -178,6 +184,7 @@ export async function templateBuilder ({ content: templateResult.content, template, outputRecords, + outputRegistry, }) } else { throw new Error(`Template file returned unknown return type: ${typeof templateResult}`) @@ -205,6 +212,7 @@ export async function templateBuilder ({ * @param {string} params.content * @param {TemplateInfo} params.template * @param {DomstackManifestRecord[]} params.outputRecords + * @param {OutputRegistry} params.outputRegistry */ async function writeTemplateOutput ({ dest, @@ -213,14 +221,20 @@ async function writeTemplateOutput ({ content, template, outputRecords, + outputRegistry, }) { - const filepath = resolve(fileDir, outputName) + const filepath = resolve(fileDir, outputName.replaceAll('\\', '/')) assertInsideDest(dest, filepath, `Template output escapes dest: ${filepath}`) + const outputRelname = toPosix(relative(dest, filepath)) + outputRegistry.claim(outputRelname, { + id: `template:${template.templateFile.filepath}`, + type: 'template', + path: template.templateFile.relname, + }) const filePathDirname = dirname(filepath) await mkdir(filePathDirname, { recursive: true }) await writeFile(filepath, content) - const outputRelname = toPosix(relative(dest, filepath)) outputRecords.push(createDomstackManifestRecord({ dest, filepath, diff --git a/lib/build-static/index.js b/lib/build-static/index.js index 3bbdb0e9..f9a17886 100644 --- a/lib/build-static/index.js +++ b/lib/build-static/index.js @@ -1,9 +1,10 @@ /** - * @import { BuildStep, BuildStepResult } from '../builder.js' + * @import { BuildStep, BuildStepResult, DomStackOpts } from '../builder.js' */ import { processedExtensions } from '../file-conventions.js' -import { copy } from 'cpx2' -import { createCopiedDomstackManifestRecords } from '../helpers/cpx2-report.js' +/** @import { copy } from 'cpx2' */ +import { stagedCopy } from '../helpers/staged-copy.js' +import { OutputRegistry } from '../output-registry.js' /** * @typedef {Awaited> | Record} StaticBuilderReport @@ -29,9 +30,13 @@ export function getCopyGlob (src) { /** * run CPX2 on src folder * - * @type {StaticBuildStep} + * @param {string} src + * @param {string} dest + * @param {unknown} _siteData + * @param {DomStackOpts | null} [opts] + * @param {OutputRegistry} [registry] */ -export async function buildStatic (src, dest, _siteData, opts) { +export async function buildStatic (src, dest, _siteData, opts, registry = new OutputRegistry()) { /** @type {StaticBuildStepResult} */ const results = { type: 'static', @@ -42,14 +47,10 @@ export async function buildStatic (src, dest, _siteData, opts) { } try { - const report = await copy(getCopyGlob(src), dest, ...(opts?.ignore ? [{ ignore: opts.ignore }] : [])) - results.report = report - results.outputs = createCopiedDomstackManifestRecords({ - src, - dest, - report, - kind: 'static', - }) + if (opts?.static === false) return results + const copied = await stagedCopy(getCopyGlob(src), src, dest, 'static', registry, opts?.ignore) + results.report = copied.report + results.outputs = copied.outputs } catch (err) { const buildError = new Error('Error copying static files', { cause: err }) results.errors.push(buildError) diff --git a/lib/builder.js b/lib/builder.js index 238ba455..9c2d98c3 100644 --- a/lib/builder.js +++ b/lib/builder.js @@ -6,6 +6,7 @@ * @import { PageBuildStepResult } from './build-pages/index.js' * @import { StaticBuildStepResult } from './build-static/index.js' * @import { CopyBuildStepResult } from './build-copy/index.js' + * @import { OutputClaim } from './output-registry.js' * @import { DomstackManifest, DomstackManifestConfig, DomstackManifestRecord } from './domstack-manifest/index.js' */ @@ -14,9 +15,15 @@ import { identifyPages } from './identify-pages.js' import { buildStatic } from './build-static/index.js' import { buildCopy } from './build-copy/index.js' import { buildEsbuild, buildServiceWorkerEsbuild } from './build-esbuild/index.js' +import { cp, mkdir, mkdtemp, realpath, rm } from 'node:fs/promises' +import { join, resolve } from 'node:path' import { DomStackAggregateError } from './helpers/domstack-aggregate-error.js' -import { ensureDest } from './helpers/ensure-dest.js' + +import { OutputRegistry, isCaseInsensitiveDest } from './output-registry.js' + +import { remapCopyReport } from './helpers/staged-copy.js' import { + DEFAULT_DOMSTACK_MANIFEST_FILENAME, isDomstackManifestEnabled, reconcileDomstackManifest, resolveDomstackManifestOptions, @@ -90,6 +97,7 @@ import { * @property {PageBuildStepResult} [pageBuildResults] * @property {DomstackManifest} [domstackManifest] * @property {BuildStepWarnings} warnings + * @property {OutputClaim[]} [outputClaims] - Internal full-watch ownership snapshot. */ /** @@ -101,6 +109,7 @@ import { * @param {string} src - The source directory from which the site should be built. * @param {string} dest - The destination directory where the built site should be placed. * @param {DomStackOpts} opts - Options for the build process. + * @param {{ watch?: boolean, caseInsensitive?: boolean, promoteOutputs?: (claims: OutputClaim[], write: () => Promise) => Promise }} [internal] * @returns {Promise} * * @example @@ -116,11 +125,50 @@ import { * console.error(error) * } */ -export async function builder (src, dest, opts) { +export async function builder (src, dest, opts, internal = {}) { + if (!internal.watch) { + const results = await buildInto(src, dest, opts, dest, false, internal.caseInsensitive) + delete results.outputClaims + return results + } + await mkdir(dest, { recursive: true }) + const stageDest = await mkdtemp(join(resolve(dest), '.domstack-stage-')) + try { + const results = await buildInto(src, stageDest, { ...opts, ignore: [...(opts.ignore ?? []), '.domstack-stage-*'] }, dest, internal.watch ?? false, internal.caseInsensitive) + remapBuildResults(results, stageDest, dest) + + const write = async () => { + await mkdir(dest, { recursive: true }) + await cp(stageDest, await realpath(dest), { recursive: true, force: true }) + } + if (internal.promoteOutputs) await internal.promoteOutputs(results.outputClaims ?? [], write) + else await write() + delete results.outputClaims + return results + } catch (error) { + if (error instanceof DomStackAggregateError && error.results?.esbuildResults) remapBuildResults(error.results, stageDest, dest) + throw error + } finally { + await rm(stageDest, { recursive: true, force: true }) + } +} + +/** + * Build into the requested destination (or the caller's full-watch stage). + * + * @param {string} src + * @param {string} dest + * @param {DomStackOpts} opts + * @param {string} publicDest + * @param {boolean} watch + * @param {boolean} [casePolicy] + * @returns {Promise} + */ +async function buildInto (src, dest, opts, publicDest, watch, casePolicy) { const errors = [] /** @type {BuildStepErrors} */ const warnings = [] /** @type {BuildStepWarnings} */ - const siteData = await identifyPages(src, opts) /** @type {SiteData} */ + const siteData = await identifyPages(src, { ...(opts.ignore ? { ignore: opts.ignore } : {}), ...(opts.buildDrafts !== undefined ? { buildDrafts: opts.buildDrafts } : {}) }) /** @type {SiteData} */ errors.push(...siteData.errors) warnings.push(...siteData.warnings) @@ -130,7 +178,7 @@ export async function builder (src, dest, opts) { throw pageWalkErrors } - await ensureDest(dest, siteData) + await mkdir(dest, { recursive: true }) const domstackManifestSettingsPath = siteData?.domstackManifestSettings?.filepath const domstackManifestEnabled = isDomstackManifestEnabled({ @@ -142,16 +190,18 @@ export async function builder (src, dest, opts) { opts, }) + const caseInsensitive = casePolicy ?? await isCaseInsensitiveDest(publicDest) + const outputRegistry = new OutputRegistry([], { caseInsensitive }) const [ esbuildResults, staticResults, copyResults, ] = await Promise.all([ - buildEsbuild(src, dest, siteData, opts), - opts.static - ? buildStatic(src, dest, siteData, opts) + buildEsbuild(src, dest, siteData, opts, outputRegistry, watch, publicDest), + opts.static !== false + ? buildStatic(src, dest, siteData, opts, outputRegistry) : Promise.resolve(null), - buildCopy(src, dest, siteData, opts), + buildCopy(src, dest, siteData, opts, outputRegistry), ]) /** @type {Results} */ @@ -179,7 +229,12 @@ export async function builder (src, dest, opts) { throw preBuildError } - const pageBuildResults = await buildPages(src, dest, siteData, opts) + const pageBuildResults = await buildPages(src, dest, siteData, { + ...opts, + previousOutputClaims: outputRegistry.snapshot(), + caseInsensitive, + trackWatchDependencies: watch, + }) errors.push(...pageBuildResults.errors) warnings.push(...pageBuildResults.warnings) @@ -190,6 +245,11 @@ export async function builder (src, dest, opts) { throw buildError } + outputRegistry.releaseOwnerIds(pageBuildResults.report.replacedOwnerIds ?? []) + for (const claim of pageBuildResults.report.newClaims ?? []) outputRegistry.claim(claim.outputRelname, claim.owner) + delete pageBuildResults.report.newClaims + delete pageBuildResults.report.replacedOwnerIds + const baseOutputRecords = collectOutputRecords( esbuildResults, staticResults, @@ -211,7 +271,14 @@ export async function builder (src, dest, opts) { const domstackManifest = domstackManifestReconciliation?.manifest const domstackManifestBuiltHookResult = domstackManifest - ? await runDomstackManifestBuiltHooks(dest, domstackManifest, domstackManifestOptions) + ? await runDomstackManifestBuiltHooks(dest, domstackManifest, domstackManifestOptions, { + publicDest, + claimOutput: (outputRelname, hookIndex) => outputRegistry.claim(outputRelname, { + id: `manifest-hook:${hookIndex}`, + type: 'manifest hook', + path: `manifestBuilt hook #${hookIndex + 1}`, + }), + }) : { serviceWorkerDefines: {} } const serviceWorkerBuildDefines = { @@ -224,7 +291,8 @@ export async function builder (src, dest, opts) { dest, siteData, esbuildResults.report.buildOpts, - serviceWorkerBuildDefines + serviceWorkerBuildDefines, + outputRegistry ) errors.push(...serviceWorkerEsbuildResults.errors) @@ -238,16 +306,48 @@ export async function builder (src, dest, opts) { if (domstackManifest) results.domstackManifest = domstackManifest if (domstackManifest && shouldWriteDomstackManifest(opts)) { + outputRegistry.claim(DEFAULT_DOMSTACK_MANIFEST_FILENAME, { + id: 'domstack-manifest', + type: 'metadata', + path: 'generated domstack manifest', + }) await writeDomstackManifest(dest, domstackManifest) } + results.outputClaims = outputRegistry.snapshot() return results } /** - * @param {...(BuildOutputStepResult | null)} results + * @param {...(BuildOutputStepResult | null | undefined)} results * @returns {DomstackManifestRecord[]} */ function collectOutputRecords (...results) { return results.flatMap(result => result?.outputs ?? []) } + +/** + * Staging is internal; public reports continue to describe the requested dest. + * + * @param {Results} results + * @param {string} stageDest + * @param {string} dest + */ +function remapBuildResults (results, stageDest, dest) { + const steps = [ + results.esbuildResults, + results.staticResults, + results.copyResults, + results.pageBuildResults, + ] + for (const record of collectOutputRecords(...steps)) { + record.filepath = resolve(dest, record.outputRelname) + } + + for (const page of results.pageBuildResults?.report.pages ?? []) { + page.pageFilePath = resolve(dest, page.pageFilePath.slice(resolve(stageDest).length + 1)) + } + + if (results.staticResults) remapCopyReport(results.staticResults.report, dest) + for (const report of Object.values(results.copyResults?.report ?? {})) remapCopyReport(report, dest) +} diff --git a/lib/domstack-manifest/hooks.js b/lib/domstack-manifest/hooks.js index 009e14dd..2faa7ffc 100644 --- a/lib/domstack-manifest/hooks.js +++ b/lib/domstack-manifest/hooks.js @@ -23,7 +23,7 @@ export async function writeDomstackManifest (dest, domstackManifest) { * @param {string | Uint8Array} contents */ async function writeGeneratedManifestFile (dest, outputRelname, contents) { - const filepath = resolve(dest, outputRelname) + const filepath = resolve(dest, outputRelname.replaceAll('\\', '/')) assertInsideDest(dest, filepath) await mkdir(dirname(filepath), { recursive: true }) await writeFile(filepath, contents) @@ -35,15 +35,16 @@ async function writeGeneratedManifestFile (dest, outputRelname, contents) { * @param {string} dest * @param {DomstackManifest} manifest * @param {DomstackManifestOptions} options + * @param {{ claimOutput?: (outputRelname: string, hookIndex: number) => void, publicDest?: string }} [buildOptions] * @returns {Promise} */ -export async function runDomstackManifestBuiltHooks (dest, manifest, options) { +export async function runDomstackManifestBuiltHooks (dest, manifest, options, buildOptions = {}) { const serviceWorkerDefines = /** @type {Record} */ ({}) const hooks = options.hooks?.manifestBuilt ?? [] - for (const hook of hooks) { + for (const [hookIndex, hook] of hooks.entries()) { await hook({ - dest, + dest: buildOptions.publicDest ?? dest, manifest, defineServiceWorkerConstant: (identifier, value) => { const serializedValue = JSON.stringify(value) @@ -52,7 +53,10 @@ export async function runDomstackManifestBuiltHooks (dest, manifest, options) { } serviceWorkerDefines[identifier] = serializedValue }, - writeFile: (outputRelname, contents) => writeGeneratedManifestFile(dest, outputRelname, contents), + writeFile: (outputRelname, contents) => { + buildOptions.claimOutput?.(outputRelname, hookIndex) + return writeGeneratedManifestFile(dest, outputRelname, contents) + }, }) } diff --git a/lib/helpers/domstack-error.js b/lib/helpers/domstack-error.js index 9a03915e..6ce7877e 100644 --- a/lib/helpers/domstack-error.js +++ b/lib/helpers/domstack-error.js @@ -30,7 +30,7 @@ export class DomStackDataError extends Error { /** * @typedef DomStackOutputConflictErrorClaim - * @property {'page'} type - The kind of output producer. + * @property {string} type - The kind of output producer. * @property {string} path - Human-readable source or output path for the producer. */ diff --git a/lib/helpers/staged-copy.js b/lib/helpers/staged-copy.js new file mode 100644 index 00000000..e66c3a7a --- /dev/null +++ b/lib/helpers/staged-copy.js @@ -0,0 +1,61 @@ +/** + * @import { OutputRegistry } from '../output-registry.js' + * @import { NormalizedOptions } from 'cpx2' + */ +import normalizeOptions from 'cpx2/lib/utils/normalize-options.js' +import applyAction from 'cpx2/lib/utils/apply-action.js' +import copy from 'cpx2/lib/utils/copy-file.js' +import { copyFile, mkdir, mkdtemp, rm } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { createCopiedDomstackManifestRecords } from './cpx2-report.js' + +/** + * Keep cpx's matching and mapping semantics, but inventory before writing so + * case aliases and file/directory aliases cannot collapse in the staging tree. + * These cpx internals are intentionally confined to this adapter. + * @param {string} source + * @param {string} src + * @param {string} dest + * @param {'static' | 'copy'} kind + * @param {OutputRegistry} registry + * @param {string[]} [ignore] + * @param {string} [ownerPrefix] - Identity of this configured copy-root occurrence. + */ +export async function stagedCopy (source, src, dest, kind, registry, ignore = [], ownerPrefix = '') { + const options = normalizeOptions(source, dest, { ignore }) + const sources = /** @type {string[]} */ (await applyAction(options.source, options, path => path)) + const report = { + cleaned: [], + copied: sources.map(source => ({ source, output: options.toDestination(source), skipped: false })), + options, + } + const outputs = createCopiedDomstackManifestRecords({ src, dest, report, kind }) + registry.claimRecords(outputs, ownerPrefix) + await mkdir(dest, { recursive: true }) + const stage = await mkdtemp(join(dest, '.domstack-copy-')) + try { + for (const [index, entry] of report.copied.entries()) { + const stagedPath = join(stage, String(index)) + await copy(entry.source, stagedPath, options) + } + for (const [index, output] of outputs.entries()) { + const target = resolve(dest, output.outputRelname) + await mkdir(dirname(target), { recursive: true }) + await copyFile(join(stage, String(index)), target) + } + return { report, outputs } + } finally { + await rm(stage, { recursive: true, force: true }) + } +} + +/** Rebuild the mapper as well as the known report paths after full-watch staging. + * @param {object} value + * @param {string} dest + */ +export function remapCopyReport (value, dest) { + if (!('options' in value)) return + const report = /** @type {{ options: NormalizedOptions, copied: { source: string, output: string }[] }} */ (value) + report.options = normalizeOptions(report.options.source, dest, { ignore: report.options.ignore ?? [] }) + for (const entry of report.copied) entry.output = report.options.toDestination(entry.source) +} diff --git a/lib/output-registry.js b/lib/output-registry.js new file mode 100644 index 00000000..ba92f84c --- /dev/null +++ b/lib/output-registry.js @@ -0,0 +1,174 @@ +/** + * @import { DomstackManifestRecord } from './domstack-manifest/index.js' + * @import { DomStackOutputConflictErrorClaim } from './helpers/domstack-error.js' + */ + +import { join, posix } from 'node:path' +import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { DomStackOutputConflictError } from './helpers/domstack-error.js' +import { toPosix } from './helpers/path.js' + +/** + * @typedef OutputOwner + * @property {string} id - Stable identity used to replace an owner's outputs in watch mode. + * @property {string} type - Human-readable producer type. + * @property {string} path - Human-readable source or build-step path. + */ + +/** + * @typedef OutputClaim + * @property {string} outputRelname + * @property {OutputOwner} owner + */ + +/** + * Track destination-relative output ownership for one successful build state. + */ +export class OutputRegistry { + /** @type {Map} */ + #claims = new Map() + #caseInsensitive + + /** + * @param {OutputClaim[]} [previousClaims] + * @param {{ replaceOwnerIds?: Iterable, caseInsensitive?: boolean }} [options] + */ + constructor (previousClaims = [], options = {}) { + this.#caseInsensitive = options.caseInsensitive ?? false + const replaceOwnerIds = new Set(options.replaceOwnerIds ?? []) + for (const claim of previousClaims) { + if (replaceOwnerIds.has(claim.owner.id)) continue + this.#claims.set(this.#key(claim.outputRelname), structuredClone(claim)) + } + } + + /** + * @param {string} outputRelname + * @param {OutputOwner} owner + */ + claim (outputRelname, owner) { + const normalized = normalizeOutputRelname(outputRelname) + const conflict = this.#findConflict(normalized) + if (conflict) throw createConflictError(normalized, conflict.owner, owner) + this.#claims.set(this.#key(normalized), { outputRelname: normalized, owner: { ...owner } }) + } + + /** @param {string} path */ + #key (path) { + const normalized = normalizeOutputRelname(path) + return this.#caseInsensitive ? normalized.normalize('NFC').toLowerCase() : normalized + } + + /** @param {Iterable} ownerIds */ + releaseOwnerIds (ownerIds) { + const released = new Set(ownerIds) + for (const [outputRelname, claim] of this.#claims) { + if (released.has(claim.owner.id)) this.#claims.delete(outputRelname) + } + } + + /** + * @param {DomstackManifestRecord[]} records + * @param {string} [ownerPrefix] + */ + claimRecords (records, ownerPrefix = '') { + const seen = new Set() + for (const record of records) { + // Reporting the same record twice is not a second write. Dynamic writers + // use claim() directly, where repeated outputs always conflict. + const key = JSON.stringify(record) + if (seen.has(key)) continue + seen.add(key) + const owner = outputOwnerForRecord(record) + this.claim(record.outputRelname, { ...owner, id: `${ownerPrefix}${owner.id}` }) + } + } + + /** @returns {OutputClaim[]} */ + snapshot () { + return Array.from(this.#claims.values(), claim => structuredClone(claim)) + } + + /** + * @param {string} outputRelname + * @returns {OutputClaim | undefined} + */ + #findConflict (outputRelname) { + outputRelname = this.#key(outputRelname) + const exact = this.#claims.get(outputRelname) + if (exact) return exact + + for (const [key, claim] of this.#claims) { + if (outputRelname.startsWith(`${key}/`) || key.startsWith(`${outputRelname}/`)) { + return claim + } + } + } +} + +/** Probe the destination volume rather than assuming case behavior from the OS. + * @param {string} dest + */ +export async function isCaseInsensitiveDest (dest) { + await mkdir(dest, { recursive: true }) + const probe = await mkdtemp(join(dest, '.domstack-case-')) + try { + await writeFile(join(probe, 'probe'), '') + try { + await access(join(probe, 'PROBE')) + return true + } catch (error) { + if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error + return false + } + } finally { + await rm(probe, { recursive: true, force: true }) + } +} + +/** + * @param {DomstackManifestRecord} record + * @returns {OutputOwner} + */ +export function outputOwnerForRecord (record) { + const source = record.sourceRelname ?? record.entryPoint + const path = source ?? describeBuildStep(record.kind) + return { + id: `${record.kind}:${source ?? record.outputRelname}`, + type: record.kind, + path, + } +} + +/** + * @param {string} outputRelname + * @returns {string} + */ +export function normalizeOutputRelname (outputRelname) { + const normalized = posix.normalize(toPosix(outputRelname).replaceAll('\\', '/')) + if (!normalized || normalized === '.' || normalized === '..' || normalized.startsWith('../') || normalized.startsWith('/') || /^[a-z]:/i.test(normalized)) { + throw new Error(`Output path must be destination-relative: ${outputRelname}`) + } + return normalized +} + +/** + * @param {string} outputPath + * @param {OutputOwner} a + * @param {OutputOwner} b + */ +function createConflictError (outputPath, a, b) { + const first = /** @type {DomStackOutputConflictErrorClaim} */ ({ type: a.type, path: a.path }) + const second = /** @type {DomStackOutputConflictErrorClaim} */ ({ type: b.type, path: b.path }) + return new DomStackOutputConflictError( + `Output path conflict: ${outputPath} is produced by both ${first.path} and ${second.path}.`, + { outputPath, a: first, b: second } + ) +} + +/** @param {DomstackManifestRecord['kind']} kind */ +function describeBuildStep (kind) { + if (kind === 'metadata') return 'domstack esbuild metadata' + if (kind === 'service-worker') return 'service worker build' + return `${kind} build step` +} diff --git a/test-cases/output-conflicts/index.test.js b/test-cases/output-conflicts/index.test.js new file mode 100644 index 00000000..668f79c0 --- /dev/null +++ b/test-cases/output-conflicts/index.test.js @@ -0,0 +1,614 @@ +/** + * @import { TestContext } from 'node:test' + * @import { DomStackOpts } from '../../lib/builder.js' + */ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { mkdir, mkdtemp, readFile, readdir, rename, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { createHash } from 'node:crypto' +import { builder } from '../../lib/builder.js' +import { stagedCopy } from '../../lib/helpers/staged-copy.js' +import { buildPages } from '../../lib/build-pages/index.js' +import { identifyPages } from '../../lib/identify-pages.js' +import { DomStackAggregateError } from '../../lib/helpers/domstack-aggregate-error.js' +import pino from 'pino' +import { DomStack } from '../../index.js' +import { OutputRegistry, isCaseInsensitiveDest } from '../../lib/output-registry.js' + +/** @param {string} root @param {Record} files */ +async function writeFiles (root, files) { + for (const [name, content] of Object.entries(files)) { + const path = join(root, name) + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, content) + } +} + +/** @param {TestContext} t @param {Record} files @param {DomStackOpts} [opts] */ +async function setup (t, files, opts = {}) { + const tmp = await mkdtemp(join(import.meta.dirname, 'tmp-')) + const src = join(tmp, 'src') + const dest = join(tmp, 'public') + await writeFiles(src, { + 'global.vars.js': "export default { layout: 'root' }", + 'root.layout.js': 'export default ({ children }) => children', + ...files, + }) + const logs = /** @type {string[]} */ ([]) + const site = new DomStack(src, dest, { ...opts, logger: pino({ level: 'debug' }, { write: line => logs.push(line) }) }) + const sites = [site] + t.after(async () => { + for (const site of sites) if (site.watching) await site.stopWatching() + await rm(tmp, { recursive: true, force: true }) + }) + return { src, dest, tmp, site, sites, logs } +} + +/** @param {unknown} error @returns {Error & { code?: string, contextData?: { outputPath?: string } } | undefined} */ +function conflict (error) { + if (!(error instanceof Error)) return + const value = /** @type {Error & { code?: string, errors?: unknown[], contextData?: { outputPath?: string } }} */ (error) + if (value.code === 'DOM_STACK_ERROR_OUTPUT_CONFLICT') return value + return conflict(value.cause) ?? value.errors?.map(conflict).find(Boolean) +} + +/** @param {string} output @param {string} [content] */ +const template = (output, content = 'template') => `export default () => ({ outputName: ${JSON.stringify(output)}, content: ${JSON.stringify(content)} })` + +for (const scenario of [ + { name: 'regular page/template', files: { 'page.html': 'page', 'a.template.js': template('index.html') }, output: 'index.html', sources: ['page.html', 'a.template.js'] }, + { name: 'generated page/template', files: { 'a.pages.js': "export default { outputName: 'generated.html', children: 'page' }", 'b.template.js': template('generated.html') }, output: 'generated.html', sources: ['a.pages.js', 'b.template.js'] }, + { name: 'template objects', files: { 'a.template.js': template('feed.xml'), 'b.template.js': template('feed.xml') }, output: 'feed.xml', sources: ['a.template.js', 'b.template.js'] }, + { name: 'template array', files: { 'a.template.js': "export default () => [{ outputName: 'feed.xml', content: 'one' }, { outputName: 'feed.xml', content: 'two' }]" }, output: 'feed.xml', sources: ['a.template.js'] }, + { name: 'template async iterator', files: { 'a.template.js': "export default async function * () { yield { outputName: 'feed.xml', content: 'one' }; yield { outputName: 'feed.xml', content: 'two' } }" }, output: 'feed.xml', sources: ['a.template.js'] }, + { name: 'file/directory templates', files: { 'a.template.js': template('feed'), 'b.template.js': template('feed/index.xml') }, output: 'feed', sources: ['a.template.js', 'b.template.js'] }, + { name: 'static/template', files: { 'feed.xml': 'static', 'a.template.js': template('feed.xml') }, output: 'feed.xml', sources: ['feed.xml', 'a.template.js'] }, + + { name: 'esbuild/template', files: { 'page.html': 'page', 'client.js': 'console.log(1)', 'esbuild.settings.js': "export default opts => ({ ...opts, entryNames: '[dir]/[name]' })", 'a.template.js': template('client.js') }, output: 'client.js', sources: ['client.js', 'a.template.js'] }, + { name: 'esbuild settings cannot bypass claims', files: { 'page.html': 'page', 'client.js': 'console.log(1)', 'esbuild.settings.js': "export default opts => ({ ...opts, entryNames: '[dir]/[name]', metafile: false, write: true })", 'a.template.js': template('client.js') }, output: 'client.js', sources: ['client.js', 'a.template.js'] }, + { name: 'service worker/template', files: { 'service-worker.js': 'console.log(1)', 'a.template.js': template('service-worker.js') }, output: 'service-worker.js', sources: ['service-worker.js', 'a.template.js'] }, + { name: 'workers.json/template', files: { 'page.html': 'page', 'test.worker.js': 'console.log(1)', 'a.template.js': template('workers.json') }, output: 'workers.json', sources: ['page.html', 'a.template.js'] }, + { name: 'metadata/template', files: { 'a.template.js': template('domstack-esbuild-meta.json') }, output: 'domstack-esbuild-meta.json', sources: ['domstack esbuild metadata', 'a.template.js'] }, + { name: 'manifest/template', files: { 'a.template.js': template('domstack-manifest.json') }, output: 'domstack-manifest.json', sources: ['generated domstack manifest', 'a.template.js'] }, + { name: 'normalized separators', files: { 'a.template.js': template('feed/index.xml'), 'b.template.js': template('feed\\index.xml') }, output: 'feed/index.xml', sources: ['a.template.js', 'b.template.js'] }, +]) { + test(`one-shot rejects ${scenario.name} without overwriting the first producer`, async t => { + const { site, dest } = await setup(t, scenario.files, { domstackManifest: true }) + await writeFiles(dest, { 'sentinel.txt': 'last successful build' }) + await assert.rejects(site.build(), error => { + const found = conflict(error) + assert.ok(found, String(error)) + assert.ok(found.message.includes(scenario.output), found.message) + for (const source of scenario.sources) assert.ok(found.message.includes(source), found.message) + return true + }) + assert.ok(!(await readdir(dest)).some(name => name.startsWith('.domstack-'))) + if (scenario.name === 'static/template') assert.equal(await readFile(join(dest, 'feed.xml'), 'utf8'), 'static') + if (scenario.name.startsWith('esbuild/')) assert.match(await readFile(join(dest, 'client.js'), 'utf8'), /console.log/) + if (scenario.name === 'manifest/template') assert.equal(await readFile(join(dest, 'domstack-manifest.json'), 'utf8'), 'template') + if (scenario.name === 'service worker/template') assert.equal(await readFile(join(dest, 'service-worker.js'), 'utf8'), 'template') + assert.equal(await readFile(join(dest, 'sentinel.txt'), 'utf8'), 'last successful build') + }) +} + +test('copy producers are isolated before file/directory and cross-step checks', async t => { + for (const mode of ['copy-copy', 'copy-static', 'copy-page', 'copy-esbuild']) { + await t.test(mode, async t => { + const { tmp, src, dest } = await setup(t, { + ...(mode === 'copy-static' ? { feed: 'static' } : {}), + ...(mode === 'copy-page' ? { 'feed/page.html': 'page' } : {}), + ...(mode === 'copy-esbuild' ? { 'page.html': 'page', 'client.js': 'console.log(1)', 'esbuild.settings.js': "export default opts => ({ ...opts, entryNames: '[dir]/[name]' })" } : {}), + }) + const a = join(tmp, 'a') + const b = join(tmp, 'b') + await writeFiles(a, { [mode === 'copy-static' ? 'feed/index.xml' : mode === 'copy-esbuild' ? 'client.js' : 'feed']: 'copy a' }) + await writeFiles(b, { 'feed/index.xml': 'copy b' }) + const site = new DomStack(src, dest, { copy: mode === 'copy-copy' ? [a, b] : [a] }) + await assert.rejects(site.build(), error => { + assert.ok(conflict(error), String(error)) + return true + }) + assert.ok(!(await readdir(dest)).some(name => name.startsWith('.domstack-'))) + if (mode === 'copy-page') assert.equal(await readFile(join(dest, 'feed'), 'utf8'), 'copy a') + }) + } +}) + +test('stages are unique, do not delete lookalike user directories, and stay out of metadata', async t => { + const { site, dest, tmp } = await setup(t, { 'page.html': 'page', 'client.js': 'console.log(1)' }, { domstackManifest: true }) + await writeFiles(`${dest}.domstack-stage`, { 'keep.txt': 'user data' }) + const [a, b] = await Promise.all([site.build(), site.build()]) + assert.equal(a.domstackManifest?.version, b.domstackManifest?.version) + assert.equal(await readFile(join(`${dest}.domstack-stage`, 'keep.txt'), 'utf8'), 'user data') + assert.ok(!(await readFile(join(dest, 'domstack-esbuild-meta.json'), 'utf8')).includes('.domstack-stage')) + assert.doesNotMatch(JSON.stringify(a), /\.domstack-stage-[a-zA-Z0-9]{6}/) + assert.ok(!(await readdir(tmp)).some(name => name.startsWith('.domstack-stage-'))) +}) + +test('case-insensitive destination collisions follow the destination filesystem', async t => { + const { site, dest } = await setup(t, { 'a.template.js': template('Feed.xml'), 'b.template.js': template('feed.xml') }) + if (await isCaseInsensitiveDest(dest)) await assert.rejects(site.build(), error => !!conflict(error)) + else { + await site.build() + assert.equal(await readFile(join(dest, 'Feed.xml'), 'utf8'), 'template') + assert.equal(await readFile(join(dest, 'feed.xml'), 'utf8'), 'template') + } +}) + +test('registry distinguishes duplicate records from duplicate writes and bounds replacement state', () => { + const owner = { id: 'template:a', type: 'template', path: 'a.template.js' } + let registry = new OutputRegistry([], { caseInsensitive: true }) + registry.claim('Feed\\index.xml', owner) + assert.throws(() => registry.claim('feed/index.xml', owner), error => !!conflict(error)) + assert.throws(() => registry.claim('FEED', { ...owner, id: 'b' }), error => !!conflict(error)) + for (let i = 0; i < 100; i++) { + registry = new OutputRegistry(registry.snapshot(), { replaceOwnerIds: [owner.id] }) + registry.claim(`feed-${i}.xml`, owner) + assert.equal(registry.snapshot().length, 1) + } + const record = { filepath: '/dest/a', outputRelname: 'a', kind: /** @type {const} */ ('static'), sourceRelname: 'a' } + registry.claimRecords([record, { ...record }]) + assert.equal(registry.snapshot().length, 2) + for (const path of ['../bad', '/bad', 'C:\\bad', '.']) assert.throws(() => registry.claim(path, owner)) +}) + +/** @param {() => boolean | Promise} predicate */ +async function waitFor (predicate) { + const deadline = Date.now() + 5000 + while (!await predicate()) { + assert.ok(Date.now() < deadline, 'Timed out waiting for the watch result') + await new Promise(resolve => setTimeout(resolve, 25)) + } +} + +/** @param {DomStack} site */ +async function settle (site) { + await new Promise(resolve => setTimeout(resolve, 850)) + await site.settled() +} + +test('filtered watch conflicts retain successful outputs and recover after template renames and removals', { timeout: 30_000 }, async t => { + const { site, src, dest, logs } = await setup(t, { + 'a.template.js': template('a.txt', 'A'), + 'b.template.js': template('b.txt', 'B'), + }) + await site.watch({ serve: false }) + await writeFile(join(src, 'a.template.js'), template('b.txt', 'conflict')) + await settle(site) + assert.ok(logs.some(line => line.includes('Output path conflict'))) + assert.equal(await readFile(join(dest, 'a.txt'), 'utf8'), 'A') + assert.equal(await readFile(join(dest, 'b.txt'), 'utf8'), 'B') + await writeFile(join(src, 'a.template.js'), template('c.txt', 'C')) + await settle(site) + await assert.rejects(stat(join(dest, 'a.txt')), { code: 'ENOENT' }) + assert.equal(await readFile(join(dest, 'c.txt'), 'utf8'), 'C') + await rename(join(src, 'a.template.js'), join(src, 'renamed.template.js')) + await settle(site) + assert.equal(await readFile(join(dest, 'c.txt'), 'utf8'), 'C') + await rm(join(src, 'renamed.template.js')) + await settle(site) + await assert.rejects(stat(join(dest, 'c.txt')), { code: 'ENOENT' }) + await writeFile(join(src, 'b.template.js'), template('c.txt', 'reclaimed')) + await settle(site) + assert.equal(await readFile(join(dest, 'c.txt'), 'utf8'), 'reclaimed') + await assert.rejects(stat(join(dest, 'b.txt')), { code: 'ENOENT' }) +}) + +test('copy watch rejects page-owned paths and releases removed copy outputs', { timeout: 30_000 }, async t => { + const { tmp, src, dest, logs, sites } = await setup(t, { 'page.html': 'page' }) + const copy = join(tmp, 'copy') + await writeFiles(copy, { 'copy.txt': 'old' }) + const site = new DomStack(src, dest, { copy: [copy], logger: pino({}, { write: line => logs.push(line) }) }) + sites.push(site) + await site.watch({ serve: false }) + const page = await readFile(join(dest, 'index.html'), 'utf8') + await writeFiles(copy, { 'index.html': 'collision' }) + await waitFor(() => logs.some(line => line.includes('Output path conflict'))) + await settle(site) + assert.equal(await readFile(join(dest, 'index.html'), 'utf8'), page) + assert.ok(logs.some(line => line.includes('Output path conflict'))) + await rm(join(copy, 'index.html')) + await rename(join(copy, 'copy.txt'), join(copy, 'renamed.txt')) + await settle(site) + await assert.rejects(stat(join(dest, 'copy.txt')), { code: 'ENOENT' }) + assert.equal(await readFile(join(dest, 'renamed.txt'), 'utf8'), 'old') +}) + +test('service worker sourcemaps retain phase ownership across rebuilds and removal', { timeout: 30_000 }, async t => { + const { site, src, dest, logs } = await setup(t, { + 'page.html': 'page', + 'client.js': 'console.log(1)', + 'service-worker.js': 'console.log(1)', + 'a.template.js': template('a.txt'), + }) + await site.watch({ serve: false }) + await writeFile(join(src, 'service-worker.js'), 'console.log(2)') + await writeFile(join(src, 'client.js'), 'console.log(2)') + await settle(site) + assert.ok(!logs.some(line => line.includes('Output path conflict'))) + await writeFile(join(src, 'a.template.js'), template('service-worker.js.map')) + await settle(site) + assert.ok(logs.some(line => line.includes('Output path conflict'))) + assert.ok((await readFile(join(dest, 'service-worker.js.map'), 'utf8')).includes('sources')) + await rm(join(src, 'service-worker.js')) + await settle(site) + await writeFile(join(src, 'a.template.js'), template('service-worker.js.map', 'reclaimed')) + await settle(site) + assert.equal(await readFile(join(dest, 'service-worker.js.map'), 'utf8'), 'reclaimed') +}) + +test('initial watch conflicts leave the previous destination untouched and recover', { timeout: 30_000 }, async t => { + const { site, src, dest, logs } = await setup(t, { + 'page.html': 'new page', + 'client.js': 'console.log(1)', + 'a.template.js': template('client.js', 'collision'), + }) + await writeFiles(dest, { 'index.html': 'old page', 'client.js': 'old bundle' }) + await site.watch({ serve: false }) + assert.ok(logs.some(line => line.includes('Output path conflict'))) + assert.equal(await readFile(join(dest, 'index.html'), 'utf8'), 'old page') + assert.equal(await readFile(join(dest, 'client.js'), 'utf8'), 'old bundle') + await writeFile(join(src, 'a.template.js'), template('safe.txt', 'recovered')) + await settle(site) + assert.equal(await readFile(join(dest, 'index.html'), 'utf8'), 'new page') + assert.equal(await readFile(join(dest, 'safe.txt'), 'utf8'), 'recovered') + assert.match(await readFile(join(dest, 'client.js'), 'utf8'), /console.log/) +}) + +test('page promotion revalidates ownership acquired by esbuild during rendering', { timeout: 30_000 }, async t => { + const { site, src, dest, logs } = await setup(t, { + 'page.html': 'page', + 'client.js': 'console.log(1)', + 'payload.bin': 'esbuild asset', + 'a.template.js': template('a.txt', 'old template'), + 'esbuild.settings.js': "export default opts => ({ ...opts, assetNames: '[name]', loader: { ...opts.loader, '.bin': 'file' } })", + }, { static: false }) + await site.watch({ serve: false }) + await writeFile(join(src, 'a.template.js'), `import { writeFile } from 'node:fs/promises' +export default async () => { + await writeFile(new URL('./.rendering', import.meta.url), '') + await new Promise(resolve => setTimeout(resolve, 1200)) + return { outputName: 'payload.bin', content: 'template collision' } +}`) + for (let i = 0; i < 100; i++) { + if (await stat(join(src, '.rendering')).then(() => true, () => false)) break + await new Promise(resolve => setTimeout(resolve, 20)) + } + await stat(join(src, '.rendering')) + await writeFile(join(src, 'client.js'), "import asset from './payload.bin'; console.log(asset)") + await settle(site) + assert.equal(await readFile(join(dest, 'payload.bin'), 'utf8'), 'esbuild asset') + assert.equal(await readFile(join(dest, 'a.txt'), 'utf8'), 'old template') + assert.ok(logs.some(line => line.includes('Output path conflict'))) + await writeFile(join(src, 'a.template.js'), template('a.txt', 'recovered')) + await settle(site) + assert.equal(await readFile(join(dest, 'a.txt'), 'utf8'), 'recovered') + assert.equal(await readFile(join(dest, 'payload.bin'), 'utf8'), 'esbuild asset') +}) + +test('a producer can replace its own file with a directory and back', { timeout: 30_000 }, async t => { + const { site, src, dest } = await setup(t, { 'a.template.js': template('feed', 'file') }) + await site.watch({ serve: false }) + await writeFile(join(src, 'a.template.js'), template('feed/index.xml', 'nested')) + await settle(site) + assert.equal(await readFile(join(dest, 'feed/index.xml'), 'utf8'), 'nested') + await writeFile(join(src, 'a.template.js'), template('feed', 'file again')) + await settle(site) + assert.equal(await readFile(join(dest, 'feed'), 'utf8'), 'file again') +}) + +test('manifest hooks claim outputs and receive the public destination', async t => { + const { src, dest } = await setup(t, { 'page.html': 'page' }) + const site = new DomStack(src, dest, { + domstackManifest: { + hooks: { + manifestBuilt: [async ({ dest: hookDest, writeFile }) => { + assert.equal(hookDest, dest) + await writeFile('index.html', 'conflict') + }], + }, + }, + }) + await assert.rejects(site.build(), error => !!conflict(error)) + assert.equal(await readFile(join(dest, 'index.html'), 'utf8'), 'page') +}) + +test('one-shot hooks can immediately read their writes at the public destination', async t => { + const { src, dest } = await setup(t, { 'page.html': 'page' }) + await writeFiles(dest, { 'hook.txt': 'old' }) + const site = new DomStack(src, dest, { + domstackManifest: { + hooks: { + manifestBuilt: [async ({ dest: hookDest, writeFile }) => { + await writeFile('hook.txt', 'new') + assert.equal(await readFile(join(hookDest, 'hook.txt'), 'utf8'), 'new') + await writeFile('brand-new.txt', 'first write') + assert.equal(await readFile(join(hookDest, 'brand-new.txt'), 'utf8'), 'first write') + }], + }, + }, + }) + await site.build() +}) + +for (const watch of [false, true]) { + test(`copy reports keep live public mappers (full-watch staging: ${watch})`, async t => { + const { src, dest, tmp } = await setup(t, { 'asset.txt': 'static', 'client.js': 'console.log(1)', 'service-worker.js': 'console.log(2)' }) + const copyDir = join(tmp, 'copy') + await writeFiles(copyDir, { 'copied.txt': 'copy' }) + const result = await builder(src, dest, { copy: [copyDir] }, { watch }) + for (const value of [result.staticResults?.report, ...Object.values(result.copyResults?.report ?? {})]) { + const report = /** @type {{ options: { outputDir: string, toDestination: (source: string) => string }, copied: { source: string, output: string }[] }} */ (value) + assert.equal(report.options.outputDir, dest) + for (const file of report.copied) { + assert.equal(report.options.toDestination(file.source), file.output) + assert.equal(await readFile(file.output, 'utf8'), await readFile(file.source, 'utf8')) + } + } + assert.doesNotMatch(JSON.stringify(result), /\.domstack-(stage|copy|pages)-[a-zA-Z0-9]{6}/) + for (const name of (await readdir(dest)).filter(name => name.endsWith('.map'))) { + const map = JSON.parse(await readFile(join(dest, name), 'utf8')) + for (const source of map.sources) await stat(resolve(dest, source)) + } + }) +} + +for (const sameContents of [false, true]) { + test(`esbuild entry aliases identify both sources (identical contents: ${sameContents})`, async t => { + const { site } = await setup(t, { + 'a.js': 'console.log(1)', + 'b.js': sameContents ? 'console.log(1)' : 'console.log(2)', + 'esbuild.settings.js': 'export default opts => ({ ...opts, entryPoints: [new URL(\'./a.js\', import.meta.url).pathname, new URL(\'./b.js\', import.meta.url).pathname], entryNames: \'shared\' })', + }) + await assert.rejects(site.build(), error => { + const found = conflict(error) + assert.ok(found) + assert.match(found.message, /a\.js/) + assert.match(found.message, /b\.js/) + assert.match(found.message, /shared\.js/) + return true + }) + }) +} + +for (const paths of [['Feed.xml', 'feed.xml'], ['Feed', 'feed/index.xml']]) { + test(`copy inventory checks case aliases before staging: ${paths.join(', ')}`, async t => { + const { src, dest } = await setup(t, {}) + if (await isCaseInsensitiveDest(src)) return t.skip('Source filesystem cannot represent both aliases') + await writeFiles(src, Object.fromEntries(paths.map((path, i) => [path, String(i)]))) + const registry = new OutputRegistry([], { caseInsensitive: true }) + await assert.rejects(stagedCopy(join(src, '**'), src, dest, 'copy', registry), error => !!conflict(error)) + assert.ok(paths[0]) + await assert.rejects(stat(join(dest, paths[0])), { code: 'ENOENT' }) + }) +} + +test('worker returns a replacement delta including empty factories and deleted producers without dependency tracking', async t => { + const { src, dest } = await setup(t, { 'empty.pages.js': 'export default []' }) + const emptyOwner = `pages-file:${join(src, 'empty.pages.js')}` + const deletedOwner = `template:${join(src, 'deleted.template.js')}` + const previousOutputClaims = [ + { outputRelname: 'old.html', owner: { id: emptyOwner, type: 'page', path: 'empty.pages.js' } }, + { outputRelname: 'deleted.txt', owner: { id: deletedOwner, type: 'template', path: 'deleted.template.js' } }, + { outputRelname: 'asset.txt', owner: { id: 'static:asset.txt', type: 'static', path: 'asset.txt' } }, + ] + const result = await buildPages(src, dest, await identifyPages(src), { previousOutputClaims }) + assert.deepEqual(result.errors, []) + assert.deepEqual(result.report.newClaims, []) + assert.deepEqual(new Set(result.report.replacedOwnerIds), new Set([emptyOwner, deletedOwner])) + assert.ok(!('outputClaims' in result.report)) +}) + +test('watch uses its live copy inventory when files are renamed before initial rendering', { timeout: 30_000 }, async t => { + const { site, src, dest, logs } = await setup(t, { + 'old.txt': 'copied', + 'a.template.js': template('old.txt', 'page now owns old path'), + 'esbuild.settings.js': `import { access, writeFile } from 'node:fs/promises' +export default async opts => { + await writeFile(new URL('./.esbuild-started', import.meta.url), '') + while (!await access(new URL('./.continue', import.meta.url)).then(() => true, () => false)) await new Promise(resolve => setTimeout(resolve, 20)) + return opts +}`, + }, { ignore: ['.esbuild-started', '.continue'] }) + const startup = site.watch({ serve: false }) + await waitFor(() => stat(join(src, '.esbuild-started')).then(() => true, () => false)) + await rename(join(src, 'old.txt'), join(src, 'new.txt')) + const stageName = (await readdir(dest)).find(name => name.startsWith('.domstack-copy-watch-')) + assert.ok(stageName) + const stagedOld = join(dest, stageName, createHash('sha256').update(join(src, 'old.txt')).digest('hex')) + await waitFor(() => stat(stagedOld).then(() => false, () => true)) + await waitFor(() => logs.some(line => line.includes('Copy ') && line.includes('new.txt'))) + await writeFile(join(src, '.continue'), '') + await startup + assert.equal(await readFile(join(dest, 'old.txt'), 'utf8'), 'page now owns old path') + assert.equal(await readFile(join(dest, 'new.txt'), 'utf8'), 'copied') + assert.ok(!logs.some(line => line.includes('Output path conflict'))) + assert.equal(logs.filter(line => line.includes('Copy ') && line.includes('old.txt')).length, 1) +}) + +test('copy changes during initial rendering and onInitialBuild are drained', { timeout: 30_000 }, async t => { + const { site, src, dest, logs } = await setup(t, { + 'asset.txt': 'initial', + 'a.template.js': `import { writeFile } from 'node:fs/promises' +export default async () => { + await writeFile(new URL('./asset.txt', import.meta.url), 'during render') + await new Promise(resolve => setTimeout(resolve, 500)) + return { outputName: 'page.txt', content: 'page' } +}`, + }) + await site.watch({ + serve: false, + onInitialBuild: async () => { + assert.equal(await readFile(join(dest, 'asset.txt'), 'utf8'), 'during render') + const before = logs.filter(line => line.includes('Copy ') && line.includes('asset.txt')).length + await writeFile(join(src, 'asset.txt'), 'during callback') + await waitFor(() => logs.filter(line => line.includes('Copy ') && line.includes('asset.txt')).length > before) + } + }) + await site.settled() + assert.equal(await readFile(join(dest, 'asset.txt'), 'utf8'), 'during callback') +}) + +test('watch recovers after a page promotion I/O failure once the obstruction is removed', { timeout: 30_000 }, async t => { + const { site, src, dest, logs } = await setup(t, { 'a.template.js': template('old.txt', 'old') }) + await site.watch({ serve: false }) + await writeFiles(dest, { 'blocked.txt/unowned.txt': 'keep' }) + await writeFile(join(src, 'a.template.js'), template('blocked.txt', 'new')) + await settle(site) + assert.equal(await readFile(join(dest, 'blocked.txt/unowned.txt'), 'utf8'), 'keep') + assert.ok(logs.some(line => line.includes('EISDIR'))) + await rm(join(dest, 'blocked.txt'), { recursive: true }) + await writeFile(join(src, 'a.template.js'), template('blocked.txt', 'recovered')) + await settle(site) + assert.equal(await readFile(join(dest, 'blocked.txt'), 'utf8'), 'recovered') + await assert.rejects(stat(join(dest, 'old.txt')), { code: 'ENOENT' }) +}) + +test('native esbuild CSS bundle collisions identify both entry sources', async t => { + const { site } = await setup(t, { + 'a/client.js': "import './imported.css'", + 'a/imported.css': 'body { color: red }', + 'b/client.css': 'body { color: blue }', + 'esbuild.settings.js': 'export default opts => ({ ...opts, entryPoints: [new URL(\'./a/client.js\', import.meta.url).pathname, new URL(\'./b/client.css\', import.meta.url).pathname], entryNames: \'[name]\' })', + }) + await assert.rejects(site.build(), error => { + const found = conflict(error) + assert.ok(found, String(error)) + assert.match(found.message, /a\/client\.js/) + assert.match(found.message, /b\/client\.css/) + assert.match(found.message, /client\.css/) + return true + }) +}) + +test('one-shot and full-watch stages support a symlinked destination', async t => { + const { src, dest, tmp, site } = await setup(t, { 'page.html': 'page', 'asset.txt': 'static' }) + const actualDest = join(tmp, 'actual') + await mkdir(actualDest) + await symlink(actualDest, dest, 'dir') + for (const watch of [false, true]) { + const result = await builder(src, dest, {}, { watch }) + assert.equal(result.pageBuildResults?.outputs[0]?.filepath, join(dest, 'index.html')) + assert.equal(await readFile(join(actualDest, 'index.html'), 'utf8'), 'page') + assert.ok(!(await readdir(actualDest)).some(name => name.startsWith('.domstack-'))) + } + await site.watch({ serve: false }) + assert.equal(await readFile(join(actualDest, 'asset.txt'), 'utf8'), 'static') + await site.stopWatching() + assert.ok(!(await readdir(actualDest)).some(name => name.startsWith('.domstack-'))) +}) + +for (const watch of [false, true]) { + test(`failed page reports describe public paths (full-watch staging: ${watch})`, async t => { + const { src, dest } = await setup(t, { + 'page.html': 'successful render', + 'a.template.js': `export default async () => { + await new Promise(resolve => setTimeout(resolve, 100)) + throw new Error('render failed') +}`, + }) + await assert.rejects(builder(src, dest, {}, { watch }), error => { + assert.ok(error instanceof DomStackAggregateError) + assert.equal(error.results.pageBuildResults.report.pages[0].pageFilePath, join(dest, 'index.html')) + assert.doesNotMatch(JSON.stringify(error.results), /\.domstack-(stage|copy|pages)-[a-zA-Z0-9]{6}/) + return true + }) + await assert.rejects(stat(join(dest, 'index.html')), { code: 'ENOENT' }) + assert.ok(!(await readdir(dest)).some(name => name.startsWith('.domstack-'))) + }) +} + +for (const phase of ['copy', 'esbuild']) { + test(`initial ${phase} conflicts abort watch and clean up startup resources`, async t => { + const { src, dest, tmp, sites } = await setup(t, { + ...(phase === 'copy' + ? { 'asset.txt': 'static' } + : { + 'a.js': 'console.log(1)', + 'b.js': 'console.log(2)', + 'esbuild.settings.js': 'export default opts => ({ ...opts, entryPoints: [new URL(\'./a.js\', import.meta.url).pathname, new URL(\'./b.js\', import.meta.url).pathname], entryNames: \'shared\' })', + }), + }) + const copyDir = join(tmp, 'copy') + await writeFiles(copyDir, { 'asset.txt': 'copy' }) + const site = new DomStack(src, dest, { copy: phase === 'copy' ? [copyDir] : [], logger: pino({ level: 'silent' }) }) + sites.push(site) + await writeFiles(dest, { 'sentinel.txt': 'old destination' }) + await assert.rejects(site.watch({ serve: false }), error => !!conflict(error)) + assert.equal(site.watching, false) + assert.deepEqual(await readdir(dest), ['sentinel.txt']) + }) +} + +test('overlapping copy roots retain both mappings through watch startup, edits, full rebuilds, renames and removal', { timeout: 30_000 }, async t => { + const { src, dest, tmp, sites, logs } = await setup(t, { 'page.html': 'page' }) + const assets = join(tmp, 'assets') + const nested = join(assets, 'nested') + await writeFiles(nested, { 'asset.txt': 'initial' }) + const opts = { copy: [assets, nested], logger: pino({ level: 'debug' }, { write: line => logs.push(line) }) } + const site = new DomStack(src, dest, opts) + sites.push(site) + const assertMappings = async (/** @type {string} */ name, /** @type {string} */ content) => { + for (const path of [name, join('nested', name)]) assert.equal(await readFile(join(dest, path), 'utf8'), content) + } + const assertRemoved = async (/** @type {string} */ name) => { + for (const path of [name, join('nested', name)]) await assert.rejects(stat(join(dest, path)), { code: 'ENOENT' }) + } + await site.build() + await assertMappings('asset.txt', 'initial') + await rm(dest, { recursive: true }) + await site.watch({ serve: false }) + await assertMappings('asset.txt', 'initial') + await writeFile(join(nested, 'asset.txt'), 'edited') + await settle(site) + await assertMappings('asset.txt', 'edited') + + await writeFile(join(src, 'global.vars.js'), "export default { layout: 'root', rebuilt: true }") + await settle(site) + assert.ok(logs.some(line => line.includes('Triggering full rebuild'))) + await assertMappings('asset.txt', 'edited') + await writeFile(join(nested, 'asset.txt'), 'after full rebuild') + await settle(site) + await assertMappings('asset.txt', 'after full rebuild') + + await rename(join(nested, 'asset.txt'), join(nested, 'renamed.txt')) + await settle(site) + await assertRemoved('asset.txt') + await assertMappings('renamed.txt', 'after full rebuild') + await rm(join(nested, 'renamed.txt')) + await settle(site) + await assertRemoved('renamed.txt') + await writeFile(join(nested, 'asset.txt'), 'recreated') + await settle(site) + await assertMappings('asset.txt', 'recreated') + assert.ok(!logs.some(line => line.includes('Output path conflict'))) + + // A new file in the outer root cannot take the inner mapping's output, and + // removing that rejected producer must not remove the successful mapping. + await writeFile(join(assets, 'asset.txt'), 'conflicting outer file') + await settle(site) + assert.ok(logs.some(line => line.includes('Output path conflict'))) + await assertMappings('asset.txt', 'recreated') + await rm(join(assets, 'asset.txt')) + await settle(site) + await assertMappings('asset.txt', 'recreated') +}) + +for (const repeatedRoot of [false, true]) { + test(`copy root mappings that emit the same output still conflict (repeated root: ${repeatedRoot})`, async t => { + const { src, dest, tmp, sites } = await setup(t, {}) + const assets = join(tmp, 'assets') + const nested = join(assets, 'nested') + await writeFiles(assets, { 'asset.txt': 'root', 'nested/asset.txt': 'nested' }) + const opts = { copy: [assets, repeatedRoot ? assets : nested], logger: pino({ level: 'silent' }) } + for (const watch of [false, true]) { + await assert.rejects(builder(src, dest, opts, { watch }), error => !!conflict(error)) + } + await rm(dest, { recursive: true, force: true }) + const site = new DomStack(src, dest, opts) + sites.push(site) + await assert.rejects(site.watch({ serve: false }), error => !!conflict(error)) + assert.equal(site.watching, false) + }) +} From d53fedf5f8334acf769f687325f54e7cedb0b99f Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Sun, 13 Sep 2026 19:09:32 -0700 Subject: [PATCH 2/8] Add page-owned additional outputs from page and layout hooks --- docs/layouts/README.md | 26 +++ docs/pages/README.md | 111 ++++++++- index.js | 50 ++++- .../additional-outputs-types.test.ts | 44 ++++ lib/build-pages/additional-outputs.js | 97 ++++++++ lib/build-pages/additional-outputs.test.js | 31 +++ lib/build-pages/index.js | 32 ++- .../page-builders/additional-output-writer.js | 115 ++++++++++ .../additional-output-writer.test.js | 138 ++++++++++++ lib/build-pages/page-builders/js/index.js | 13 +- lib/build-pages/page-builders/page-writer.js | 13 ++ .../page-data-additional-outputs.test.js | 210 ++++++++++++++++++ lib/build-pages/page-data.js | 79 ++++++- lib/builder.js | 25 ++- lib/domstack-manifest/schema.js | 1 + lib/domstack-manifest/schema.json | 1 + lib/helpers/additional-output-promotion.js | 68 ++++++ .../additional-output-promotion.test.js | 83 +++++++ lib/helpers/path.js | 2 +- test-cases/page-additional-outputs/helpers.js | 90 ++++++++ .../page-additional-outputs/index.test.js | 193 ++++++++++++++++ .../page-additional-outputs/promotion.test.js | 73 ++++++ .../page-additional-outputs/watch.test.js | 173 +++++++++++++++ types.ts | 9 + 24 files changed, 1645 insertions(+), 32 deletions(-) create mode 100644 lib/build-pages/additional-outputs-types.test.ts create mode 100644 lib/build-pages/additional-outputs.js create mode 100644 lib/build-pages/additional-outputs.test.js create mode 100644 lib/build-pages/page-builders/additional-output-writer.js create mode 100644 lib/build-pages/page-builders/additional-output-writer.test.js create mode 100644 lib/build-pages/page-data-additional-outputs.test.js create mode 100644 lib/helpers/additional-output-promotion.js create mode 100644 lib/helpers/additional-output-promotion.test.js create mode 100644 test-cases/page-additional-outputs/helpers.js create mode 100644 test-cases/page-additional-outputs/index.test.js create mode 100644 test-cases/page-additional-outputs/promotion.test.js create mode 100644 test-cases/page-additional-outputs/watch.test.js diff --git a/docs/layouts/README.md b/docs/layouts/README.md index 7f887c52..ac36cae1 100644 --- a/docs/layouts/README.md +++ b/docs/layouts/README.md @@ -63,6 +63,7 @@ DOMStack recognizes these exports from a layout module: | `default` | Yes | A synchronous or asynchronous [layout render function](#layout-render-function). | | `vars` | No | An object, or a sync/async function returning an object, providing [layout defaults](#layout-variables). | | `parentLayout` | No | A non-empty string naming the immediate outer layout; see [Declaring nested layouts](#declaring-nested-layouts). | +| `additionalOutputs` | No | A build-only hook declaring [page-owned additional outputs](../pages/#additional-outputs), such as raw Markdown or JSON sidecars. | ## Declaring nested layouts @@ -104,6 +105,31 @@ See [Data subscriptions in nested layouts](../cookbook/nested-layouts/#data-subs See [Compose nested layouts](../cookbook/nested-layouts/) for a complete example and asset guidance. +## Additional outputs + +A layout can define shared output policy for its source-backed pages by exporting `additionalOutputs`. +The current source page owns the files, even though the layout declares the hook. +For example, a documentation layout can publish raw Markdown alongside each rendered page: + +```js +// src/docs.layout.js +export default ({ children }) => children + +export async function additionalOutputs ({ page, vars }) { + if (page.type !== 'md' || vars.rawExport === false) return [] + return { + outputName: page.outputName.replace(/\.html$/, '.source.md'), + content: await page.readMarkdownContent(), + } +} +``` + +The filename is relative to the current page's output directory, not the layout directory. +Using the page's HTML filename avoids collisions when several loose Markdown pages share a directory. +Nested hooks run outermost layout → innermost layout → page, and each layout hook shares only that layout renderer's `vars.dataDeps` subscriptions. +Generated pages skip these hooks, including inherited layout hooks. +See [Additional outputs](../pages/#additional-outputs) for the complete API, companion modules, path rules, and watch behavior. + ## Layout variables Layouts may also export an optional [`vars` variable provider](../pages/#variable-providers) containing defaults for pages that use the layout: diff --git a/docs/pages/README.md b/docs/pages/README.md index b3868307..03ad07fa 100644 --- a/docs/pages/README.md +++ b/docs/pages/README.md @@ -319,7 +319,116 @@ export default async () => { Page variable files have higher precedence than `global.vars.ts` variables, but lower precedence than frontmatter or `vars` exports from `ts` pages. See [Variables](../../docs/pages/#variables) for the full variable cascade. -### Draft pages +## Additional outputs + +Source-backed pages can publish extra files alongside their normal HTML through a named `additionalOutputs` export. +Use a [layout hook](../layouts/#additional-outputs) for shared policy, a JS/TS page-module hook for executable pages, or the page's directly associated vars companion for page-specific Markdown, HTML, or JS/TS behavior. +The hook declares files; it must not write directly to the destination. + +```js +// src/article/page.js +export const vars = { + title: 'An article', + dataDeps: ['siteMetadata'], +} + +export default ({ vars }) => `

${vars.title}

` + +export const additionalOutputs = ({ vars, data }) => ({ + outputName: './metadata.json', + content: JSON.stringify({ title: vars.title, site: data.siteMetadata }), +}) +``` + +### Companion hooks + +The existing vars companion can export the same named hook without changing its default vars export: + +```js +// src/article/page.vars.js (alongside page.md) +export default { title: 'An article' } + +export async function* additionalOutputs ({ page, vars }) { + yield { + outputName: './source.md', + content: await page.readMarkdownContent(), + } + yield { + outputName: './metadata.json', + content: JSON.stringify({ title: vars.title }), + } +} +``` + +For HTML and JS/TS companions, return suitable text or JSON instead of calling `readMarkdownContent()`. +The hook is a named module export, not a property of resolved vars or executable Markdown frontmatter. +Only the directly associated companion provides a page-level hook; global vars and inherited directory vars do not provide hooks. +If both a JS/TS page module and its companion export `additionalOutputs`, the build fails with a provider-conflict error identifying both modules. +Choose one provider rather than relying on precedence. + +### Hook arguments and results + +Every hook receives `{ page, vars, data }`: + +- `page` is a read-only handle to the current source page, including `type`, `path`, `url`, `outputName`, `outputRelname`, `draft`, and read-only `pageFile` metadata. + Its `readMarkdownContent()` method reads the Markdown body with YAML frontmatter removed, without rendering Markdown, substituting Handlebars, or rewriting links. + The method throws for non-Markdown pages. + The handle does not expose rendering methods, output-writing methods, or another consumer's data. +- `vars` contains the fully resolved page variables and is read-only. +- `data` contains only the declaring renderer's subscribed global data. + Page-module and companion hooks share the page renderer's subscriptions; each layout hook shares that specific layout renderer's subscriptions. + Declare these with the existing static `vars.dataDeps` array convention, or `dataDeps` in the companion's default vars object. + There is no `additionalOutputsDataDeps` export, and undeclared keys are not implicitly available. + See [Data subscriptions](../data/#data-subscriptions). + +A hook returns one explicit `{ outputName: string, content: string }` record, an array of records, or an async iterable of records, directly or through a promise. +Only string content is supported; serialize JSON yourself. +Bare strings are invalid because there is no implicit filename. +An empty array or an async iterator that yields nothing declares no files for that hook. + +Applicable hooks execute in outermost layout → innermost layout → page order, and all their outputs are additive. +Returning `[]` from the page hook does not suppress layout outputs. +For an application-specific opt-out, have the layout inspect a resolved variable such as `rawExport: false` and return `[]` itself. +Hooks run only in the owning page's output-build phase, not when collection or global-data code calls `renderInnerPage()` or `renderFullPage()`. +Generated `*.pages.*` pages skip additional-output hooks entirely, including inherited layout hooks. + +### Output paths and failures + +Output names resolve beneath the configured destination, including custom destinations: + +| Output name | Resolution for a page at `docs/article/index.html` | +| --- | --- | +| `metadata.json` or `./metadata.json` | `docs/article/metadata.json` | +| `../source/article.md` | `docs/source/article.md` | +| `/raw/article.md` | `raw/article.md` at the destination root | + +A leading `/` means destination-root-relative, never filesystem-absolute. +Relative paths use the current page's output directory even when a layout declares them. +Parent traversal is allowed only while the resolved target remains inside the destination. +Escapes and invalid file targets fail the build. +These rules do not change existing template output-path semantics. + +Duplicate destinations fail even when content is identical, including duplicates between hooks and conflicts with normal HTML, other pages, templates, copied assets, or bundles. +There is no implicit override mechanism. +Hook, iterator, validation, and collision failures publish none of the staged page-phase outputs and do not clean up stale page outputs or replace prior ownership. +An iterator that throws after yielding records therefore cannot publish those earlier yields to the live destination. +This is a page-phase guarantee, not a transaction across every build phase or a rollback guarantee for filesystem I/O failures during publication. + +### Watch behavior and ownership + +Additional files belong to the source page and appear in page build reports and the build manifest. +Ownership tracking and cleanup also work when public build-manifest generation is disabled. +After a successful rebuild, DOMStack removes previously owned files no longer returned, including renamed outputs and files from removed hooks. +Source deletion, source rename, or draft exclusion also removes the page's old outputs. +Adding, editing, removing, or renaming a companion updates the owning page's hook and output set. + +Hooks rerun when the owning page rebuilds, including changes to applicable page or layout data subscriptions. +A dependency used only by a hook still triggers an HTML rebuild because both outputs rebuild together. +Byte-identical additional files are not rewritten in the live destination, but remain in the complete owned output set. +An article body edit can update that article's HTML and raw export without invalidating sibling raw exports; a shared navigation rebuild can rerun all affected hooks without changing unchanged raw-file mtimes. +Keep collection-wide search indexes and feeds in templates, while using these hooks for per-page artifacts. + +## Draft pages A complete draft page can use the same colocated files as a published page: diff --git a/index.js b/index.js index 7c0ad7e1..66d10793 100644 --- a/index.js +++ b/index.js @@ -38,6 +38,7 @@ import { createServer } from '@domstack/sync' import { find } from '@11ty/dependency-tree-typescript' import { assertInsideDest, toPosix } from './lib/helpers/path.js' +import { prepareAdditionalOutputPromotion } from './lib/helpers/additional-output-promotion.js' import { getCopyGlob } from './lib/build-static/index.js' import { getCopyDirs } from './lib/build-copy/index.js' import { isProcessedFile, globalBundleAssets, pageBundleAssets, layoutBundleAssets } from './lib/file-conventions.js' @@ -119,6 +120,8 @@ export class DomStack { #caseInsensitive = false /** @type {string | null} Unpublished initial watch outputs, retained for recovery. */ #initialStage = null + /** @type {Map} Sidecars staged by successful initial page transactions. */ + #initialAdditionalOutputs = new Map() // One session owns the resources above until shutdown finishes. // Normal path: absent → starting → watching → stopping → absent. @@ -308,7 +311,7 @@ export class DomStack { trackWatchDependencies: true, previousOutputClaims: this.#outputClaims, caseInsensitive: this.#caseInsensitive, - promoteOutputs: (report, write) => this.#promotePageOutputs(report, write), + promoteOutputs: (report, write, preflight) => this.#promotePageOutputs(report, write, preflight), }) this.#remapInitialPageReport(pageBuildResults) if (pageBuildResults.errors.length > 0) { @@ -477,15 +480,18 @@ export class DomStack { * watch build queue: esbuild callbacks also run during queued context startup. * @param {() => OutputClaim[]} nextClaims * @param {() => Promise} write + * @param {(removablePaths: string[]) => Promise} [preflight] */ - #commitOutputs (nextClaims, write) { + #commitOutputs (nextClaims, write, preflight) { const transaction = this.#outputLock.then(async () => { const next = nextClaims() const paths = new Set(next.map(claim => claim.outputRelname)) - for (const claim of this.#outputClaims) { - if (paths.has(claim.outputRelname)) continue - const writeDest = this.#initialStage ?? this.#dest - const target = resolve(writeDest, claim.outputRelname) + const stale = this.#outputClaims.filter(claim => !paths.has(claim.outputRelname)).map(claim => claim.outputRelname) + const writeDest = this.#initialStage ?? this.#dest + await prepareAdditionalOutputPromotion(writeDest, [], stale) + await preflight?.(stale) + for (const name of stale) { + const target = resolve(writeDest, name) assertInsideDest(writeDest, target) await rm(target, { force: true }) // Only remove empty directories; never recursively delete unowned files. @@ -505,7 +511,14 @@ export class DomStack { const publish = this.#outputLock.then(async () => { if (!this.#initialStage) return await mkdir(this.#dest, { recursive: true }) - await cp(this.#initialStage, await realpath(this.#dest), { recursive: true, force: true }) + const stage = this.#initialStage + const unchanged = await prepareAdditionalOutputPromotion(this.#dest, [...this.#initialAdditionalOutputs.values()]) + await cp(stage, await realpath(this.#dest), { + recursive: true, + force: true, + filter: source => !unchanged.has(toPosix(relative(stage, source))), + }) + this.#initialAdditionalOutputs.clear() await rm(this.#initialStage, { recursive: true, force: true }) this.#initialStage = null }) @@ -534,14 +547,27 @@ export class DomStack { * its possibly stale pre-render snapshot. * @param {PageBuildStepResult} report * @param {() => Promise} write + * @param {(removablePaths: string[]) => Promise} preflight */ - #promotePageOutputs (report, write) { + #promotePageOutputs (report, write, preflight) { return this.#commitOutputs(() => { const replaced = new Set(report.report.replacedOwnerIds ?? []) const registry = new OutputRegistry(this.#outputClaims, { replaceOwnerIds: replaced, caseInsensitive: this.#caseInsensitive }) for (const claim of report.report.newClaims ?? []) registry.claim(claim.outputRelname, claim.owner) return registry.snapshot() - }, write) + }, async () => { + await write() + if (this.#initialStage) { + const replaced = new Set(report.report.replacedOwnerIds ?? []) + const removed = new Set(this.#outputClaims.filter(claim => replaced.has(claim.owner.id)).map(claim => claim.outputRelname)) + for (const name of removed) this.#initialAdditionalOutputs.delete(name) + for (const output of report.outputs) { + if (output.kind === 'page-additional') { + this.#initialAdditionalOutputs.set(output.outputRelname, { ...output, filepath: resolve(this.#initialStage, output.outputRelname) }) + } + } + } + }, preflight) } /** @@ -598,11 +624,12 @@ export class DomStack { const results = await builder(this.#src, this.#dest, { ...this.opts, domstackManifest: false }, { watch: true, caseInsensitive: this.#caseInsensitive, - promoteOutputs: (claims, write) => this.#commitOutputs(() => claims, write), + promoteOutputs: (claims, write, preflight) => this.#commitOutputs(() => claims, write, preflight), }) if (this.#initialStage) { await rm(this.#initialStage, { recursive: true, force: true }) this.#initialStage = null + this.#initialAdditionalOutputs.clear() } const { siteData, pageBuildResults } = results this.#siteData = siteData @@ -734,7 +761,7 @@ export class DomStack { trackWatchDependencies: true, previousOutputClaims: this.#outputClaims, caseInsensitive: this.#caseInsensitive, - promoteOutputs: (report, write) => this.#promotePageOutputs(report, write), + promoteOutputs: (report, write, preflight) => this.#promotePageOutputs(report, write, preflight), }) this.#remapInitialPageReport(pageBuildResults) if (pageBuildResults.errors.length > 0) { @@ -971,6 +998,7 @@ export class DomStack { await this.#outputLock results.push(...await Promise.allSettled([...this.#cpxWatchStages, ...(this.#initialStage ? [this.#initialStage] : [])].map(stage => rm(stage, { recursive: true, force: true })))) this.#initialStage = null + this.#initialAdditionalOutputs.clear() this.#watcher = null this.#cpxWatchers = [] this.#cpxWatchStages = [] diff --git a/lib/build-pages/additional-outputs-types.test.ts b/lib/build-pages/additional-outputs-types.test.ts new file mode 100644 index 00000000..56e62c76 --- /dev/null +++ b/lib/build-pages/additional-outputs-types.test.ts @@ -0,0 +1,44 @@ +import type { + AdditionalOutput, + AdditionalOutputProvenance, + AdditionalOutputsFunction, + AdditionalOutputsFunctionParams, + AdditionalOutputsPage, + AdditionalOutputsResult, + CollectedAdditionalOutput, + PageData, +} from '../../types.ts' + +// Compile-only assertions for the public type entry and the narrow hook contract. +export function checkAdditionalOutputsTypes (pageData: PageData<{ title: string }>, page: AdditionalOutputsPage) { + const output: AdditionalOutput = { outputName: 'feed.json', content: '' } + const provenance: AdditionalOutputProvenance = { kind: 'layout', source: 'base.layout.ts', layoutName: 'base' } + const collected: CollectedAdditionalOutput = { ...output, provenance } + const result: AdditionalOutputsResult = [output] + const hook: AdditionalOutputsFunction<{ title: string }, { posts: string[] }> = async ({ page, vars, data }) => ({ + outputName: 'feed.json', content: vars.title + page.url + data.posts.join(','), + }) + const params: AdditionalOutputsFunctionParams<{ title: string }, { posts: string[] }> = { page, vars: { title: 'title' }, data: { posts: [] } } + const promise: Promise = pageData.collectAdditionalOutputs() + const iterator: AdditionalOutputsFunction = async function * () { yield output } + const promisedIterator: AdditionalOutputsFunction = async () => (async function * () { yield output })() + // @ts-expect-error Bare strings are not hook results. + const badHook: AdditionalOutputsFunction = () => 'html' + // @ts-expect-error Content must be a string. + const badOutput: AdditionalOutput = { outputName: 'feed.json', content: {} } + // @ts-expect-error Page metadata is read-only. + page.url = '/other' + // @ts-expect-error Source metadata is read-only. + page.pageFile.filepath = '/other.md' + // @ts-expect-error Hooks cannot render pages. + page.renderFullPage() + // @ts-expect-error Hooks cannot access a PageData instance. + const bypass = page.pageInfo + // @ts-expect-error Global data is only accessible through the subscribed params.data. + const globalData = page.data + // @ts-expect-error Vars are read-only. + params.vars.title = 'other' + // @ts-expect-error Only declared data is available. + const secret = params.data.secret + return { hook, params, promise, collected, result, iterator, promisedIterator, badHook, badOutput, bypass, globalData, secret } +} diff --git a/lib/build-pages/additional-outputs.js b/lib/build-pages/additional-outputs.js new file mode 100644 index 00000000..a33e351f --- /dev/null +++ b/lib/build-pages/additional-outputs.js @@ -0,0 +1,97 @@ +/** + * @import { PageInfo } from '../identify-pages.js' + * + * @typedef {object} AdditionalOutput + * @property {string} outputName + * @property {string} content + * + * @typedef {object} AdditionalOutputProvenance + * @property {'page' | 'companion' | 'layout'} kind + * @property {string} source - Provider module path (layout name when no path is available). + * @property {string} [layoutName] + * + * @typedef {AdditionalOutput & { provenance: AdditionalOutputProvenance }} CollectedAdditionalOutput + * @typedef {AdditionalOutput | AdditionalOutput[] | AsyncIterable} AdditionalOutputsResult + * @typedef {Readonly> & { readonly pageFile: Readonly, readonly readMarkdownContent: () => Promise }} AdditionalOutputsPage + */ + +/** + * @template {Record} [T=Record] + * @template {object} [D=Record] + * @typedef {object} AdditionalOutputsFunctionParams + * @property {AdditionalOutputsPage} page - Read-only source metadata, without rendering or global-data access. + * @property {Readonly} vars + * @property {D} data - The same subscriptions as this provider's renderer. + */ + +/** + * @template {Record} [T=Record] + * @template {object} [D=Record] + * @callback AdditionalOutputsFunction + * @param {AdditionalOutputsFunctionParams} params + * @returns {AdditionalOutputsResult | Promise} + */ + +/** + * @param {unknown} hook + * @param {string} source + * @returns {AdditionalOutputsFunction | undefined} + */ +export function validateAdditionalOutputsHook (hook, source) { + if (hook === undefined) return undefined + if (typeof hook !== 'function') throw new TypeError(`additionalOutputs in "${source}" must be a function`) + return /** @type {AdditionalOutputsFunction} */ (hook) +} + +/** + * Normalize one provider's result, preserving order and diagnostic context. + * Output destination containment and collisions are enforced by the output writer. + * @param {unknown} result + * @param {AdditionalOutputProvenance} provenance + * @returns {Promise} + */ +export async function normalizeAdditionalOutputs (result, provenance) { + /** @type {CollectedAdditionalOutput[]} */ + const outputs = [] + /** @param {unknown} record */ + const append = (record) => { + if (!record || typeof record !== 'object' || + !('outputName' in record) || typeof record.outputName !== 'string' || !record.outputName.trim() || + !('content' in record) || typeof record.content !== 'string') { + throw new TypeError(`Record ${outputs.length + 1} must be { outputName: non-empty string, content: string }`) + } + outputs.push({ outputName: record.outputName, content: record.content, provenance: { ...provenance } }) + } + try { + const resolved = await result + if (Array.isArray(resolved)) { + for (const record of resolved) append(record) + } else if (resolved && typeof resolved === 'object' && Symbol.asyncIterator in resolved) { + for await (const record of /** @type {AsyncIterable} */ (resolved)) append(record) + } else { + append(resolved) + } + return outputs + } catch (cause) { + throw new Error(`Invalid additionalOutputs from ${provenance.kind} "${provenance.source}": ${cause instanceof Error ? cause.message : String(cause)}`, { cause }) + } +} + +/** + * @param {PageInfo} info + * @param {() => Promise} readMarkdownContent - Bound to the source, never to caller-supplied metadata. + * @returns {AdditionalOutputsPage} + */ +export function createAdditionalOutputsPage (info, readMarkdownContent) { + const { type, path, url, outputName, outputRelname, draft, pageFile } = info + return Object.freeze({ + type, + path, + url, + outputName, + outputRelname, + draft, + pageFile: Object.freeze({ ...pageFile }), + readMarkdownContent, + }) +} diff --git a/lib/build-pages/additional-outputs.test.js b/lib/build-pages/additional-outputs.test.js new file mode 100644 index 00000000..06b25bf4 --- /dev/null +++ b/lib/build-pages/additional-outputs.test.js @@ -0,0 +1,31 @@ +/** @import { AdditionalOutputProvenance } from './additional-outputs.js' */ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { normalizeAdditionalOutputs, validateAdditionalOutputsHook } from './additional-outputs.js' + +/** @type {AdditionalOutputProvenance} */ +const provenance = { kind: 'page', source: '/src/page.ts' } +const record = { outputName: 'feed.json', content: '' } + +test('additionalOutputs normalizes records, arrays, promises and async iterables', async () => { + const expected = [{ ...record, provenance }] + assert.deepEqual(await normalizeAdditionalOutputs(record, provenance), expected) + assert.deepEqual(await normalizeAdditionalOutputs(Promise.resolve([record]), provenance), expected) + async function * records () { yield record; yield { ...record, outputName: 'second.json' } } + assert.deepEqual(await normalizeAdditionalOutputs(Promise.resolve(records()), provenance), [...expected, { ...record, outputName: 'second.json', provenance }]) + assert.deepEqual(await normalizeAdditionalOutputs([], provenance), []) + async function * empty () {} + assert.deepEqual(await normalizeAdditionalOutputs(empty(), provenance), []) +}) + +test('additionalOutputs rejects invalid hooks and records with source context', async () => { + for (const hook of [null, true, {}, 'content']) { + assert.throws(() => validateAdditionalOutputsHook(hook, provenance.source), /additionalOutputs.*\/src\/page.ts.*function/) + } + for (const result of ['bare string', undefined, null, {}, { outputName: '', content: '' }, { outputName: 'x', content: 1 }, [record, 'bad'], new Set([record])]) { + await assert.rejects(normalizeAdditionalOutputs(result, provenance), /Invalid additionalOutputs.*\/src\/page.ts.*Record/) + } + async function * broken () { yield record; throw new Error('iterator failed') } + await assert.rejects(normalizeAdditionalOutputs(broken(), provenance), /\/src\/page.ts.*iterator failed/) + await assert.rejects(normalizeAdditionalOutputs(Promise.reject(new Error('promise failed')), provenance), /\/src\/page.ts.*promise failed/) +}) diff --git a/lib/build-pages/index.js b/lib/build-pages/index.js index b7f71a7a..67ca76ab 100644 --- a/lib/build-pages/index.js +++ b/lib/build-pages/index.js @@ -27,6 +27,7 @@ import { createSubscribedData, resolveDataDeps } from './data-deps.js' import { WatchDependencyTracker as WatchDependencyTrackerClass } from './watch-dependencies.js' import { OutputRegistry, isCaseInsensitiveDest } from '../output-registry.js' import { assertInsideDest } from '../helpers/path.js' +import { prepareAdditionalOutputPromotion } from '../helpers/additional-output-promotion.js' const MAX_CONCURRENCY = Math.min(cpus().length, 24) @@ -98,7 +99,7 @@ const __dirname = import.meta.dirname * @property {OutputClaim[] | null | undefined} [previousOutputClaims] - Output ownership from the latest successful build. * @property {boolean} [caseInsensitive] - * @property {(report: PageBuildStepResult, write: () => Promise) => Promise} [promoteOutputs] + * @property {(report: PageBuildStepResult, write: () => Promise, preflight: (removablePaths: string[]) => Promise) => Promise} [promoteOutputs] */ /** @@ -488,9 +489,16 @@ export async function buildPages (src, dest, siteData, opts) { try { if (buildReport.errors.length === 0) { - const write = () => promotePageOutputs(dest, buildReport) - if (opts?.promoteOutputs) await opts.promoteOutputs(buildReport, write) - else await write() + let unchanged = new Set() + const preflight = async (/** @type {string[]} */ removablePaths) => { + unchanged = await prepareAdditionalOutputPromotion(dest, buildReport.outputs, removablePaths) + } + const write = () => promotePageOutputs(dest, buildReport, unchanged) + if (opts?.promoteOutputs) await opts.promoteOutputs(buildReport, write, preflight) + else { + await preflight([]) + await write() + } } return buildReport } finally { @@ -509,14 +517,17 @@ export async function buildPages (src, dest, siteData, opts) { * * @param {string} dest * @param {PageBuildStepResult} buildReport + * @param {Set} unchanged */ -async function promotePageOutputs (dest, buildReport) { +async function promotePageOutputs (dest, buildReport, unchanged) { const resolvedDest = resolve(dest) for (const output of buildReport.outputs) { const target = resolve(resolvedDest, output.outputRelname) assertInsideDest(resolvedDest, target) - await mkdir(dirname(target), { recursive: true }) - await copyFile(output.filepath, target) + if (!unchanged.has(output.outputRelname)) { + await mkdir(dirname(target), { recursive: true }) + await copyFile(output.filepath, target) + } output.filepath = target } } @@ -768,6 +779,13 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { const buildResult = await pageWriter({ dest, page, + claimAdditionalOutput: (outputRelname, provenance) => { + const owner = pageOutputOwner(page.pageInfo) + outputRegistry.claim(outputRelname, { + ...owner, + path: provenance ? `${owner.path} (${provenance.kind} additionalOutputs: ${provenance.source})` : owner.path, + }) + }, }) result.report.pages.push({ diff --git a/lib/build-pages/page-builders/additional-output-writer.js b/lib/build-pages/page-builders/additional-output-writer.js new file mode 100644 index 00000000..148a7863 --- /dev/null +++ b/lib/build-pages/page-builders/additional-output-writer.js @@ -0,0 +1,115 @@ +/** + * @import { PageInfo } from '../../identify-pages.js' + * @import { DomstackManifestRecord } from '../../domstack-manifest/index.js' + * @import { AdditionalOutputProvenance } from '../additional-outputs.js' + */ +import { lstat, mkdir, writeFile } from 'node:fs/promises' +import { dirname, isAbsolute, relative, resolve, sep } from 'node:path' +import { createDomstackManifestRecord } from '../../domstack-manifest/index.js' + +/** + * Resolve sidecar names independently of legacy page/template path semantics. + * Backslashes are separators on every platform; only a single leading forward + * slash means destination-root relative (drive paths and UNC paths are invalid). + * + * @param {string} dest + * @param {string} pageFilePath + * @param {string} outputName + */ +export function resolveAdditionalOutputPath (dest, pageFilePath, outputName) { + if (typeof outputName !== 'string' || !outputName.trim()) { + throw new TypeError('Additional outputName must be a non-empty file path') + } + const name = outputName.replaceAll('\\', '/') + if (outputName.startsWith('\\') || name.startsWith('//') || /^[a-z]:/i.test(name)) { + throw new Error(`Additional outputName must not be a drive or UNC path: ${outputName}`) + } + const parts = name.split('/') + if (!parts.at(-1) || ['.', '..'].includes(parts.at(-1) ?? '')) { + throw new Error(`Additional outputName must name a file: ${outputName}`) + } + for (const part of parts) { + if (!part || part === '.' || part === '..') continue + // Reject Windows aliases and special files even when building on POSIX. + if (/[<>:"|?*]/u.test(part) || Array.from(part).some(character => character.charCodeAt(0) < 32) || /[. ]$/.test(part) || /^(con|prn|aux|nul|com[1-9¹²³]|lpt[1-9¹²³])(?:\.|$)/i.test(part)) { + throw new Error(`Additional outputName contains an invalid file path component: ${outputName}`) + } + } + const filepath = name.startsWith('/') + ? resolve(dest, name.slice(1)) + : resolve(dirname(pageFilePath), name) + const relname = relative(resolve(dest), filepath) + if (!relname || relname === '..' || relname.startsWith(`..${sep}`) || isAbsolute(relname)) { + throw new Error(`Additional outputName escapes dest or names its directory: ${outputName}`) + } + return { filepath, outputRelname: relname.split(sep).join('/') } +} + +/** + * Check existing components, including the leaf, without following symlinks. + * The caller owns a private stage; this is not a defense against concurrent + * hostile filesystem mutations or a substitute for final promotion checks. + * + * @param {string} dest + * @param {string} filepath + */ +async function assertWritablePath (dest, filepath) { + let current = resolve(dest) + const components = relative(current, filepath).split(sep) + for (let index = -1; index < components.length; index++) { + const component = components[index] + if (component !== undefined) current = resolve(current, component) + let info + try { + info = await lstat(current) + } catch (error) { + if (/** @type {NodeJS.ErrnoException} */ (error).code === 'ENOENT') continue + throw error + } + if (info.isSymbolicLink()) throw new Error(`Additional output path contains a symlink: ${current}`) + const leaf = index === components.length - 1 + if (leaf ? !info.isFile() : !info.isDirectory()) { + throw new Error(`Additional output path is not a ${leaf ? 'file' : 'directory'}: ${current}`) + } + } +} + +/** + * Write only into the supplied build stage; ownership and promotion belong to + * build-pages. Extra hook provenance does not change the owning source page. + * + * @param {object} params + * @param {string} params.dest + * @param {string} params.pageFilePath + * @param {PageInfo} params.pageInfo + * @param {Array<{outputName: string, content: string, provenance?: AdditionalOutputProvenance}>} params.additionalOutputs + * @param {(outputRelname: string, provenance?: AdditionalOutputProvenance) => void} [params.claimOutput] + * @returns {Promise} + */ +export async function writeAdditionalOutputs ({ dest, pageFilePath, pageInfo, additionalOutputs, claimOutput }) { + const planned = additionalOutputs.map(output => { + if (typeof output.content !== 'string') throw new TypeError('Additional output content must be a string') + return { ...resolveAdditionalOutputPath(dest, pageFilePath, output.outputName), content: output.content, provenance: output.provenance } + }) + // Reserve the whole batch before touching the stage, including duplicate records. + for (const output of planned) claimOutput?.(output.outputRelname, output.provenance) + for (const output of planned) await assertWritablePath(dest, output.filepath) + const records = [] + for (const { filepath, outputRelname, content } of planned) { + await mkdir(dirname(filepath), { recursive: true }) + await writeFile(filepath, content) + records.push({ + ...createDomstackManifestRecord({ + dest, + filepath, + outputRelname, + kind: 'page-additional', + sourceRelname: pageInfo.pageFile.relname, + pagePath: pageInfo.path, + pageUrl: pageInfo.url, + }), + pagePath: pageInfo.path, + }) + } + return records +} diff --git a/lib/build-pages/page-builders/additional-output-writer.test.js b/lib/build-pages/page-builders/additional-output-writer.test.js new file mode 100644 index 00000000..837aed27 --- /dev/null +++ b/lib/build-pages/page-builders/additional-output-writer.test.js @@ -0,0 +1,138 @@ +/** + * @import { PageInfo } from '../../identify-pages.js' + * @import { PageData } from '../page-data.js' + */ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { resolveAdditionalOutputPath, writeAdditionalOutputs } from './additional-output-writer.js' +import { pageWriter } from './page-writer.js' +import { createEntry } from '../../domstack-manifest/records.js' + +const pageInfo = /** @type {PageInfo} */ ({ + path: 'posts', + outputName: 'index.html', + outputRelname: 'posts/index.html', + url: '/posts/', + pageFile: { relname: 'posts/page.js' }, +}) + +test('additional output paths use the actual page output directory and destination root', () => { + const dest = resolve('public') + const page = join(dest, 'posts', 'nested', 'index.html') + /** @type {[string, string][]} */ + const cases = [ + ['data.json', 'posts/nested/data.json'], + ['../feed.json', 'posts/feed.json'], + ['../../feed.json', 'feed.json'], + ['/feed.json', 'feed.json'], + ['..\\feed.json', 'posts/feed.json'], + ['./data.json', 'posts/nested/data.json'], + ['..hidden/data.json', 'posts/nested/..hidden/data.json'], + ] + for (const [name, expected] of cases) { + const result = resolveAdditionalOutputPath(dest, page, name) + assert.equal(result.outputRelname, expected) + assert.equal(result.filepath, join(dest, expected)) + } +}) + +test('additional output paths reject escapes, directory names and nonportable file names', () => { + const dest = resolve('public') + const page = join(dest, 'posts/index.html') + for (const name of ['', ' ', '/', '.', '..', 'foo/', 'foo\\', 'foo/.', 'foo/..', '../../escape.json', '/../escape.json', '//server/file', '\\file', '\\\\server\\file', 'C:/file', 'C:file', 'file:stream', 'file\u0000.json', 'file?', 'NUL.json', 'aux', 'COM1.txt', 'dir./file', 'dir /file']) { + assert.throws(() => resolveAdditionalOutputPath(dest, page, name), Error, name) + } +}) + +test('writer stages sidecars with page ownership and non-navigation JSON records', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-')) + t.after(() => rm(dest, { recursive: true, force: true })) + const outputs = await writeAdditionalOutputs({ + dest, + pageFilePath: join(dest, 'posts/index.html'), + pageInfo, + additionalOutputs: [{ outputName: '/feed.json', content: '{"ok":true}' }, { outputName: 'nested/data.txt', content: 'hello' }], + }) + assert.equal(await readFile(join(dest, 'feed.json'), 'utf8'), '{"ok":true}') + assert.equal(await readFile(join(dest, 'posts/nested/data.txt'), 'utf8'), 'hello') + assert.ok(outputs[0]) + assert.equal(outputs[0].kind, 'page-additional') + assert.equal(outputs[0].sourceRelname, 'posts/page.js') + assert.equal(outputs[0].pagePath, 'posts') + assert.equal(outputs[0].pageUrl, '/posts/') + assert.equal(outputs[0].url, '/feed.json') + assert.equal(outputs[0].page, undefined) + const entry = await createEntry({ dest, record: outputs[0] }) + assert.equal(entry?.role, 'subresource') +}) + +test('writer rejects symlink components and existing directories without touching their targets', async t => { + const root = await mkdtemp(join(tmpdir(), 'domstack-additional-links-')) + t.after(() => rm(root, { recursive: true, force: true })) + const dest = join(root, 'stage') + const outside = join(root, 'outside') + await mkdir(dest) + await mkdir(outside) + await writeFile(join(outside, 'data.json'), 'unchanged') + await symlink(outside, join(dest, 'linked'), 'dir') + await symlink(join(outside, 'data.json'), join(dest, 'leaf.json'), 'file') + await mkdir(join(dest, 'directory')) + for (const outputName of ['linked/data.json', 'leaf.json', 'directory']) { + await assert.rejects(writeAdditionalOutputs({ + dest, + pageFilePath: join(dest, 'index.html'), + pageInfo, + additionalOutputs: [{ outputName, content: 'changed' }], + }), /symlink|not a file/) + } + assert.equal(await readFile(join(outside, 'data.json'), 'utf8'), 'unchanged') +}) + +test('writer validates all sidecar paths before writing the batch', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-invalid-')) + t.after(() => rm(dest, { recursive: true, force: true })) + await assert.rejects(writeAdditionalOutputs({ + dest, + pageFilePath: join(dest, 'index.html'), + pageInfo, + additionalOutputs: [{ outputName: 'valid.json', content: '{}' }, { outputName: '../escape.json', content: '{}' }], + }), /escapes dest/) + await assert.rejects(readFile(join(dest, 'valid.json')), { code: 'ENOENT' }) +}) + +test('writer preserves duplicate records for main to reject through output claims', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-claims-')) + t.after(() => rm(dest, { recursive: true, force: true })) + const outputs = await writeAdditionalOutputs({ + dest, + pageFilePath: join(dest, 'index.html'), + pageInfo, + additionalOutputs: [{ outputName: 'data.json', content: 'first' }, { outputName: './data.json', content: 'second' }], + }) + assert.equal(outputs.length, 2) + assert.ok(outputs[0]) + assert.ok(outputs[1]) + assert.equal(outputs[0].outputRelname, outputs[1].outputRelname) + assert.equal(outputs[0].sourceRelname, outputs[1].sourceRelname) +}) + +test('page writer collects once after rendering and reports HTML and sidecars together', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-page-')) + t.after(() => rm(dest, { recursive: true, force: true })) + /** @type {string[]} */ + const events = [] + const page = /** @type {PageData} */ (/** @type {unknown} */ ({ + pageInfo, + vars: {}, + async renderFullPage () { events.push('render'); return '

Page

' }, + async collectAdditionalOutputs () { events.push('collect'); return [{ outputName: 'data.json', content: '{}' }] }, + })) + const result = await pageWriter({ dest, page }) + assert.deepEqual(events, ['render', 'collect']) + assert.deepEqual(result.outputs.map(output => output.kind), ['page', 'page-additional']) + assert.equal(await readFile(result.pageFilePath, 'utf8'), '

Page

') + assert.equal(await readFile(join(dest, 'posts/data.json'), 'utf8'), '{}') +}) diff --git a/lib/build-pages/page-builders/js/index.js b/lib/build-pages/page-builders/js/index.js index 53c7f62a..cf0a3b45 100644 --- a/lib/build-pages/page-builders/js/index.js +++ b/lib/build-pages/page-builders/js/index.js @@ -1,9 +1,15 @@ /** * @import { PageInfo } from '../../../identify-pages.js' * @import { PageBuilderResult } from '../page-writer.js' + * @import { AdditionalOutputsFunction } from '../../additional-outputs.js' + * + * @template {Record} T + * @template [U=any] + * @typedef {PageBuilderResult & { additionalOutputs?: AdditionalOutputsFunction }} JsPageBuilderResult */ import assert from 'node:assert' +import { validateAdditionalOutputsHook } from '../../additional-outputs.js' /** * Resolve a JavaScript page module. @@ -11,7 +17,7 @@ import assert from 'node:assert' * @template [U=any] U - The return type of the page function * @param {object} params * @param {PageInfo} params.pageInfo - * @returns {Promise>} + * @returns {Promise>} */ export async function jsBuilder ({ pageInfo }) { assert(pageInfo.type === 'js', 'js page builder requires "js" page type') @@ -26,10 +32,11 @@ export async function jsBuilder ({ pageInfo }) { } } - const { default: pageLayout, vars } = await import(pageInfo.pageFile.filepath) + const { default: pageLayout, vars, additionalOutputs } = await import(pageInfo.pageFile.filepath) assert(pageLayout, 'js pages must export a page layout default export') assert(typeof pageLayout === 'function', 'js pages pageLayout must be a function') - return { vars, pageLayout } + const hook = validateAdditionalOutputsHook(additionalOutputs, pageInfo.pageFile.filepath) + return { vars, pageLayout, ...(hook ? { additionalOutputs: hook } : {}) } } diff --git a/lib/build-pages/page-builders/page-writer.js b/lib/build-pages/page-builders/page-writer.js index 982e252b..53b52495 100644 --- a/lib/build-pages/page-builders/page-writer.js +++ b/lib/build-pages/page-builders/page-writer.js @@ -2,11 +2,13 @@ * @import { PageInfo } from '../../identify-pages.js' * @import { PageData as PageDataClass } from '../page-data.js' * @import { DomstackManifestRecord } from '../../domstack-manifest/index.js' + * @import { AdditionalOutputsFunction, AdditionalOutputProvenance } from '../additional-outputs.js' */ import { join } from 'path' import { writeFile, mkdir } from 'fs/promises' import { createDomstackManifestRecord } from '../../domstack-manifest/index.js' +import { writeAdditionalOutputs } from './additional-output-writer.js' /** * @typedef {Object} BuilderOptions @@ -74,6 +76,7 @@ import { createDomstackManifestRecord } from '../../domstack-manifest/index.js' * @typedef PageBuilderResult * @property {Partial} vars - Any variables resolved by the builder * @property {InternalPageFunction} pageLayout - The function that returns the rendered page + * @property {AdditionalOutputsFunction} [additionalOutputs] - Optional build-only additional-output hook. */ /** @@ -96,11 +99,13 @@ import { createDomstackManifestRecord } from '../../domstack-manifest/index.js' * @param {object} params * @param {string} params.dest - The dest folder. * @param {PageData} params.page - The PageInfo object of the current page + * @param {(outputRelname: string, provenance?: AdditionalOutputProvenance) => void} [params.claimAdditionalOutput] * @returns {Promise<{ pageFilePath: string, outputs: DomstackManifestRecord[] }>} */ export async function pageWriter ({ dest, page, + claimAdditionalOutput, }) { if (!page.pageInfo) throw new Error('Uninitialzied page detected') const pageDir = join(dest, page.pageInfo.path) @@ -163,6 +168,14 @@ export async function pageWriter ({ } } + outputs.push(...await writeAdditionalOutputs({ + dest, + pageFilePath, + pageInfo: page.pageInfo, + additionalOutputs: await page.collectAdditionalOutputs(), + claimOutput: claimAdditionalOutput, + })) + return { pageFilePath, outputs } } diff --git a/lib/build-pages/page-data-additional-outputs.test.js b/lib/build-pages/page-data-additional-outputs.test.js new file mode 100644 index 00000000..c5e145b4 --- /dev/null +++ b/lib/build-pages/page-data-additional-outputs.test.js @@ -0,0 +1,210 @@ +/** + * @import { PageInfo } from '../identify-pages.js' + * @import { ResolvedLayout } from './page-data.js' + * @import { TestContext } from 'node:test' + */ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { mkdtemp, writeFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PageData, resolveLayout } from './page-data.js' +import { identifyPages } from '../identify-pages.js' + +/** + * @param {TestContext} t + * @param {{ module?: string, companion?: string, extension?: string }} [options] + */ +async function fixture (t, { module, companion, extension = 'mjs' } = {}) { + const dir = await mkdtemp(join(tmpdir(), 'domstack-hooks-')) + t.after(() => rm(dir, { recursive: true, force: true })) + const filepath = join(dir, module ? `page.${extension}` : 'page.md') + await writeFile(filepath, module ?? '---\ntitle: Source\n---\n# Body\n') + const file = (/** @type {string} */ path) => ({ root: dir, filepath: join(dir, path), relname: path, basename: path, parentName: '' }) + /** @type {PageInfo} */ + const pageInfo = { + pageFile: file(module ? `page.${extension}` : 'page.md'), + type: module ? 'js' : 'md', + path: '', + url: '/', + outputName: 'index.html', + outputRelname: 'index.html', + draft: false, + } + if (companion !== undefined) { + await writeFile(join(dir, 'page.vars.mjs'), companion) + pageInfo.pageVars = file('page.vars.mjs') + } + const pd = new PageData({ pageInfo, globalVars: { layout: 'inner', additionalOutputs: () => { throw new Error('global vars are not providers') } }, globalStyle: undefined, globalClient: undefined, defaultStyle: null, defaultClient: null, builderOptions: {} }) + /** @type {Record>} */ + const layouts = { + outer: { name: 'outer', render: ({ children }) => children, vars: {}, layoutStylePath: null, layoutClientPath: null }, + inner: { name: 'inner', parentLayout: 'outer', render: ({ children }) => children, vars: {}, layoutStylePath: null, layoutClientPath: null }, + } + return { pd, layouts, dir } +} + +test('explicit collection runs outer -> inner -> page with renderer subscriptions, not on rendering', async t => { + const { pd, layouts, dir } = await fixture(t, { + module: `export const vars = { dataDeps: ['pageKey'] } + export let hookCalls = 0 + export default ({ data }) => data.pageKey + export const additionalOutputs = ({ page, vars, data }) => { + hookCalls++ + return { outputName: 'page.txt', content: data.pageKey + data.companionKey + page.url } + }`, + companion: "export default { dataDeps: ['companionKey'] }", + }) + /** @type {string[]} */ + const calls = [] + for (const name of ['outer', 'inner']) { + const path = join(dir, `${name}.layout.mjs`) + await writeFile(path, `export const vars = { dataDeps: ['${name}Key'] }; export default ({ children }) => children; + export const additionalOutputs = ({ data }) => ({ outputName: '${name}.txt', content: data.${name}Key });`) + const layout = layouts[name] + assert.ok(layout) + const resolved = await resolveLayout(path) + Object.assign(layout, resolved, { parentLayout: name === 'inner' ? 'outer' : undefined }) + const hook = layout.additionalOutputs + assert.ok(hook) + layout.additionalOutputs = params => { + calls.push(name) + assert.deepEqual(Object.keys(params.data), [`${name}Key`]) + assert.throws(() => params.data.pageKey, /undeclared/) + return hook(params) + } + } + await assert.rejects(pd.collectAdditionalOutputs(), /initialized/) + await pd.init({ layouts }) + await assert.rejects(pd.collectAdditionalOutputs(), /outer.*not available/) + assert.deepEqual(pd.dataDeps, ['companionKey', 'innerKey', 'outerKey', 'pageKey']) + pd.setGlobalData({ pageKey: 'p', companionKey: 'c', outerKey: 'o', innerKey: 'i', secret: 'hidden' }) + await pd.renderInnerPage() + await pd.renderFullPage() + assert.deepEqual(calls, []) + const pageModule = await import(pd.pageInfo.pageFile.filepath) + assert.equal(pageModule.hookCalls, 0) + const outputs = await pd.collectAdditionalOutputs() + assert.equal(pageModule.hookCalls, 1) + assert.deepEqual(calls, ['outer', 'inner']) + assert.deepEqual(outputs.map(({ outputName, content }) => ({ outputName, content })), [ + { outputName: 'outer.txt', content: 'o' }, { outputName: 'inner.txt', content: 'i' }, { outputName: 'page.txt', content: 'pc/' }, + ]) + assert.deepEqual(outputs[0]?.provenance, { kind: 'layout', source: join(dir, 'outer.layout.mjs'), layoutName: 'outer' }) + assert.deepEqual(outputs[2]?.provenance, { kind: 'page', source: join(dir, 'page.mjs') }) +}) + +test('markdown companion gets a frozen narrow source handle and subscribed data', async t => { + const { pd, layouts, dir } = await fixture(t, { + companion: ` + import assert from 'node:assert/strict' + export default { dataDeps: ['selected'] } + export const dataDeps = ['secret'] // A named export is not a subscription declaration. + export async function additionalOutputs ({ page, vars, data }) { + assert.equal(Object.isFrozen(page), true) + assert.equal(Object.isFrozen(page.pageFile), true) + assert.equal(Object.isFrozen(vars), true) + for (const key of ['data', 'pageInfo', 'renderInnerPage', 'renderFullPage', 'setGlobalData', 'collectAdditionalOutputs']) assert.equal(key in page, false) + assert.throws(() => { page.pageFile.filepath = '/other.md' }, TypeError) + assert.throws(() => data.secret, /undeclared/) + const read = page.readMarkdownContent + return { outputName: 'body.md', content: data.selected + await read.call({ pageInfo: {} }) } + }` + }) + await pd.init({ layouts }) + await assert.rejects(pd.collectAdditionalOutputs(), /companion.*not available/) + pd.setGlobalData({ selected: 'selected:', secret: 'hidden' }) + assert.deepEqual(await pd.collectAdditionalOutputs(), [{ outputName: 'body.md', content: 'selected:\n# Body\n', provenance: { kind: 'companion', source: join(dir, 'page.vars.mjs') } }]) +}) + +test('JS and TS page providers work and cannot read markdown or undeclared data', async t => { + for (const extension of ['mjs', 'ts']) { + const { pd, layouts } = await fixture(t, { + extension, module: ` + import assert from 'node:assert/strict' + export default () => 'page' + export async function additionalOutputs ({ page, data }) { + await assert.rejects(page.readMarkdownContent(), /only.*markdown/) + assert.throws(() => data.secret, /undeclared/) + return [] + }` + }) + await pd.init({ layouts }) + pd.setGlobalData({ secret: 'hidden' }) + assert.deepEqual(await pd.collectAdditionalOutputs(), []) + } +}) + +test('companions also provide outputs for JS and HTML pages', async t => { + for (const type of ['js', 'html']) { + const { pd, layouts } = await fixture(t, { + ...(type === 'js' ? { module: "export const vars = { dataDeps: ['selected'] }; export default () => ''" } : {}), + companion: `export default { dataDeps: ['selected'] }; + export const additionalOutputs = async ({ data }) => [{ outputName: 'extra.txt', content: data.selected }]`, + }) + if (type === 'html') { + pd.pageInfo.type = 'html' + await writeFile(pd.pageInfo.pageFile.filepath, '

HTML source

') + } + await pd.init({ layouts }) + pd.setGlobalData({ selected: type }) + const outputs = await pd.collectAdditionalOutputs() + assert.equal(outputs[0]?.content, type) + assert.equal(outputs[0]?.provenance.kind, 'companion') + } +}) + +test('conflicting page providers and invalid exports identify their sources', async t => { + const { pd, layouts } = await fixture(t, { module: "export default () => ''; export const additionalOutputs = () => []", companion: 'export const additionalOutputs = () => []' }) + await assert.rejects(pd.init({ layouts }), /page.mjs.*page.vars.mjs.*both export additionalOutputs/) + for (const options of [{ module: "export default () => ''; export const additionalOutputs = 1" }, { companion: 'export const additionalOutputs = null' }]) { + const { pd, layouts } = await fixture(t, options) + await assert.rejects(pd.init({ layouts }), /additionalOutputs.*page.*must be a function/) + } + const { dir } = await fixture(t) + const path = join(dir, 'bad.layout.mjs') + await writeFile(path, "export default () => ''; export const additionalOutputs = {}") + await assert.rejects(resolveLayout(path), /additionalOutputs.*bad.layout.mjs.*function/) +}) + +test('generated pages skip page, companion and all layout hooks', async t => { + const { pd, layouts } = await fixture(t, { module: "throw new Error('must not import source module')", companion: 'export const additionalOutputs = 123' }) + pd.pageInfo.generated = { pagesFile: { pagesFile: pd.pageInfo.pageFile, path: '', name: 'generated' }, children: 'generated' } + for (const layout of Object.values(layouts)) { + layout.vars = { dataDeps: ['unready'] } + layout.additionalOutputs = () => { throw new Error('generated hook ran') } + } + await pd.init({ layouts }) + assert.deepEqual(await pd.collectAdditionalOutputs(), []) +}) + +test('discovery excludes ignored providers and disabled drafts but enabled drafts collect outputs', async t => { + const { pd, layouts, dir } = await fixture(t, { + companion: "export const additionalOutputs = () => ({ outputName: 'draft.txt', content: 'draft output' })", + }) + await rm(pd.pageInfo.pageFile.filepath) + await writeFile(join(dir, 'page.draft.md'), '# Draft') + await writeFile(join(dir, 'page.js'), "throw new Error('ignored provider must not load')") + const disabled = await identifyPages(dir, { ignore: ['page.js'] }) + assert.equal(disabled.pages.length, 0) + const enabled = await identifyPages(dir, { ignore: ['page.js'], buildDrafts: true }) + assert.equal(enabled.pages.length, 1) + const [pageInfo] = enabled.pages + assert.ok(pageInfo) + assert.equal(pageInfo.draft, true) + pd.pageInfo = pageInfo + await pd.init({ layouts }) + pd.setGlobalData({}) + assert.equal((await pd.collectAdditionalOutputs())[0]?.content, 'draft output') +}) + +test('page hook failures include page and provider context, and no providers is valid', async t => { + for (const body of ["throw new Error('hook failed')", "return 'bare string'", "return (async function * () { throw new Error('iterator failed') })()"]) { + const { pd, layouts } = await fixture(t, { companion: `export function additionalOutputs () { ${body} }` }) + await pd.init({ layouts }) + await assert.rejects(pd.collectAdditionalOutputs(), /additionalOutputs for page "page.md" from companion.*page.vars.mjs.*(failed|Record)/) + } + const { pd, layouts } = await fixture(t) + await pd.init({ layouts }) + assert.deepEqual(await pd.collectAdditionalOutputs(), []) +}) diff --git a/lib/build-pages/page-data.js b/lib/build-pages/page-data.js index 0abe277c..bc521818 100644 --- a/lib/build-pages/page-data.js +++ b/lib/build-pages/page-data.js @@ -1,6 +1,8 @@ /** * @import { PageInfo } from '../identify-pages.js' * @import { BuilderOptions, InternalPageFunction } from './page-builders/page-writer.js' + * @import { AdditionalOutputsFunction, AdditionalOutputProvenance, CollectedAdditionalOutput } from './additional-outputs.js' + * @import { JsPageBuilderResult } from './page-builders/js/index.js' */ import { readFile } from 'node:fs/promises' @@ -11,6 +13,7 @@ import { createSubscribedData, extractDataDeps } from './data-deps.js' import { DomStackDataError } from '../helpers/domstack-error.js' import pretty from 'pretty' import { resolveLayoutChain } from './resolve-layout-chain.js' +import { createAdditionalOutputsPage, normalizeAdditionalOutputs, validateAdditionalOutputsHook } from './additional-outputs.js' /** * @typedef {Object} WorkerFiles @@ -25,10 +28,10 @@ import { resolveLayoutChain } from './resolve-layout-chain.js' * @template [V=string] V - The return type of the layout function (defaults to string) * @template {object} [D=Record] - Declared global data. * @param {string} layoutPath - The string path to the layout ESM module. - * @returns {Promise<{ render: InternalLayoutFunction, vars: Partial, parentLayout: string | undefined }>} The resolved layout module exports. + * @returns {Promise<{ render: InternalLayoutFunction, vars: Partial, parentLayout: string | undefined, additionalOutputs: AdditionalOutputsFunction | undefined, source: string }>} The resolved layout module exports. */ export async function resolveLayout (layoutPath) { - const { default: layout, vars, parentLayout } = await import(layoutPath) + const { default: layout, vars, parentLayout, additionalOutputs } = await import(layoutPath) if (typeof layout !== 'function') throw new TypeError(`Layout "${layoutPath}" must export a default render function`) if (parentLayout !== undefined && (typeof parentLayout !== 'string' || !parentLayout.trim())) { throw new TypeError(`Layout "${layoutPath}" parentLayout must be a non-empty string`) @@ -37,6 +40,8 @@ export async function resolveLayout (layoutPath) { return { render: layout, parentLayout, + source: layoutPath, + additionalOutputs: validateAdditionalOutputsHook(additionalOutputs, layoutPath), vars: /** @type {Partial} */ (await resolveVarsExport(vars, 'Layout vars')), } } @@ -128,6 +133,8 @@ export async function resolveLayout (layoutPath) { * @typedef ResolvedLayout * @property {InternalLayoutFunction} render - The layout function * @property {Partial} [vars] - Variables exported by the layout module. + * @property {AdditionalOutputsFunction | undefined} [additionalOutputs] - Explicit output-phase hook. + * @property {string} [source] - Layout module path for diagnostics. * @property {string} name - The name of the layout * @property {string | undefined} [parentLayout] - Name of the optional outer layout. * @property {string | null} layoutStylePath - The string path to the layout style @@ -151,6 +158,7 @@ export class PageData { /** @type {Partial | null} */ builderVars = null /** @type {string[]} Union of the page and entire layout chain, for output invalidation. */ dataDeps = [] /** @type {string[]} */ #pageDataDeps = [] + /** @type {{ hook: AdditionalOutputsFunction, provenance: AdditionalOutputProvenance } | undefined} */ #pageAdditionalOutputs /** @type {Map }>} */ #layoutSubscriptions = new Map() /** @type {string[]} */ styles = [] /** @type {string[]} */ scripts = [] @@ -310,7 +318,27 @@ export class PageData { await resolvePostVars({ varsPath: pageVars?.filepath }) // throws if postVars export is detected const builder = pageBuilders[type] - const { vars: builderVars } = await builder({ pageInfo, options: this.builderOptions }) + const built = await builder({ pageInfo, options: this.builderOptions }) + const { vars: builderVars } = built + if (!pageInfo.generated) { + const moduleHook = type === 'js' ? /** @type {JsPageBuilderResult} */ (built).additionalOutputs : undefined + const companionHook = pageVars?.filepath + ? validateAdditionalOutputsHook((await import(pageVars.filepath)).additionalOutputs, pageVars.filepath) + : undefined + if (moduleHook && companionHook) { + throw new Error(`Page "${pageInfo.pageFile.filepath}" and companion "${pageVars?.filepath}" both export additionalOutputs; only one page-level provider is allowed`) + } + const hook = moduleHook ?? companionHook + if (hook) { + this.#pageAdditionalOutputs = { + hook, + provenance: { + kind: moduleHook ? 'page' : 'companion', + source: moduleHook ? pageInfo.pageFile.filepath : /** @type {string} */ (pageVars?.filepath), + }, + } + } + } const layoutName = resolveLayoutName(globalVars, resolvedPageVars, builderVars) @@ -366,6 +394,51 @@ export class PageData { this.#initialized = true } + /** + * Run hooks only when explicitly requested by the output phase. + * Layouts run outermost first, followed by the single page-level provider. + * @returns {Promise} + */ + async collectAdditionalOutputs () { + if (!this.#initialized) throw new Error('Must be initialized before collecting additionalOutputs') + if (this.pageInfo.generated) return [] + /** @type {CollectedAdditionalOutput[]} */ + const outputs = [] + // Capture source metadata so rebinding the reader cannot change its source. + const sourceInfo = { ...this.pageInfo, pageFile: { ...this.pageInfo.pageFile } } + const page = createAdditionalOutputsPage(sourceInfo, async () => { + if (sourceInfo.type !== 'md') throw new Error('Markdown content can only be read from markdown pages') + return parseMdFileContents(await readFile(sourceInfo.pageFile.filepath, 'utf8')).markdownContent + }) + /** + * @param {AdditionalOutputsFunction} hook + * @param {AdditionalOutputProvenance} provenance + * @param {() => object} getData + */ + const collect = async (hook, provenance, getData) => { + try { + outputs.push(...await normalizeAdditionalOutputs(hook({ page, vars: this.vars, data: getData() }), provenance)) + } catch (cause) { + throw new Error(`additionalOutputs for page "${this.pageInfo.pageFile.relname}" from ${provenance.kind} "${provenance.source}" failed: ${cause instanceof Error ? cause.message : String(cause)}`, { cause }) + } + } + for (const layout of this.layoutChain) { + const source = layout.source ?? layout.name + const hook = validateAdditionalOutputsHook(layout.additionalOutputs, source) + if (!hook) continue + await collect(hook, { kind: 'layout', source, layoutName: layout.name }, () => { + const subscription = this.#layoutSubscriptions.get(layout.name) + if (!this.#dataReady && subscription?.keys.length) throw this.#dataNotReadyError() + return subscription?.data ?? Object.freeze({}) + }) + } + if (this.#pageAdditionalOutputs) { + const { hook, provenance } = this.#pageAdditionalOutputs + await collect(hook, provenance, () => this.data) + } + return outputs + } + /** * Render the inner contents of a page. * @returns {Promise>} The page's render value, before any layout runs. diff --git a/lib/builder.js b/lib/builder.js index 9c2d98c3..baa30887 100644 --- a/lib/builder.js +++ b/lib/builder.js @@ -16,7 +16,9 @@ import { buildStatic } from './build-static/index.js' import { buildCopy } from './build-copy/index.js' import { buildEsbuild, buildServiceWorkerEsbuild } from './build-esbuild/index.js' import { cp, mkdir, mkdtemp, realpath, rm } from 'node:fs/promises' -import { join, resolve } from 'node:path' +import { join, relative, resolve } from 'node:path' +import { toPosix } from './helpers/path.js' +import { prepareAdditionalOutputPromotion } from './helpers/additional-output-promotion.js' import { DomStackAggregateError } from './helpers/domstack-aggregate-error.js' import { OutputRegistry, isCaseInsensitiveDest } from './output-registry.js' @@ -109,7 +111,7 @@ import { * @param {string} src - The source directory from which the site should be built. * @param {string} dest - The destination directory where the built site should be placed. * @param {DomStackOpts} opts - Options for the build process. - * @param {{ watch?: boolean, caseInsensitive?: boolean, promoteOutputs?: (claims: OutputClaim[], write: () => Promise) => Promise }} [internal] + * @param {{ watch?: boolean, caseInsensitive?: boolean, promoteOutputs?: (claims: OutputClaim[], write: () => Promise, preflight: (removablePaths: string[]) => Promise) => Promise }} [internal] * @returns {Promise} * * @example @@ -135,14 +137,25 @@ export async function builder (src, dest, opts, internal = {}) { const stageDest = await mkdtemp(join(resolve(dest), '.domstack-stage-')) try { const results = await buildInto(src, stageDest, { ...opts, ignore: [...(opts.ignore ?? []), '.domstack-stage-*'] }, dest, internal.watch ?? false, internal.caseInsensitive) - remapBuildResults(results, stageDest, dest) + let unchanged = new Set() + const preflight = async (/** @type {string[]} */ removablePaths) => { + unchanged = await prepareAdditionalOutputPromotion(dest, results.pageBuildResults?.outputs ?? [], removablePaths) + } const write = async () => { await mkdir(dest, { recursive: true }) - await cp(stageDest, await realpath(dest), { recursive: true, force: true }) + await cp(stageDest, await realpath(dest), { + recursive: true, + force: true, + filter: source => !unchanged.has(toPosix(relative(stageDest, source))), + }) } - if (internal.promoteOutputs) await internal.promoteOutputs(results.outputClaims ?? [], write) - else await write() + if (internal.promoteOutputs) await internal.promoteOutputs(results.outputClaims ?? [], write, preflight) + else { + await preflight([]) + await write() + } + remapBuildResults(results, stageDest, dest) delete results.outputClaims return results } catch (error) { diff --git a/lib/domstack-manifest/schema.js b/lib/domstack-manifest/schema.js index d973c148..d75cc038 100644 --- a/lib/domstack-manifest/schema.js +++ b/lib/domstack-manifest/schema.js @@ -18,6 +18,7 @@ export const domstackManifestKindSchema = /** @satisfies {JSONSchema} */ (/** @t description: 'Classifies the build pipeline step or artifact type that produced this output.', enum: [ 'page', + 'page-additional', 'template', 'script', 'style', diff --git a/lib/domstack-manifest/schema.json b/lib/domstack-manifest/schema.json index df3308f9..6d7e7ad3 100644 --- a/lib/domstack-manifest/schema.json +++ b/lib/domstack-manifest/schema.json @@ -33,6 +33,7 @@ "description": "Classifies the build pipeline step or artifact type that produced this output.", "enum": [ "page", + "page-additional", "template", "script", "style", diff --git a/lib/helpers/additional-output-promotion.js b/lib/helpers/additional-output-promotion.js new file mode 100644 index 00000000..a49187da --- /dev/null +++ b/lib/helpers/additional-output-promotion.js @@ -0,0 +1,68 @@ +/** + * @import { DomstackManifestRecord } from '../domstack-manifest/index.js' + */ +import { lstat, readFile, realpath } from 'node:fs/promises' +import { isAbsolute, relative, resolve, sep } from 'node:path' + +/** + * Validate every stale removal and sidecar destination before cleanup or writing. + * The root may resolve through symlinks, but no component below it may do so. + * Only intermediate regular files in this transaction's removal set may block a + * new destination. This preflight does not guard against external filesystem races. + * + * @param {string} dest + * @param {DomstackManifestRecord[]} outputs + * @param {Iterable} [removablePaths] Destination-relative stale claims. + * @returns {Promise>} Unchanged page-additional outputRelnames only. + */ +export async function prepareAdditionalOutputPromotion (dest, outputs, removablePaths = []) { + const sidecars = outputs.filter(output => output.kind === 'page-additional') + const removals = [...removablePaths] + const unchanged = new Set() + if (sidecars.length === 0 && removals.length === 0) return unchanged + let root = resolve(dest) + try { root = await realpath(root) } catch (error) { + if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error + } + const removable = new Set(removals.map(name => resolve(root, name))) + /** @param {string} name @param {boolean} allowRemoval */ + async function inspect (name, allowRemoval) { + const target = resolve(root, name) + const rel = relative(root, target) + if (!rel || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { + throw new Error(`Additional output path escapes dest: ${name}`) + } + let current = root + const components = rel.split(sep) + for (const [index, component] of components.entries()) { + current = resolve(current, component) + let info + try { info = await lstat(current) } catch (error) { + if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error + return { target, comparable: false } + } + if (info.isSymbolicLink()) throw new Error(`Additional output path contains a symlink: ${name}`) + if (index < components.length - 1 && !info.isDirectory()) { + if (allowRemoval && info.isFile() && removable.has(current)) return { target, comparable: false } + throw Object.assign(new Error(`Output path has a non-directory ancestor: ${name}`), { code: 'ENOTDIR' }) + } + if (index === components.length - 1) return { target, comparable: !info.isDirectory() && !removable.has(target) } + } + return { target, comparable: false } + } + // Validate the entire removal set even when no sidecars are being emitted. + for (const name of removals) await inspect(name, false) + const targets = [] + for (const output of sidecars) targets.push({ output, ...await inspect(output.outputRelname, true) }) + for (const { output, target, comparable } of targets) { + // Missing staged sources are errors, not evidence that a destination changed. + const source = await readFile(output.filepath) + if (!comparable) continue + try { + if (source.equals(await readFile(target))) unchanged.add(output.outputRelname) + } catch (error) { + if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error + } + } + return unchanged +} diff --git a/lib/helpers/additional-output-promotion.test.js b/lib/helpers/additional-output-promotion.test.js new file mode 100644 index 00000000..747058c4 --- /dev/null +++ b/lib/helpers/additional-output-promotion.test.js @@ -0,0 +1,83 @@ +/** @import { TestContext } from 'node:test' */ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { prepareAdditionalOutputPromotion } from './additional-output-promotion.js' +import { assertInsideDest } from './path.js' + +/** @param {TestContext} t */ +async function fixture (t) { + const root = await mkdtemp(join(tmpdir(), 'additional-promotion-')) + t.after(() => rm(root, { recursive: true, force: true })) + const dest = join(root, 'dest') + await mkdir(dest) + const filepath = join(root, 'staged') + await writeFile(filepath, Buffer.from([0, 255, 1])) + const output = { kind: /** @type {const} */ ('page-additional'), outputRelname: 'feed.bin', filepath } + return { root, dest, filepath, output } +} + +test('validates stale paths without sidecars and permits only owned regular blockers', async t => { + const { root, dest, output } = await fixture(t) + await writeFile(join(dest, 'raw'), 'old') + const nested = { ...output, outputRelname: 'raw/article.md' } + await assert.rejects(prepareAdditionalOutputPromotion(dest, [nested]), { code: 'ENOTDIR' }) + await assert.rejects(prepareAdditionalOutputPromotion(dest, [nested], ['other']), { code: 'ENOTDIR' }) + assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [nested], ['raw']), new Set()) + await symlink(root, join(dest, 'unsafe')) + await assert.rejects(prepareAdditionalOutputPromotion(dest, [], ['raw', 'unsafe/stale']), /symlink/) + await assert.rejects(prepareAdditionalOutputPromotion(dest, [], ['../escape']), /escapes dest/) + await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: 'unsafe/new' }], ['unsafe']), /symlink/) + const alias = join(root, 'alias') + await symlink(dest, alias) + assert.deepEqual(await prepareAdditionalOutputPromotion(alias, [], ['raw']), new Set()) + assert.doesNotThrow(() => assertInsideDest(dest, join(dest, '..hidden'))) + assert.throws(() => assertInsideDest(dest, join(dest, '../hidden')), /escapes dest/) +}) + +test('compares exact bytes only for sidecars, allowing missing destinations', async t => { + const { dest, output } = await fixture(t) + assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [output]), new Set()) + await writeFile(join(dest, 'feed.bin'), Buffer.from([0, 255, 1])) + assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [output]), new Set(['feed.bin'])) + assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [{ ...output, kind: 'page' }]), new Set()) + await writeFile(join(dest, 'feed.bin'), Buffer.from([0, 254, 1])) + assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [output]), new Set()) +}) + +test('allows realpath destination root but rejects leaf, ancestor and dangling symlinks', async t => { + const { root, dest, output } = await fixture(t) + const alias = join(root, 'alias') + await symlink(dest, alias) + await writeFile(join(dest, 'feed.bin'), Buffer.from([0, 255, 1])) + assert.deepEqual(await prepareAdditionalOutputPromotion(alias, [output]), new Set(['feed.bin'])) + for (const [name, target] of /** @type {[string, string][]} */ ([['leaf', output.filepath], ['inside', dest], ['outside', root], ['dangling', join(root, 'absent')]])) { + await symlink(target, join(dest, name)) + await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: name }]), /symlink/) + await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: `${name}/child` }]), /symlink/) + } +}) + +test('validates all paths before comparison and rejects escapes', async t => { + const { root, dest, output } = await fixture(t) + await symlink(root, join(dest, 'unsafe')) + await assert.rejects(prepareAdditionalOutputPromotion(dest, [ + { ...output, filepath: join(root, 'missing-source') }, + { ...output, outputRelname: 'unsafe/file' }, + ]), /symlink/) + for (const outputRelname of ['../escape', dest, '.']) { + await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname }]), /escapes dest/) + } +}) + +test('directory targets remain changed for owned-descendant cleanup; other errors propagate', async t => { + const { root, dest, output } = await fixture(t) + await mkdir(join(dest, 'feed.bin')) + assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [output]), new Set()) + await writeFile(join(dest, 'parent'), '') + await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: 'parent/file' }]), { code: 'ENOTDIR' }) + await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: 'missing', filepath: join(root, 'absent') }]), { code: 'ENOENT' }) + await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: 'missing', filepath: dest }]), { code: 'EISDIR' }) +}) diff --git a/lib/helpers/path.js b/lib/helpers/path.js index a042cdcd..0cbf45bc 100644 --- a/lib/helpers/path.js +++ b/lib/helpers/path.js @@ -20,7 +20,7 @@ export function assertInsideDest (dest, filepath, message = `Output path escapes const absDest = resolve(dest) const absFilepath = resolve(filepath) const rel = relative(absDest, absFilepath) - if (rel !== '' && (rel.startsWith('..') || isAbsolute(rel))) { + if (rel !== '' && (rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel))) { throw new Error(message) } } diff --git a/test-cases/page-additional-outputs/helpers.js b/test-cases/page-additional-outputs/helpers.js new file mode 100644 index 00000000..0cf8d7e1 --- /dev/null +++ b/test-cases/page-additional-outputs/helpers.js @@ -0,0 +1,90 @@ +/** + * @import { TestContext } from 'node:test' + */ +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { inspect } from 'node:util' +import pino from 'pino' +import { DomStack } from '../../index.js' +import { builder } from '../../lib/builder.js' + +/** @param {string} root @param {Record} files */ +export async function writeFiles (root, files) { + for (const [name, content] of Object.entries(files)) { + const path = join(root, name) + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, content) + } +} + +/** @param {TestContext} t @param {Record} files */ +export async function setup (t, files) { + const tmp = await mkdtemp(join(import.meta.dirname, '.tmp-')) + const src = join(tmp, 'src') + const dest = join(tmp, 'custom-output') + await writeFiles(src, { + 'global.vars.js': "export default { layout: 'root' }", + 'root.layout.js': 'export default ({ children }) => children', + ...files, + }) + const logs = /** @type {string[]} */ ([]) + const options = { domstackManifest: false, logger: pino({ level: 'debug' }, { write: line => logs.push(line) }) } + const site = new DomStack(src, dest, options) + t.after(async () => { + if (site.watching) await site.stopWatching() + await rm(tmp, { recursive: true, force: true }) + }) + return { + src, + dest, + site, + logs, + build: () => builder(src, dest, options), + /** @param {string} name */ + read: name => readFile(join(dest, name), 'utf8'), + /** @param {string} name */ + mtime: async name => (await stat(join(dest, name))).mtimeMs, + } +} + +/** + * Capture the cursor before mutating sources so startup/previous builds cannot + * satisfy the wait, even when chokidar has not detected the change yet. + * @param {DomStack} site + * @param {string[]} logs + * @param {() => Promise} mutate + * @param {string} [expectedError] + */ +export async function settle (site, logs, mutate, expectedError) { + const cursor = logs.length + await mutate() + const deadline = performance.now() + 10_000 + while (true) { + const messages = logs.slice(cursor).map(line => JSON.parse(line).msg) + if (messages.includes('Build Failed!')) { + if (!expectedError || !messages.some(message => message.includes(expectedError))) { + throw new Error(`Unexpected watch build failure:\n${messages.join('\n')}`) + } + break + } + if (messages.includes('Build Success!') && !expectedError) break + if (performance.now() >= deadline) { + throw new Error(`Timed out waiting for watch ${expectedError ? `failure: ${expectedError}` : 'build success'}:\n${messages.join('\n')}`) + } + await new Promise(resolve => setTimeout(resolve, 25)) + } + await site.settled() + const errors = logs.slice(cursor).filter(line => JSON.parse(line).level >= 50) + if (!expectedError && errors.length) throw new Error(`Unexpected watch errors:\n${errors.join('\n')}`) +} + +/** @param {unknown} error @returns {string} */ +export function errorText (error) { + if (!(error instanceof Error)) return inspect(error, { depth: null }) + return [error.message, error.cause ? errorText(error.cause) : '', + ...('errors' in error && Array.isArray(error.errors) ? error.errors.map(errorText) : []), + ].join('\n') +} + +/** @param {string} outputName @param {string} [content] */ +export const hook = (outputName, content = 'sidecar') => `export const additionalOutputs = () => ({ outputName: ${JSON.stringify(outputName)}, content: ${JSON.stringify(content)} })` diff --git a/test-cases/page-additional-outputs/index.test.js b/test-cases/page-additional-outputs/index.test.js new file mode 100644 index 00000000..17aa3d02 --- /dev/null +++ b/test-cases/page-additional-outputs/index.test.js @@ -0,0 +1,193 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { stat } from 'node:fs/promises' +import { join } from 'node:path' +import { errorText, hook, setup, writeFiles } from './helpers.js' + +const rawLayout = `export default ({ children }) => '
' + children + '
' +export const additionalOutputs = async ({ page }) => ({ outputName: './source.txt', content: await page.readMarkdownContent() })` + +test('builder renders Markdown and exports the unrendered body from its layout at a custom destination', async t => { + const body = '# Article\n\nKeep **Markdown**, {{ vars.title }}, and [links](./other.md).\n' + const { build, read, dest } = await setup(t, { + 'root.layout.js': rawLayout, + 'docs/page.md': '---\ntitle: Resolved title\n---\n' + body, + }) + const result = await build() + assert.match(await read('docs/index.html'), /
\s*

Markdown<\/strong>/) + assert.equal(await read('docs/source.txt'), '\n' + body) + const record = result.pageBuildResults?.outputs.find(output => output.outputRelname === 'docs/source.txt') + assert.ok(record, 'additional output is included in the page build report') + assert.equal(record.filepath, join(dest, 'docs/source.txt')) + assert.equal(record.sourceRelname, 'docs/page.md') +}) + +test('nested hooks run outer -> inner -> companion with isolated renderer data and resolved vars', async t => { + const { build, read } = await setup(t, { + 'global.vars.js': "export default { layout: 'inner', title: 'global' }; export const additionalOutputs = () => { throw Error('global provider ran') }", + 'global.data.js': "export default { outer: 'O', inner: 'I', selected: 'P', secret: 'hidden' }", + 'root.layout.js': `import assert from 'node:assert/strict' + export const vars = { dataDeps: ['outer'] } + export default ({ children, data }) => data.outer + children + export const additionalOutputs = ({ page, vars, data }) => { + assert.equal(vars.title, 'page title') + assert.throws(() => data.selected, /undeclared/) + assert.equal('renderFullPage' in page, false) + assert.equal('data' in page, false) + globalThis[page.pageFile.filepath] = ['outer'] + return { outputName: 'outer.txt', content: data.outer } + }`, + 'inner.layout.js': `import assert from 'node:assert/strict' + export const parentLayout = 'root' + export const vars = { dataDeps: ['inner'] } + export default ({ children, data }) => data.inner + children + export async function* additionalOutputs ({ page, data }) { + assert.throws(() => data.outer, /undeclared/) + globalThis[page.pageFile.filepath].push('inner') + yield { outputName: './inner.txt', content: data.inner } + }`, + 'docs/page.md': '---\ntitle: page title\n---\n# Body\n', + 'docs/page.vars.js': `import assert from 'node:assert/strict' + export default { dataDeps: ['selected'] } + export const additionalOutputs = async ({ page, vars, data }) => { + assert.throws(() => data.secret, /undeclared/) + assert.throws(() => data.inner, /undeclared/) + assert.equal(Object.isFrozen(page), true) + assert.equal(Object.isFrozen(vars), true) + const order = globalThis[page.pageFile.filepath] + delete globalThis[page.pageFile.filepath] + return [ + { outputName: '/metadata.json', content: JSON.stringify({ title: vars.title, selected: data.selected, order: [...order, 'page'] }) }, + { outputName: '../source/article.txt', content: await page.readMarkdownContent() }, + ] + }`, + }) + await build() + assert.deepEqual(JSON.parse(await read('metadata.json')), { title: 'page title', selected: 'P', order: ['outer', 'inner', 'page'] }) + assert.equal(await read('docs/outer.txt'), 'O') + assert.equal(await read('docs/inner.txt'), 'I') + assert.equal(await read('source/article.txt'), '\n# Body\n') + assert.match(await read('docs/index.html'), /^OI\s*

{ + const { build, read } = await setup(t, { + 'global.data.js': "export default { selected: 'subscribed', secret: 'private' }", + [`article/page.${extension}`]: extension === 'html' ? '

{{ vars.title }}

' : 'export default ({ vars, data }) => vars.title + data.selected', + 'article/page.vars.js': `import assert from 'node:assert/strict' + export default { title: 'Companion', dataDeps: ['selected'] } + export async function additionalOutputs ({ page, vars, data }) { + await assert.rejects(page.readMarkdownContent()) + assert.throws(() => data.secret, /undeclared/) + return { outputName: 'metadata.json', content: JSON.stringify({ title: vars.title, value: data.selected }) } + }`, + }) + await build() + assert.match(await read('article/index.html'), /Companion/) + assert.deepEqual(JSON.parse(await read('article/metadata.json')), { title: 'Companion', value: 'subscribed' }) + }) +} + +test('JS page modules support promised async iterables, arrays, and empty results', async t => { + const { build, read } = await setup(t, { + 'page.js': `export default () => 'main'; export const additionalOutputs = async () => (async function* () { + yield { outputName: 'one.txt', content: 'one' }; yield { outputName: './two.txt', content: 'two' } + })()`, + 'array/page.js': "export default () => 'array'; export const additionalOutputs = () => [{ outputName: 'array.txt', content: 'array' }]", + 'empty/page.js': "export default () => 'empty'; export const additionalOutputs = () => []", + 'iterator/page.js': "export default () => 'empty iterator'; export async function* additionalOutputs () {}", + }) + await build() + for (const name of ['one', 'two']) assert.equal(await read(`${name}.txt`), name) + assert.equal(await read('array/array.txt'), 'array') + assert.equal(await read('empty/index.html'), 'empty') + assert.equal(await read('iterator/index.html'), 'empty iterator') +}) + +test('generated pages skip inherited layout hooks', async t => { + const { build, read } = await setup(t, { + 'root.layout.js': "export default ({ children }) => children; export const additionalOutputs = () => { throw Error('generated hook ran') }", + 'items.pages.js': "export default { outputName: 'generated.html', children: 'Generated' }", + }) + await build() + assert.equal(await read('generated.html'), 'Generated') +}) + +test('duplicate JS and companion providers fail with both module names', async t => { + const { build } = await setup(t, { + 'page.js': "export default () => 'main'; " + hook('page.txt'), + 'page.vars.js': 'export default {}; ' + hook('companion.txt'), + }) + await assert.rejects(build(), error => { + const message = errorText(error) + assert.match(message, /page\.js/) + assert.match(message, /page\.vars\.js/) + assert.match(message, /additionalOutputs/) + assert.match(message, /both|conflict/i) + return true + }) +}) + +for (const scenario of [ + { name: 'own HTML', output: 'index.html', files: {} }, + { name: 'other page HTML', output: 'other/index.html', files: { 'other/page.html': 'Other' } }, + { name: 'template', output: 'shared.txt', files: { 'shared.txt.template.js': "export default () => 'template'" } }, + { name: 'asset', output: 'shared.txt', files: { 'shared.txt': 'asset' } }, + { name: 'bundle', output: 'client.js', files: { 'client.js': 'console.log(1)', 'esbuild.settings.js': "export default opts => ({ ...opts, entryNames: '[dir]/[name]' })" } }, + { name: 'layout hook', output: 'shared.txt', files: { 'root.layout.js': 'export default ({ children }) => children; ' + hook('shared.txt') } }, + { name: 'other page hook', output: 'shared.txt', files: { 'other/page.js': "export default () => 'other'; " + hook('/shared.txt') } }, +]) { + test(`builder rejects sidecar collision with ${scenario.name}`, async t => { + const { build, dest, read } = await setup(t, { + 'page.js': "export default () => 'new main'; " + hook(scenario.output), + ...scenario.files, + }) + await writeFiles(dest, { 'index.html': 'previous main', 'sentinel.txt': 'keep' }) + await assert.rejects(build(), error => { + assert.match(errorText(error), /Output path conflict/) + assert.ok(errorText(error).includes(scenario.output), errorText(error)) + return true + }) + assert.equal(await read('index.html'), 'previous main', 'failed page phase never publishes main HTML') + assert.equal(await read('sentinel.txt'), 'keep') + }) +} + +for (const result of [ + "'bare string'", + "{ outputName: 'bad.txt', content: 42 }", + "[{ outputName: 'same.txt', content: 'same' }, { outputName: './same.txt', content: 'same' }]", + "{ outputName: '../escape.txt', content: 'bad' }", + "{ outputName: '/', content: 'bad' }", +]) { + test(`builder rejects invalid additional output: ${result}`, async t => { + const { build, dest, read } = await setup(t, { + 'page.js': `export default () => 'new'; export const additionalOutputs = () => (${result})`, + }) + await writeFiles(dest, { 'index.html': 'old' }) + await assert.rejects(build()) + assert.equal(await read('index.html'), 'old') + await assert.rejects(stat(join(dest, '../escape.txt')), { code: 'ENOENT' }) + }) +} + +test('iterator failure after a yield preserves all live page-phase outputs', async t => { + const { build, dest, read } = await setup(t, { + 'a/page.js': "export default () => 'new sibling'; " + hook('sibling.txt', 'new sibling sidecar'), + 'z/page.js': `export default () => 'new main'; export async function* additionalOutputs () { + yield { outputName: 'old.txt', content: 'replacement' } + yield { outputName: 'partial.txt', content: 'must not publish' } + throw Error('iterator exploded') + }`, + }) + const previous = { 'a/index.html': 'old sibling', 'a/sibling.txt': 'old sibling sidecar', 'z/index.html': 'old main', 'z/old.txt': 'old sidecar', 'z/stale.txt': 'retain on failure' } + await writeFiles(dest, previous) + await assert.rejects(build(), error => { + assert.match(errorText(error), /iterator exploded/) + return true + }) + for (const [name, content] of Object.entries(previous)) assert.equal(await read(name), content) + await assert.rejects(stat(join(dest, 'z/partial.txt')), { code: 'ENOENT' }) +}) diff --git a/test-cases/page-additional-outputs/promotion.test.js b/test-cases/page-additional-outputs/promotion.test.js new file mode 100644 index 00000000..54b40ec2 --- /dev/null +++ b/test-cases/page-additional-outputs/promotion.test.js @@ -0,0 +1,73 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { mkdir, readFile, rename, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { hook, setup, settle } from './helpers.js' + +for (const full of [false, true]) { + const mode = full ? 'full' : 'targeted' + test(`${mode} watch converges owned file to directory and back`, { timeout: 30_000 }, async t => { + const { site, src, dest, read, mtime, logs } = await setup(t, { + 'page.js': "export default () => 'main'; export const additionalOutputs = ({ vars }) => ({ outputName: vars.rawName, content: 'raw bytes' })", + 'global.vars.js': "export default { layout: 'root', rawName: '/raw' }", + 'page.vars.js': 'export default {}', + }) + await site.watch({ serve: false }) + const change = async (/** @type {string} */ name) => { + await settle(site, logs, async () => { + await writeFile(join(src, full ? 'global.vars.js' : 'page.vars.js'), `export default { layout: 'root', rawName: ${JSON.stringify(name)} }`) + }) + } + await change('/raw/article.md') + assert.equal(await read('raw/article.md'), 'raw bytes') + const time = await mtime('raw/article.md') + await change('/raw/article.md') + assert.equal(await mtime('raw/article.md'), time) + await change('/raw') + assert.equal(await read('raw'), 'raw bytes') + assert.equal((await stat(join(dest, 'raw'))).isFile(), true) + }) + + test(`${mode} watch validates all stale paths before removing any output`, { timeout: 30_000 }, async t => { + const { site, src, dest, read, logs } = await setup(t, { + 'page.js': "export default () => 'old main'; export const additionalOutputs = () => [{ outputName: '/first.txt', content: 'first' }, { outputName: '/raw/article.md', content: 'owned' }]", + }) + await site.watch({ serve: false }) + const outside = join(src, '..', 'outside') + await mkdir(outside) + await writeFile(join(outside, 'article.md'), 'external sentinel') + await rename(join(dest, 'raw'), join(dest, 'saved-raw')) + await symlink(outside, join(dest, 'raw')) + await settle(site, logs, async () => { + if (full) await rm(join(src, 'page.js')) + else await writeFile(join(src, 'page.js'), "export default () => 'new main'") + }, 'symlink') + assert.ok(logs.some(line => line.includes('symlink'))) + assert.equal(await readFile(join(outside, 'article.md'), 'utf8'), 'external sentinel') + assert.equal(await read('first.txt'), 'first') + assert.equal(await read('index.html'), 'old main') + await rm(join(dest, 'raw')) + await rename(join(dest, 'saved-raw'), join(dest, 'raw')) + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), "export default () => 'recovered'; " + hook('/new.txt')) + }) + assert.equal(await read('new.txt'), 'sidecar') + for (const name of ['first.txt', 'raw/article.md']) await assert.rejects(stat(join(dest, name)), { code: 'ENOENT' }) + }) + + test(`${mode} watch rejects an unowned blocking file before stale cleanup`, { timeout: 30_000 }, async t => { + const { site, src, dest, read, logs } = await setup(t, { + 'page.js': "export default () => 'main'; export const additionalOutputs = ({ vars }) => ({ outputName: vars.rawName, content: 'owned' })", + 'global.vars.js': "export default { layout: 'root', rawName: '/old.txt' }", + 'page.vars.js': 'export default {}', + }) + await site.watch({ serve: false }) + await writeFile(join(dest, 'raw'), 'unowned sentinel') + await settle(site, logs, async () => { + await writeFile(join(src, full ? 'global.vars.js' : 'page.vars.js'), "export default { layout: 'root', rawName: '/raw/article.md' }") + }, 'non-directory ancestor') + assert.ok(logs.some(line => line.includes('non-directory ancestor'))) + assert.equal(await read('raw'), 'unowned sentinel') + assert.equal(await read('old.txt'), 'owned') + }) +} diff --git a/test-cases/page-additional-outputs/watch.test.js b/test-cases/page-additional-outputs/watch.test.js new file mode 100644 index 00000000..cb8c2e10 --- /dev/null +++ b/test-cases/page-additional-outputs/watch.test.js @@ -0,0 +1,173 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { rename, rm, stat, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { hook, setup, settle, writeFiles } from './helpers.js' + +const rawLayout = `export const vars = { dataDeps: ['navigation'] } +export default ({ children, data }) => data.navigation + children +export const additionalOutputs = async ({ page }) => ({ outputName: 'source.txt', content: await page.readMarkdownContent() })` + +test('watch updates only changed raw content and retains unchanged ownership without a public manifest', { timeout: 30_000 }, async t => { + const { site, src, dest, read, mtime, logs } = await setup(t, { + 'global.data.js': "export default { navigation: 'Navigation one' }", + 'root.layout.js': rawLayout, + 'a/page.md': '# Article A\n', + 'b/page.md': '# Article B\n', + }) + await site.watch({ serve: false }) + const siblingRaw = await mtime('b/source.txt') + const siblingHtml = await mtime('b/index.html') + const originalRaw = await mtime('a/source.txt') + await settle(site, logs, async () => { + await writeFile(join(src, 'a/page.md'), '# Edited A\n') + }) + assert.equal(await read('a/source.txt'), '# Edited A\n') + assert.match(await read('a/index.html'), /Edited A/) + assert.notEqual(await mtime('a/source.txt'), originalRaw) + assert.equal(await mtime('b/source.txt'), siblingRaw) + assert.equal(await mtime('b/index.html'), siblingHtml, 'body edit must not invalidate an unrelated sibling') + const editedRaw = await mtime('a/source.txt') + await settle(site, logs, async () => { + await writeFile(join(src, 'global.data.js'), "export default { navigation: 'Navigation two' }") + }) + for (const name of ['a', 'b']) assert.match(await read(`${name}/index.html`), /Navigation two/) + assert.equal(await mtime('a/source.txt'), editedRaw, 'navigation rebuild does not rewrite identical raw content') + assert.equal(await mtime('b/source.txt'), siblingRaw) + await settle(site, logs, async () => { + await rm(join(src, 'b/page.md')) + }) + await assert.rejects(stat(join(dest, 'b/source.txt')), { code: 'ENOENT' }) + await assert.rejects(stat(join(dest, 'b/index.html')), { code: 'ENOENT' }) + assert.equal(await read('a/source.txt'), '# Edited A\n') + await assert.rejects(stat(join(dest, 'domstack-manifest.json')), { code: 'ENOENT' }) +}) + +test('watch reconciles companion addition, output rename, hook removal, companion removal and re-addition', { timeout: 30_000 }, async t => { + const { site, src, dest, read, logs } = await setup(t, { 'article/page.html': '

Article

' }) + await site.watch({ serve: false }) + const companion = join(src, 'article/page.vars.js') + await settle(site, logs, async () => { + await writeFile(companion, 'export default {}; ' + hook('first.txt', 'first')) + }) + assert.equal(await read('article/first.txt'), 'first') + await settle(site, logs, async () => { + await writeFile(companion, 'export default {}; ' + hook('renamed.txt', 'renamed')) + }) + assert.equal(await read('article/renamed.txt'), 'renamed') + await assert.rejects(stat(join(dest, 'article/first.txt')), { code: 'ENOENT' }) + await settle(site, logs, async () => { + await writeFile(companion, 'export default { title: "no hook" }') + }) + await assert.rejects(stat(join(dest, 'article/renamed.txt')), { code: 'ENOENT' }) + await settle(site, logs, async () => { + await writeFile(companion, 'export default {}; ' + hook('again.txt')) + }) + assert.equal(await read('article/again.txt'), 'sidecar') + await settle(site, logs, async () => { + await rm(companion) + }) + await assert.rejects(stat(join(dest, 'article/again.txt')), { code: 'ENOENT' }) + await settle(site, logs, async () => { + await writeFile(companion, 'export default {}; ' + hook('restored.txt')) + }) + assert.equal(await read('article/restored.txt'), 'sidecar') + await settle(site, logs, async () => { + await rename(companion, join(src, 'article/unassociated.vars.js')) + }) + await assert.rejects(stat(join(dest, 'article/restored.txt')), { code: 'ENOENT' }) + assert.match(await read('article/index.html'), /Article/) +}) + +test('watch removes sidecars on source rename and draft exclusion', { timeout: 30_000 }, async t => { + const { site, src, dest, read, logs } = await setup(t, { + 'root.layout.js': `export default ({ children }) => children + export const additionalOutputs = async ({ page }) => ({ outputName: page.outputName + '.txt', content: await page.readMarkdownContent() })`, + 'article.md': '# Article\n', + }) + await site.watch({ serve: false }) + assert.equal(await read('article.html.txt'), '# Article\n') + await settle(site, logs, async () => { + await rename(join(src, 'article.md'), join(src, 'renamed.md')) + }) + assert.equal(await read('renamed.html.txt'), '# Article\n') + for (const name of ['article.html', 'article.html.txt']) await assert.rejects(stat(join(dest, name)), { code: 'ENOENT' }) + await settle(site, logs, async () => { + await rename(join(src, 'renamed.md'), join(src, 'renamed.draft.md')) + }) + for (const name of ['renamed.html', 'renamed.html.txt']) await assert.rejects(stat(join(dest, name)), { code: 'ENOENT' }) + await settle(site, logs, async () => { + await rename(join(src, 'renamed.draft.md'), join(src, 'renamed.md')) + }) + assert.equal(await read('renamed.html.txt'), '# Article\n') +}) + +test('watch iterator failure preserves ownership and recovery removes stale outputs', { timeout: 30_000 }, async t => { + const { site, src, dest, read, mtime, logs } = await setup(t, { + 'page.js': "export default () => 'old main'; " + hook('old.txt', 'old sidecar'), + }) + await site.watch({ serve: false }) + const oldTime = await mtime('old.txt') + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* additionalOutputs () { + yield { outputName: 'old.txt', content: 'failed replacement' } + yield { outputName: 'partial.txt', content: 'partial' } + throw Error('watch iterator exploded') + }`) + }, 'watch iterator exploded') + assert.ok(logs.some(line => line.includes('watch iterator exploded')), 'watch reports the hook failure') + assert.equal(await read('index.html'), 'old main') + assert.equal(await read('old.txt'), 'old sidecar') + assert.equal(await mtime('old.txt'), oldTime) + await assert.rejects(stat(join(dest, 'partial.txt')), { code: 'ENOENT' }) + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), "export default () => 'recovered main'; " + hook('new.txt', 'recovered')) + }) + assert.equal(await read('index.html'), 'recovered main') + assert.equal(await read('new.txt'), 'recovered') + await assert.rejects(stat(join(dest, 'old.txt')), { code: 'ENOENT' }) +}) + +for (const change of ['source deletion', 'draft exclusion', 'hook removal', 'companion deletion', 'companion rename']) { + test(`watch independently cleans up after ${change}`, { timeout: 15_000 }, async t => { + const { site, src, dest, read, logs } = await setup(t, { + 'article/page.html': 'Article', + 'article/page.vars.js': 'export default {}; ' + hook('owned.txt'), + }) + await site.watch({ serve: false }) + assert.equal(await read('article/owned.txt'), 'sidecar') + const page = join(src, 'article/page.html') + const companion = join(src, 'article/page.vars.js') + await settle(site, logs, async () => { + if (change === 'source deletion') await rm(page) + if (change === 'draft exclusion') await rename(page, join(src, 'article/page.draft.html')) + if (change === 'hook removal') await writeFile(companion, 'export default {}') + if (change === 'companion deletion') await rm(companion) + if (change === 'companion rename') await rename(companion, join(src, 'article/unassociated.vars.js')) + }) + await assert.rejects(stat(join(dest, 'article/owned.txt')), { code: 'ENOENT' }) + if (change === 'source deletion' || change === 'draft exclusion') { + await assert.rejects(stat(join(dest, 'article/index.html')), { code: 'ENOENT' }) + } else { + assert.equal(await read('article/index.html'), 'Article') + } + }) +} + +test('hook-only data subscriptions invalidate their owner but not an unrelated sibling', { timeout: 30_000 }, async t => { + const { site, src, read, mtime, logs } = await setup(t, { + 'global.data.js': "export default { selected: 'first', unrelated: 'unchanged' }", + 'a/page.html': 'A', + 'a/page.vars.js': "export default { dataDeps: ['selected'] }; export const additionalOutputs = ({ data }) => ({ outputName: 'data.txt', content: data.selected })", + 'b/page.html': 'B', + }) + await site.watch({ serve: false }) + const mainTime = await mtime('a/index.html') + const siblingTime = await mtime('b/index.html') + await settle(site, logs, async () => { + await writeFiles(src, { 'global.data.js': "export default { selected: 'second', unrelated: 'unchanged' }" }) + }) + assert.equal(await read('a/data.txt'), 'second') + assert.notEqual(await mtime('a/index.html'), mainTime) + assert.equal(await mtime('b/index.html'), siblingTime) +}) diff --git a/types.ts b/types.ts index da8a48e1..32d0d66d 100644 --- a/types.ts +++ b/types.ts @@ -6,6 +6,15 @@ import type { Results } from './lib/builder.js' export type { DataDeps } from './lib/build-pages/data-deps.js' +export type { + AdditionalOutput, + AdditionalOutputProvenance, + AdditionalOutputsFunction, + AdditionalOutputsFunctionParams, + AdditionalOutputsPage, + AdditionalOutputsResult, + CollectedAdditionalOutput, +} from './lib/build-pages/additional-outputs.js' export type { BuildOptions } from 'esbuild' export type { DomStackOpts, Results, SiteData } from './lib/builder.js' From 534210b920cf295c99db427a84f4f014b86b283e Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Sun, 13 Sep 2026 19:53:13 -0700 Subject: [PATCH 3/8] Simplify additional outputs to direct writes and duplicate warnings --- docs/assets/README.md | 30 +- docs/layouts/README.md | 3 +- docs/migrations/v12-migration.md | 9 - docs/pages/README.md | 14 +- index.js | 419 ++++-------- lib/build-copy/index.js | 30 +- lib/build-esbuild/index.js | 134 +--- lib/build-esbuild/output-conflicts.js | 70 -- lib/build-pages/index.js | 250 +++---- .../page-builders/additional-output-writer.js | 33 +- .../additional-output-writer.test.js | 6 +- lib/build-pages/page-builders/page-writer.js | 19 +- .../page-builders/template-builder.js | 18 +- lib/build-static/index.js | 27 +- lib/builder.js | 140 +--- lib/domstack-manifest/hooks.js | 14 +- lib/helpers/additional-output-promotion.js | 68 -- .../additional-output-promotion.test.js | 83 --- lib/helpers/domstack-error.js | 2 +- lib/helpers/domstack-warning.js | 1 + lib/helpers/output-warnings.js | 27 + lib/helpers/staged-copy.js | 61 -- lib/output-registry.js | 174 ----- test-cases/output-conflicts/index.test.js | 614 ------------------ test-cases/page-additional-outputs/helpers.js | 2 +- .../page-additional-outputs/index.test.js | 52 +- .../page-additional-outputs/ownership.test.js | 70 ++ .../page-additional-outputs/promotion.test.js | 73 --- .../page-additional-outputs/watch.test.js | 4 +- 29 files changed, 460 insertions(+), 1987 deletions(-) delete mode 100644 lib/build-esbuild/output-conflicts.js delete mode 100644 lib/helpers/additional-output-promotion.js delete mode 100644 lib/helpers/additional-output-promotion.test.js create mode 100644 lib/helpers/output-warnings.js delete mode 100644 lib/helpers/staged-copy.js delete mode 100644 lib/output-registry.js delete mode 100644 test-cases/output-conflicts/index.test.js create mode 100644 test-cases/page-additional-outputs/ownership.test.js delete mode 100644 test-cases/page-additional-outputs/promotion.test.js diff --git a/docs/assets/README.md b/docs/assets/README.md index f0f2d522..24fb878c 100644 --- a/docs/assets/README.md +++ b/docs/assets/README.md @@ -14,31 +14,6 @@ Build-tool configuration and site-wide variables are documented separately in [S [[toc]] -## Output conflicts - -Each output file must have one producer across pages, generated pages, templates, static files, copied directories, esbuild bundles, service workers, and generated metadata. -This includes page `workers.json` files, the optional `domstack-manifest.json`, and files written through a manifest hook's `writeFile` helper. -Two templates cannot emit the same path, and a single template cannot repeat an output in an array or async iterator. -Repeated identical reporting records within one batch are deduplicated; they are not additional writes. -File-versus-directory conflicts such as `feed` and `feed/index.xml` are also rejected. -Output separators and dot segments are normalized, and case aliases are checked using the destination filesystem's case behavior. - -One-shot builds claim outputs before writing them, so a conflicting second producer cannot overwrite the first. -Earlier successful build steps may remain in the destination after a later failure; the build is not rolled back. -Manifest hooks can read files they just wrote through `writeFile` from their supplied `dest`. -Watch rebuilds retain ownership for untouched producers, revalidate page outputs before promotion, and release obsolete paths after successful replacement or removal. -In a successfully started watch session, a failed conflict check retains the previous successful outputs and ownership, so fixing the source can recover without restarting watch mode. -Initial copy or esbuild failures abort startup and require starting watch again; initial page failures are logged and can recover within the session. -Full watch rebuilds replace the complete ownership map rather than accumulating historical paths. - -Page phases and full watch builds use unique stages on the destination filesystem, and copied sources are isolated before publication. -Staging requires additional disk space. -Publication is not an atomic filesystem transaction: an I/O failure during the final copy can still leave partially updated files. -The registry covers DOMStack-managed writers, not arbitrary filesystem writes performed directly by user code or esbuild plugins. -Manifest hooks should use their supplied `writeFile` helper to participate in conflict detection. -Identifiable esbuild entry collisions use the same conflict error; native plugin or shared-chunk collisions that cannot be attributed to two sources retain esbuild's diagnostic. -Concurrent builds use independent stages, but separate DOMStack instances should not publish different sites to the same destination concurrently. - ## Static assets All static assets in the `src` directory are copied 1:1 to the destination directory using [cpx2](https://github.com/bcomnes/cpx2). @@ -55,9 +30,8 @@ Place a file in a directory whose structure encodes its desired destination path To copy multiple directories, repeat the flag: `domstack --copy oldsite --copy archived-docs`. > [!WARNING] -> DOMStack rejects conflicting output paths with `DOM_STACK_ERROR_OUTPUT_CONFLICT`. -The error identifies the destination-relative path and both producers, including files from different `--copy` directories. -Rename or exclude one input instead of relying on copy order to select a winner. +> DOMStack does not detect conflicts between copied directories and other build output. +If multiple inputs produce the same destination path, the result is undefined. Copy folders must live **outside** of the `dest` directory. Copy directories can be in the src directory allowing for nested builds. diff --git a/docs/layouts/README.md b/docs/layouts/README.md index ac36cae1..9a6cfd07 100644 --- a/docs/layouts/README.md +++ b/docs/layouts/README.md @@ -125,7 +125,8 @@ export async function additionalOutputs ({ page, vars }) { ``` The filename is relative to the current page's output directory, not the layout directory. -Using the page's HTML filename avoids collisions when several loose Markdown pages share a directory. +Using the page's HTML filename helps keep destinations unique when several loose Markdown pages share a directory. +Exact duplicate destinations produce best-effort build warnings, not an override contract; avoid sharing output paths between pages or hooks. Nested hooks run outermost layout → innermost layout → page, and each layout hook shares only that layout renderer's `vars.dataDeps` subscriptions. Generated pages skip these hooks, including inherited layout hooks. See [Additional outputs](../pages/#additional-outputs) for the complete API, companion modules, path rules, and watch behavior. diff --git a/docs/migrations/v12-migration.md b/docs/migrations/v12-migration.md index 991accf0..a4e26239 100644 --- a/docs/migrations/v12-migration.md +++ b/docs/migrations/v12-migration.md @@ -20,15 +20,6 @@ Then apply the v12 changes below. --- -## Conflicting output paths now fail - -DOMStack rejects duplicate output paths across pages, generated pages, templates, esbuild, static assets, and `--copy` directories with `DOM_STACK_ERROR_OUTPUT_CONFLICT`. -The error names the destination-relative output and both producers. -Previously, copied files and templates could silently overwrite other outputs depending on write order. -Rename or exclude the conflicting input; there is no implicit last-writer-wins override. -This also applies to file-versus-directory conflicts and case aliases on case-insensitive destination filesystems. -See [Output conflicts](../assets/#output-conflicts) for staging, watch recovery, and custom-writer limitations. - ## Runtime requirements DOMStack v12 supports Node.js 22.18+ within the 22.x release line, and Node.js 24 or newer: diff --git a/docs/pages/README.md b/docs/pages/README.md index 03ad07fa..d9486c88 100644 --- a/docs/pages/README.md +++ b/docs/pages/README.md @@ -408,11 +408,15 @@ Parent traversal is allowed only while the resolved target remains inside the de Escapes and invalid file targets fail the build. These rules do not change existing template output-path semantics. -Duplicate destinations fail even when content is identical, including duplicates between hooks and conflicts with normal HTML, other pages, templates, copied assets, or bundles. -There is no implicit override mechanism. -Hook, iterator, validation, and collision failures publish none of the staged page-phase outputs and do not clean up stale page outputs or replace prior ownership. -An iterator that throws after yielding records therefore cannot publish those earlier yields to the live destination. -This is a page-phase guarantee, not a transaction across every build phase or a rollback guarantee for filesystem I/O failures during publication. +Duplicate destinations produce best-effort warnings based on build output reports, including exact duplicates between hooks and conflicts with normal HTML, other pages, templates, copied assets, or bundles. +They do not reject the build, even when content differs. +Watch warnings only compare outputs observed in the current page/template phase; this is not a persistent cross-build conflict registry or a case-alias check. +Choose unique destinations; do not rely on write order or cleanup behavior for conflicting outputs. + +DOMStack renders a page and collects and validates all its hook results before writing that page's HTML and additional files directly to the destination. +If rendering, a hook, an iterator, or output validation fails, that owning page's existing HTML and sidecars remain unchanged. +An iterator that throws after yielding records therefore does not write those earlier yields. +Other pages and build phases may already have written their outputs; there is no whole-build or page-phase isolation, and filesystem write failures can leave partial updates. ### Watch behavior and ownership diff --git a/index.js b/index.js index 66d10793..d99488e3 100644 --- a/index.js +++ b/index.js @@ -12,8 +12,6 @@ * @import { DomstackManifestRecord } from './lib/domstack-manifest/index.js' * @import { WatchDependencyState } from './lib/build-pages/watch-dependencies.js' * @import { WatchSnapshot, WatchEvent, WatchPlan } from './lib/watch-plan.js' - * @import { OutputClaim } from './lib/output-registry.js' - * @import { PageBuildStepResult } from './lib/build-pages/index.js' * @typedef {{ dispose: () => Promise }} DisposableBuildContext * @typedef {{ pageFilePath: string, sourcePageFilePath?: string | undefined, pagesFilePath?: string | undefined, layoutNames: string[], outputs?: DomstackManifestRecord[] | undefined }} WatchedPageReport * @typedef {object} WatchSession @@ -24,9 +22,8 @@ */ import { once } from 'events' import assert from 'node:assert' -import { cp, copyFile, mkdir, mkdtemp, readFile, realpath, rm, rmdir, stat } from 'node:fs/promises' +import { lstat, mkdtemp, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { createHash } from 'node:crypto' import chokidar from 'chokidar' import { basename, dirname, join, relative, resolve } from 'node:path' // @ts-expect-error @@ -37,8 +34,7 @@ import { inspect } from 'util' import { createServer } from '@domstack/sync' import { find } from '@11ty/dependency-tree-typescript' -import { assertInsideDest, toPosix } from './lib/helpers/path.js' -import { prepareAdditionalOutputPromotion } from './lib/helpers/additional-output-promotion.js' +import { assertInsideDest } from './lib/helpers/path.js' import { getCopyGlob } from './lib/build-static/index.js' import { getCopyDirs } from './lib/build-copy/index.js' import { isProcessedFile, globalBundleAssets, pageBundleAssets, layoutBundleAssets } from './lib/file-conventions.js' @@ -47,9 +43,9 @@ import { buildEsbuildWatch } from './lib/build-esbuild/index.js' import { buildPages } from './lib/build-pages/index.js' import { identifyPages } from './lib/identify-pages.js' import { classifyWatchEvent, planWatchEvent, planBundleChange } from './lib/watch-plan.js' +import { ensureDest } from './lib/helpers/ensure-dest.js' import { DomStackAggregateError } from './lib/helpers/domstack-aggregate-error.js' import { createDomStackLogger } from './lib/logger.js' -import { OutputRegistry, isCaseInsensitiveDest } from './lib/output-registry.js' export { PageData } from './lib/build-pages/page-data.js' export { @@ -79,8 +75,6 @@ export class DomStack { /** @type {Readonly} */ opts /** @type {FSWatcher?} */ #watcher = null /** @type {ReturnType[]} */ #cpxWatchers = [] - /** @type {string[]} */ #cpxWatchStages = [] - /** @type {Map Promise>} */ #pendingCopyUpdates = new Map() /** @type {BsInstance?} */ #syncServer = null /** @type {DisposableBuildContext?} */ #esbuildContext = null /** @type {SiteData?} */ #siteData = null @@ -107,21 +101,16 @@ export class DomStack { #globalDataDepPaths = new Set() /** @type {Set} absolute filepaths of esbuild entry points */ #esbuildEntryPoints = new Set() - + /** @type {Map>} source page or *.pages.* filepath → owned absolute output paths */ + #pageOutputMap = new Map() + /** @type {Map>} template filepath → currently claimed absolute output paths */ + #templateOutputMap = new Map() /** @type {Map>} *.pages.* filepath → layouts used by its generated pages */ #pagesFileLayoutMap = new Map() /** @type {WatchDependencyState | null} subscriptions and fingerprints from the last successful page build */ #watchDependencies = null /** @type {boolean} Failed builds may leave the previous routing state incomplete. */ #pageBuildFailed = false - /** @type {OutputClaim[]} Output ownership from the latest successful watch build. */ - #outputClaims = [] - #outputLock = Promise.resolve() - #caseInsensitive = false - /** @type {string | null} Unpublished initial watch outputs, retained for recovery. */ - #initialStage = null - /** @type {Map} Sidecars staged by successful initial page transactions. */ - #initialAdditionalOutputs = new Map() // One session owns the resources above until shutdown finishes. // Normal path: absent → starting → watching → stopping → absent. @@ -242,8 +231,6 @@ export class DomStack { } session.state = 'watching' - for (const update of this.#pendingCopyUpdates.values()) this.#enqueueBuild(session, update) - this.#pendingCopyUpdates.clear() const enqueue = (/** @type {() => Promise} */ fn) => { this.#enqueueBuild(session, fn) } @@ -273,28 +260,12 @@ export class DomStack { throw new DomStackAggregateError(siteData.errors, 'Page walk finished but there were errors.', siteData) } - await mkdir(this.#dest, { recursive: true }) - this.#initialStage = await mkdtemp(join(resolve(this.#dest), '.domstack-stage-')) - - this.#caseInsensitive = await isCaseInsensitiveDest(this.#dest) - // The watchers' initial inventories are the only copy scan at startup. - const copyDirs = getCopyDirs(this.opts.copy ?? []) - const copyStartup = await Promise.allSettled([ - ...(this.opts.static === false ? [] : [this.#startCopyWatcher(getCopyGlob(this.#src), signal, 'static', this.opts.ignore ?? [])]), - ...copyDirs.map((copyDir, index) => this.#startCopyWatcher(copyDir, signal, 'copy', [], `copy-root:${index}:`)), - ]) - const copyErrors = copyStartup.filter(result => result.status === 'rejected').map(result => result.reason) - if (copyErrors.length) throw new AggregateError(copyErrors, 'Copy watch startup failed') - await this.#drainPendingCopyUpdates() + await ensureDest(this.#dest, siteData) // Start esbuild in watch mode (stable filenames, no hash) let esbuildContext try { - const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { - logger: this.#logger, - writeDest: () => this.#initialStage ?? this.#dest, - promoteOutputs: (outputs, phase, write) => this.#promoteEsbuildOutputs(outputs, phase, write), - }) + const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { logger: this.#logger }) esbuildContext = context } catch (err) { throw new Error('Error starting esbuild watch context', { cause: err }) @@ -302,18 +273,13 @@ export class DomStack { this.#esbuildContext = esbuildContext this.#siteData = siteData - await this.#drainPendingCopyUpdates() // Build pages (initial full build) let report try { - const pageBuildResults = await buildPages(this.#src, this.#initialStage ?? this.#dest, siteData, { + const pageBuildResults = await buildPages(this.#src, this.#dest, siteData, { ...this.opts, trackWatchDependencies: true, - previousOutputClaims: this.#outputClaims, - caseInsensitive: this.#caseInsensitive, - promoteOutputs: (report, write, preflight) => this.#promotePageOutputs(report, write, preflight), }) - this.#remapInitialPageReport(pageBuildResults) if (pageBuildResults.errors.length > 0) { throw new DomStackAggregateError(pageBuildResults.errors, 'Page build finished but there were errors.', { siteData, @@ -325,13 +291,11 @@ export class DomStack { siteData, pageBuildResults, } - + await this.#removeObsoletePageOutputs(pageBuildResults, false) this.#pagesFileLayoutMap = getPagesFileLayoutMap(pageBuildResults.report.pages) this.#updatePageLayoutNames(pageBuildResults.report.pages, true) this.#pageBuildFailed = false this.#watchDependencies = pageBuildResults.report.watchDependencies ?? null - delete pageBuildResults.report.newClaims - delete pageBuildResults.report.replacedOwnerIds delete pageBuildResults.report.watchDependencies delete pageBuildResults.report.rebuiltPagesFilePaths buildLogger(report, this.#logger) @@ -347,9 +311,13 @@ export class DomStack { await this.#rebuildMaps(siteData) // Copy readiness is cancellable: cpx2 invalidates pending scans on close. - await this.#drainPendingCopyUpdates() - if (!signal.aborted && !this.#pageBuildFailed) await this.#publishInitialStage() - await this.#drainPendingCopyUpdates() + const copyDirs = getCopyDirs(this.opts.copy ?? []) + const copyStartup = await Promise.allSettled([ + this.#startCopyWatcher(getCopyGlob(this.#src), signal, this.opts.ignore ?? []), + ...copyDirs.map(copyDir => this.#startCopyWatcher(copyDir, signal)), + ]) + const copyErrors = copyStartup.filter(result => result.status === 'rejected').map(result => result.reason) + if (copyErrors.length) throw new AggregateError(copyErrors, 'Copy watch startup failed') // ── Chokidar watcher ───────────────────────────────────────────────── const ig = ignore().add(this.opts.ignore ?? []) @@ -390,57 +358,20 @@ export class DomStack { /** * @param {string} source * @param {AbortSignal} signal - * @param {'static' | 'copy'} kind * @param {string[]} [ignores] - * @param {string} [ownerPrefix] - Matches buildCopy's configured root identity. */ - async #startCopyWatcher (source, signal, kind, ignores = [], ownerPrefix = '') { - const stageDest = await mkdtemp(join(this.#dest, '.domstack-copy-watch-')) - this.#cpxWatchStages.push(stageDest) - const watcher = cpxWatch(source, stageDest, { ignore: ignores }) + async #startCopyWatcher (source, signal, ignores = []) { + const watcher = cpxWatch(source, this.#dest, { ignore: ignores }) this.#cpxWatchers.push(watcher) - // Isolate each source before cpx writes, including case and file/dir aliases. - // Retain cpx's logical mapping; only its private physical destination changes. - const toDestination = watcher.toDestination - const logicalOutputs = new Map() - watcher.toDestination = sourcePath => { - const path = join(stageDest, createHash('sha256').update(sourcePath).digest('hex')) - logicalOutputs.set(path, toPosix(relative(stageDest, toDestination(sourcePath)))) - return path - } - const ownerByOutput = new Map() let ready = false let initialCopies = 0 watcher.on('copy', (/** @type{{ srcPath: string, dstPath: string }} */e) => { - const outputRelname = logicalOutputs.get(e.dstPath) - const sourceRelname = toPosix(relative(this.#src, resolve(e.srcPath))) - const owner = { id: `${ownerPrefix}${kind}:${sourceRelname}`, type: kind, path: sourceRelname } - ownerByOutput.set(e.dstPath, owner) if (!ready) initialCopies++ this.#logger.debug(`Copy ${e.srcPath} to ${e.dstPath}`) - if (this.#watchSession) { - const update = async () => { - try { - await stat(e.srcPath) - await this.#promoteCopyOutput(e.dstPath, outputRelname, owner) - } catch (error) { - if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error - await this.#removeCopyOutput(outputRelname, owner) - } - } - if (!ready || this.#watchSession.state === 'starting') this.#pendingCopyUpdates.set(e.dstPath, update) - else this.#enqueueBuild(this.#watchSession, update) - } + if (ready) this.#logger.info(`Static asset updated: ${e.srcPath}`) }) watcher.on('remove', (/** @type{{ path: string }} */e) => { - const outputRelname = logicalOutputs.get(e.path) - const owner = ownerByOutput.get(e.path) - ownerByOutput.delete(e.path) - if (owner && this.#watchSession) { - const update = () => this.#removeCopyOutput(outputRelname, owner) - if (!ready || this.#watchSession.state === 'starting') this.#pendingCopyUpdates.set(e.path, update) - else this.#enqueueBuild(this.#watchSession, update) - } + this.#logger.info(`Remove ${e.path}`) }) watcher.on('watch-error', (/** @type{Error} */err) => { this.#logger.error(`Copy error: ${err.message}`) @@ -449,156 +380,23 @@ export class DomStack { // cpx2 reports startup failure as "watch-error", not EventEmitter's "error". // A closed session may never emit readiness, so cancellation must also settle // this wait. This does not drain file operations already started by cpx2. - const { promise, resolve: resolveReady, reject } = Promise.withResolvers() - const onAbort = () => resolveReady(undefined) - watcher.once('watch-ready', resolveReady) + const { promise, resolve, reject } = Promise.withResolvers() + const onAbort = () => resolve(undefined) + watcher.once('watch-ready', resolve) watcher.once('watch-error', reject) signal.addEventListener('abort', onAbort, { once: true }) try { if (signal.aborted) return await promise - if (signal.aborted) return ready = true - this.#logger.info(`Static asset watcher ready (${initialCopies} initial copy operations)`) + if (!signal.aborted) this.#logger.info(`Static asset watcher ready (${initialCopies} initial copy operations)`) } finally { - watcher.off('watch-ready', resolveReady) + watcher.off('watch-ready', resolve) watcher.off('watch-error', reject) signal.removeEventListener('abort', onAbort) } } - async #drainPendingCopyUpdates () { - while (this.#pendingCopyUpdates.size > 0) { - const updates = [...this.#pendingCopyUpdates.values()] - this.#pendingCopyUpdates.clear() - for (const update of updates) await update() - } - } - - /** - * Serialize ownership checks, promotion, and commits independently of the - * watch build queue: esbuild callbacks also run during queued context startup. - * @param {() => OutputClaim[]} nextClaims - * @param {() => Promise} write - * @param {(removablePaths: string[]) => Promise} [preflight] - */ - #commitOutputs (nextClaims, write, preflight) { - const transaction = this.#outputLock.then(async () => { - const next = nextClaims() - const paths = new Set(next.map(claim => claim.outputRelname)) - const stale = this.#outputClaims.filter(claim => !paths.has(claim.outputRelname)).map(claim => claim.outputRelname) - const writeDest = this.#initialStage ?? this.#dest - await prepareAdditionalOutputPromotion(writeDest, [], stale) - await preflight?.(stale) - for (const name of stale) { - const target = resolve(writeDest, name) - assertInsideDest(writeDest, target) - await rm(target, { force: true }) - // Only remove empty directories; never recursively delete unowned files. - for (let dir = dirname(target); dir !== resolve(writeDest); dir = dirname(dir)) { - try { await rmdir(dir) } catch { break } - } - } - await write() - this.#outputClaims = next - }) - this.#outputLock = transaction.catch(() => {}) - return transaction - } - - /** Publish initial watch outputs only once every initial producer succeeds. */ - async #publishInitialStage () { - const publish = this.#outputLock.then(async () => { - if (!this.#initialStage) return - await mkdir(this.#dest, { recursive: true }) - const stage = this.#initialStage - const unchanged = await prepareAdditionalOutputPromotion(this.#dest, [...this.#initialAdditionalOutputs.values()]) - await cp(stage, await realpath(this.#dest), { - recursive: true, - force: true, - filter: source => !unchanged.has(toPosix(relative(stage, source))), - }) - this.#initialAdditionalOutputs.clear() - await rm(this.#initialStage, { recursive: true, force: true }) - this.#initialStage = null - }) - this.#outputLock = publish.catch(() => {}) - await publish - } - - /** @param {PageBuildStepResult} result */ - #remapInitialPageReport (result) { - if (!this.#initialStage) return - for (const output of result.outputs) output.filepath = resolve(this.#dest, output.outputRelname) - for (const page of result.report.pages) page.pageFilePath = resolve(this.#dest, relative(this.#initialStage, page.pageFilePath)) - } - - /** - * @param {DomstackManifestRecord[]} outputs - * @param {'browser' | 'service-worker'} phase - * @param {() => Promise} write - */ - #promoteEsbuildOutputs (outputs, phase, write) { - return this.#commitOutputs(() => replaceEsbuildClaims(this.#outputClaims, outputs, phase, this.#caseInsensitive), write) - } - - /** - * Revalidate the worker's selected producers against current ownership, not - * its possibly stale pre-render snapshot. - * @param {PageBuildStepResult} report - * @param {() => Promise} write - * @param {(removablePaths: string[]) => Promise} preflight - */ - #promotePageOutputs (report, write, preflight) { - return this.#commitOutputs(() => { - const replaced = new Set(report.report.replacedOwnerIds ?? []) - const registry = new OutputRegistry(this.#outputClaims, { replaceOwnerIds: replaced, caseInsensitive: this.#caseInsensitive }) - for (const claim of report.report.newClaims ?? []) registry.claim(claim.outputRelname, claim.owner) - return registry.snapshot() - }, async () => { - await write() - if (this.#initialStage) { - const replaced = new Set(report.report.replacedOwnerIds ?? []) - const removed = new Set(this.#outputClaims.filter(claim => replaced.has(claim.owner.id)).map(claim => claim.outputRelname)) - for (const name of removed) this.#initialAdditionalOutputs.delete(name) - for (const output of report.outputs) { - if (output.kind === 'page-additional') { - this.#initialAdditionalOutputs.set(output.outputRelname, { ...output, filepath: resolve(this.#initialStage, output.outputRelname) }) - } - } - } - }, preflight) - } - - /** - * @param {string} stagedPath - * @param {string} outputRelname - * @param {{ id: string, type: string, path: string }} owner - */ - async #promoteCopyOutput (stagedPath, outputRelname, owner) { - await this.#commitOutputs(() => { - const registry = new OutputRegistry(this.#outputClaims, { replaceOwnerIds: [owner.id], caseInsensitive: this.#caseInsensitive }) - registry.claim(outputRelname, owner) - return registry.snapshot() - }, async () => { - const writeDest = this.#initialStage ?? this.#dest - const target = resolve(writeDest, outputRelname) - assertInsideDest(writeDest, target) - await mkdir(dirname(target), { recursive: true }) - await copyFile(stagedPath, target) - }) - this.#logger.info(`Static asset updated: ${owner.path}`) - } - - /** - * @param {string} outputRelname - * @param {{ id: string, type: string, path: string }} owner - */ - async #removeCopyOutput (outputRelname, owner) { - await this.#commitOutputs(() => this.#outputClaims.filter(claim => claim.owner.id !== owner.id || claim.outputRelname !== outputRelname), async () => {}) - this.#logger.info(`Remove ${outputRelname}`) - } - async #startWatchServer () { this.#syncServer = await createServer({ server: this.#dest, @@ -620,36 +418,22 @@ export class DomStack { this.#esbuildContext = null } - try { - const results = await builder(this.#src, this.#dest, { ...this.opts, domstackManifest: false }, { - watch: true, - caseInsensitive: this.#caseInsensitive, - promoteOutputs: (claims, write, preflight) => this.#commitOutputs(() => claims, write, preflight), - }) - if (this.#initialStage) { - await rm(this.#initialStage, { recursive: true, force: true }) - this.#initialStage = null - this.#initialAdditionalOutputs.clear() - } - const { siteData, pageBuildResults } = results - this.#siteData = siteData - if (pageBuildResults) { - this.#updatePageLayoutNames(pageBuildResults.report.pages, true) - this.#pagesFileLayoutMap = getPagesFileLayoutMap(pageBuildResults.report.pages) - this.#watchDependencies = pageBuildResults.report.watchDependencies ?? null - } - this.#pageBuildFailed = false - const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { - logger: this.#logger, - promoteOutputs: (outputs, phase, write) => this.#promoteEsbuildOutputs(outputs, phase, write), - }) - this.#esbuildContext = context - await this.#rebuildMaps(siteData) - buildLogger(results, this.#logger) - } catch (error) { - this.#pageBuildFailed = true - throw error + const siteData = await identifyPages(this.#src, this.opts) + + if (siteData.errors.length > 0) { + this.#logger.error(`identifyPages errors: +${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) + return } + + await ensureDest(this.#dest, siteData) + + const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { logger: this.#logger }) + this.#esbuildContext = context + this.#siteData = siteData + + await this.#runPageBuild(siteData) + await this.#rebuildMaps(siteData) } /** @returns {WatchSnapshot | undefined} */ @@ -678,10 +462,6 @@ export class DomStack { async #handleWatchEvent (changedPath, type) { const snapshot = this.#watchSnapshot() if (!snapshot) return - if (!this.#esbuildContext) { - await this.#fullRebuild() - return - } const event = classifyWatchEvent(type, changedPath) await this.#executeWatchPlan(planWatchEvent(snapshot, event), event) } @@ -717,16 +497,12 @@ export class DomStack { this.#logger.error(`identifyPages errors:\n${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) return } - await mkdir(this.#dest, { recursive: true }) + await ensureDest(this.#dest, siteData) if (this.#esbuildContext) { await this.#esbuildContext.dispose() this.#esbuildContext = null } - const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { - logger: this.#logger, - writeDest: () => this.#initialStage ?? this.#dest, - promoteOutputs: (outputs, phase, write) => this.#promoteEsbuildOutputs(outputs, phase, write), - }) + const { context } = await buildEsbuildWatch(this.#src, this.#dest, siteData, this.opts, { logger: this.#logger }) this.#esbuildContext = context this.#siteData = siteData const snapshot = this.#watchSnapshot() @@ -750,37 +526,30 @@ export class DomStack { // Retry the complete page phase after a failure: neither subscriptions nor // layout routing from a failed build can safely drive an incremental retry. if (this.#pageBuildFailed) pageFilterPaths = templateFilterPaths = pagesFileFilterPaths = null - try { - const pageBuildResults = await buildPages(this.#src, this.#initialStage ?? this.#dest, siteData, { + const pageBuildResults = await buildPages(this.#src, this.#dest, siteData, { ...this.opts, ...(pageFilterPaths ? { pageFilterPaths } : {}), ...(templateFilterPaths ? { templateFilterPaths } : {}), ...(pagesFileFilterPaths ? { pagesFileFilterPaths } : {}), previousWatchDependencies: this.#watchDependencies, trackWatchDependencies: true, - previousOutputClaims: this.#outputClaims, - caseInsensitive: this.#caseInsensitive, - promoteOutputs: (report, write, preflight) => this.#promotePageOutputs(report, write, preflight), }) - this.#remapInitialPageReport(pageBuildResults) if (pageBuildResults.errors.length > 0) { throw new DomStackAggregateError(pageBuildResults.errors, 'Page build finished but there were errors.', { siteData, pageBuildResults, }) } - await this.#publishInitialStage() const isFiltered = pageFilterPaths !== null || templateFilterPaths !== null || pagesFileFilterPaths !== null + await this.#removeObsoletePageOutputs(pageBuildResults, isFiltered) this.#updatePageLayoutNames(pageBuildResults.report.pages, !isFiltered) if (!isFiltered) { this.#pagesFileLayoutMap = getPagesFileLayoutMap(pageBuildResults.report.pages) - } else if ((pageBuildResults.report.rebuiltPagesFilePaths?.length ?? 0) > 0) { + } else { updatePagesFileLayoutMap(this.#pagesFileLayoutMap, pageBuildResults.report.rebuiltPagesFilePaths ?? [], pageBuildResults.report.pages) } this.#watchDependencies = pageBuildResults.report.watchDependencies ?? this.#watchDependencies - delete pageBuildResults.report.newClaims - delete pageBuildResults.report.replacedOwnerIds delete pageBuildResults.report.watchDependencies delete pageBuildResults.report.rebuiltPagesFilePaths await this.#rebuildMaps(siteData) @@ -797,6 +566,45 @@ export class DomStack { } } + /** + * Reconcile page ownership only after a successful page phase. Untouched page + * and template owners still protect their outputs during targeted builds. + * + * @param {Pick} results + * @param {boolean} isFiltered + */ + async #removeObsoletePageOutputs (results, isFiltered) { + const dest = resolve(this.#dest) + const rebuiltPages = getPageOutputMap(dest, results.report.pages) + const pages = isFiltered ? new Map(this.#pageOutputMap) : new Map() + const templates = isFiltered ? new Map(this.#templateOutputMap) : new Map() + + // Factories can successfully rebuild to zero pages; regular pages always + // report their HTML output, even when their additional-output hook is gone. + for (const owner of results.report.rebuiltPagesFilePaths ?? []) pages.delete(owner) + for (const [owner, outputs] of rebuiltPages) pages.set(owner, outputs) + for (const report of results.report.templates) { + templates.set(report.templateInfo.templateFile.filepath, new Set( + report.outputs.map(output => resolve(dest, report.templateInfo.path, output)) + )) + } + + const claimed = new Set(results.outputs.map(output => resolve(dest, output.outputRelname))) + for (const outputs of [...pages.values(), ...templates.values()]) { + for (const filepath of outputs) claimed.add(filepath) + } + const stale = new Set() + for (const outputs of this.#pageOutputMap.values()) { + for (const filepath of outputs) { + if (!claimed.has(filepath)) stale.add(filepath) + } + } + for (const filepath of stale) await removeStalePageOutput(dest, filepath) + + this.#pageOutputMap = pages + this.#templateOutputMap = templates + } + /** * @param {WatchSession} session * @param {() => Promise} fn @@ -995,18 +803,11 @@ export class DomStack { Promise.resolve().then(() => this.#esbuildContext?.dispose()), Promise.resolve().then(() => this.#syncServer?.exit()), ])) - await this.#outputLock - results.push(...await Promise.allSettled([...this.#cpxWatchStages, ...(this.#initialStage ? [this.#initialStage] : [])].map(stage => rm(stage, { recursive: true, force: true })))) - this.#initialStage = null - this.#initialAdditionalOutputs.clear() this.#watcher = null this.#cpxWatchers = [] - this.#cpxWatchStages = [] - this.#pendingCopyUpdates.clear() this.#esbuildContext = null this.#syncServer = null this.#siteData = null - this.#outputClaims = [] this.#buildLock = Promise.resolve() this.#watchSession = null const errors = results.filter(result => result.status === 'rejected').map(result => result.reason) @@ -1019,23 +820,47 @@ export class DomStack { */ async settled () { await this.#buildLock - await this.#outputLock } } /** - * @param {OutputClaim[]} claims - * @param {DomstackManifestRecord[]} outputs - * @param {'browser' | 'service-worker'} phase - * @param {boolean} caseInsensitive - * @returns {OutputClaim[]} + * @param {string} dest + * @param {WatchedPageReport[]} pageReports + * @returns {Map>} */ -function replaceEsbuildClaims (claims, outputs, phase, caseInsensitive) { - const prefix = `esbuild:${phase}:` - const replaceOwnerIds = claims.filter(claim => claim.owner.id.startsWith(prefix)).map(claim => claim.owner.id) - const registry = new OutputRegistry(claims, { replaceOwnerIds, caseInsensitive }) - registry.claimRecords(outputs, prefix) - return registry.snapshot() +function getPageOutputMap (dest, pageReports) { + /** @type {Map>} */ + const outputsByOwner = new Map() + for (const report of pageReports) { + const owner = report.pagesFilePath ?? report.sourcePageFilePath + if (!owner) continue + const outputs = outputsByOwner.get(owner) ?? new Set() + for (const output of report.outputs ?? []) outputs.add(resolve(dest, output.outputRelname)) + outputsByOwner.set(owner, outputs) + } + return outputsByOwner +} + +/** + * Never follow a replaced output directory outside the destination. A symlink + * at the output itself is safe to unlink; directories are never removed. + * @param {string} dest + * @param {string} filepath + */ +async function removeStalePageOutput (dest, filepath) { + assertInsideDest(dest, filepath) + if (filepath === dest) throw new Error('Refusing to remove the build destination') + try { + for (let ancestor = dirname(filepath); ; ancestor = dirname(ancestor)) { + const stats = await lstat(ancestor) + if (stats.isSymbolicLink() || !stats.isDirectory()) return + if (ancestor === dest) break + } + const stats = await lstat(filepath) + if (!stats.isDirectory()) await rm(filepath, { force: true }) + } catch (err) { + if (/** @type {NodeJS.ErrnoException} */ (err).code !== 'ENOENT') throw err + } } /** diff --git a/lib/build-copy/index.js b/lib/build-copy/index.js index 874f1175..a213eca0 100644 --- a/lib/build-copy/index.js +++ b/lib/build-copy/index.js @@ -1,11 +1,10 @@ /** - * @import { BuildStepResult, BuildStep, DomStackOpts } from '../builder.js' + * @import { BuildStepResult, BuildStep } from '../builder.js' */ -/** @import { copy } from 'cpx2' */ +import { copy } from 'cpx2' import { join } from 'node:path' -import { stagedCopy } from '../helpers/staged-copy.js' -import { OutputRegistry } from '../output-registry.js' +import { createCopiedDomstackManifestRecords } from '../helpers/cpx2-report.js' /** * @typedef {Record>>} CopyBuilderReport @@ -25,13 +24,9 @@ export function getCopyDirs (copy = []) { /** * run CPX2 on src folder * - * @param {string} src - * @param {string} dest - * @param {unknown} _siteData - * @param {DomStackOpts | null} [opts] - * @param {OutputRegistry} [registry] + * @type {CopyBuildStep} */ -export async function buildCopy (src, dest, _siteData, opts, registry = new OutputRegistry()) { +export async function buildCopy (src, dest, _siteData, opts) { /** @type {CopyBuildStepResult} */ const results = { type: 'copy', @@ -43,10 +38,8 @@ export async function buildCopy (src, dest, _siteData, opts, registry = new Outp const copyDirs = getCopyDirs(opts?.copy) - // Each configured root is a producer, even when roots overlap or repeat. - // Keep this prefix identical to the live watch inventory's mapping identity. - const copyTasks = copyDirs.map((copyDir, index) => { - return stagedCopy(copyDir, src, dest, 'copy', registry, [], `copy-root:${index}:`) + const copyTasks = copyDirs.map((copyDir) => { + return copy(copyDir, dest) }) const settled = await Promise.allSettled(copyTasks) @@ -58,8 +51,13 @@ export async function buildCopy (src, dest, _siteData, opts, registry = new Outp } else { const copyDir = copyDirs[index] if (!copyDir) continue - results.report[copyDir] = result.value.report - results.outputs.push(...result.value.outputs) + results.report[copyDir] = result.value + results.outputs.push(...createCopiedDomstackManifestRecords({ + src, + dest, + report: result.value, + kind: 'copy', + })) } } return results diff --git a/lib/build-esbuild/index.js b/lib/build-esbuild/index.js index c705f9ef..7dacd392 100644 --- a/lib/build-esbuild/index.js +++ b/lib/build-esbuild/index.js @@ -4,11 +4,9 @@ * @import { Logger as PinoLogger } from 'pino' */ -import { mkdir, writeFile } from 'fs/promises' -import { join, relative, basename, dirname, resolve, extname } from 'path' +import { writeFile } from 'fs/promises' +import { join, relative, basename, resolve, extname } from 'path' import esbuild from 'esbuild' -import { OutputRegistry } from '../output-registry.js' -import { rethrowEsbuildOutputConflict, validateEsbuildEntryOutputs } from './output-conflicts.js' import { globalBundleAssets, pageBundleAssets, layoutBundleAssets } from '../file-conventions.js' import { resolveVars } from '../build-pages/resolve-vars.js' import { @@ -217,7 +215,7 @@ async function createBrowserBuildOpts (src, dest, siteData, opts, modeOpts = {}) /** @type {EsbuildLogLevel} */ logLevel: 'silent', bundle: true, - write: false, + write: true, /** @type {EsbuildFormat} */ format: 'esm', splitting: true, @@ -263,10 +261,6 @@ async function createBrowserBuildOpts (src, dest, siteData, opts, modeOpts = {}) return { ...extendedBuildOpts, - // DomStack must own the write boundary and inventory even when settings - // override these options. opts.metafile only controls publishing the JSON. - write: false, - metafile: true, define: preserveDomstackDefines(extendedBuildOpts.define, domstackDefines), } } @@ -293,19 +287,6 @@ function preserveDomstackDefines (define, domstackDefines) { return mergedDefine } -/** - * @param {esbuild.BuildResult} result - * @param {string} [dest] - * @param {string} [writeDest] - */ -async function writeEsbuildOutputFiles (result, dest, writeDest) { - for (const output of result.outputFiles ?? []) { - const target = dest && writeDest ? resolve(writeDest, relative(dest, output.path)) : output.path - await mkdir(dirname(target), { recursive: true }) - await writeFile(target, output.contents) - } -} - /** * @param {object} params * @param {string} params.dest @@ -342,33 +323,23 @@ function emptyEsbuildReport () { /** * Build all of the bundles using esbuild. * - * @param {string} src - * @param {string} dest - * @param {SiteData} siteData - * @param {DomStackOpts | null} opts - * @param {OutputRegistry} [registry] - * @param {boolean} [watch] - * @param {string} [publicDest] - Compile paths relative to the published destination, even when staging writes. - * @returns {Promise} + * @type {EsBuildStep} */ -export async function buildEsbuild (src, dest, siteData, opts, registry = new OutputRegistry(), watch = false, publicDest = dest) { +export async function buildEsbuild (src, dest, siteData, opts) { try { - const extendedBuildOpts = await createBrowserBuildOpts(src, publicDest, siteData, opts, { watch }) + const extendedBuildOpts = await createBrowserBuildOpts(src, dest, siteData, opts, { watch: false }) - const buildResults = await buildControlled(extendedBuildOpts) + const buildResults = await esbuild.build(extendedBuildOpts) - const outputMap = applyBuildOutputMap({ dest: publicDest, result: buildResults, siteData, src }) + await writeMetafile({ dest, result: buildResults, shouldWrite: opts?.metafile !== false }) + const outputMap = applyBuildOutputMap({ dest, result: buildResults, siteData, src }) const outputs = createEsbuildOutputRecords({ src, - dest: publicDest, + dest, siteData, buildResults, includeMetafileRecord: opts?.metafile !== false, }) - registry.claimRecords(outputs, 'esbuild:browser:') - for (const output of outputs) output.filepath = resolve(dest, output.outputRelname) - await writeEsbuildOutputFiles(buildResults, publicDest, dest) - await writeMetafile({ dest, result: buildResults, shouldWrite: opts?.metafile !== false }) return { type: 'esbuild', @@ -405,10 +376,9 @@ export async function buildEsbuild (src, dest, siteData, opts, registry = new Ou * @param {SiteData} siteData * @param {EsbuildBuildOptions | undefined} browserBuildOpts * @param {ServiceWorkerBuildDefines} [defines] - * @param {OutputRegistry} [registry] * @returns {Promise} */ -export async function buildServiceWorkerEsbuild (src, dest, siteData, browserBuildOpts, defines = {}, registry = new OutputRegistry()) { +export async function buildServiceWorkerEsbuild (src, dest, siteData, browserBuildOpts, defines = {}) { if (!siteData.serviceWorker) { return { type: 'esbuild', @@ -429,26 +399,14 @@ export async function buildServiceWorkerEsbuild (src, dest, siteData, browserBui serviceWorker: siteData.serviceWorker, src, }) - const serviceWorkerBuildResults = await buildControlled(serviceWorkerBuildOpts) - - const publicDest = serviceWorkerBuildOpts.outdir ?? dest - const outputMap = applyBuildOutputMap({ dest: publicDest, result: serviceWorkerBuildResults, siteData, src }) - const outputs = createEsbuildOutputRecords({ - src, - dest: publicDest, - siteData, - buildResults: serviceWorkerBuildResults, - includeMetafileRecord: false, - }) - registry.claimRecords(outputs, 'esbuild:service-worker:') - for (const output of outputs) output.filepath = resolve(dest, output.outputRelname) - await writeEsbuildOutputFiles(serviceWorkerBuildResults, publicDest, dest) + const serviceWorkerBuildResults = await esbuild.build(serviceWorkerBuildOpts) + const outputMap = applyBuildOutputMap({ dest, result: serviceWorkerBuildResults, siteData, src }) return { type: 'esbuild', errors: serviceWorkerBuildResults.errors, warnings: serviceWorkerBuildResults.warnings, - outputs, + outputs: [], report: { buildResults: serviceWorkerBuildResults, buildOpts: serviceWorkerBuildOpts, @@ -545,8 +503,8 @@ function createDomstackDefines ({ opts, siteData, watch }) { * @param {string} dest * @param {SiteData} siteData * @param {DomStackOpts} opts - * @param {{ onEnd?: (result: esbuild.BuildResult) => void, logger?: PinoLogger, writeDest?: () => string, promoteOutputs?: (outputs: DomstackManifestRecord[], phase: 'browser' | 'service-worker', write: () => Promise) => Promise }} [watchOpts] - * @returns {Promise<{ context: DisposableBuildContext, outputMap: OutputMap, outputs: DomstackManifestRecord[], buildResults: esbuild.BuildResult, buildOpts: EsbuildBuildOptions }>} + * @param {{ onEnd?: (result: esbuild.BuildResult) => void, logger?: PinoLogger }} [watchOpts] + * @returns {Promise<{ context: DisposableBuildContext, outputMap: OutputMap, buildResults: esbuild.BuildResult, buildOpts: EsbuildBuildOptions }>} */ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = {}) { const logger = watchOpts.logger ?? opts.logger ?? createDomStackLogger() @@ -556,12 +514,8 @@ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = dest, label: 'JS/CSS', logger, - ...(watchOpts.onEnd ? { onEnd: watchOpts.onEnd } : {}), - promote: (result, write) => watchOpts.promoteOutputs - ? watchOpts.promoteOutputs(createEsbuildOutputRecords({ src, dest, siteData, buildResults: result, includeMetafileRecord: opts?.metafile !== false }), 'browser', write) - : write(), + onEnd: watchOpts.onEnd, shouldWriteMetafile: opts?.metafile !== false, - ...(watchOpts.writeDest ? { writeDest: watchOpts.writeDest } : {}), }) const initialResult = browserWatch.initialResult @@ -570,13 +524,6 @@ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = const contexts = [browserWatch.context] try { const outputMap = applyBuildOutputMap({ dest, result: initialResult, siteData, src }) - const outputs = createEsbuildOutputRecords({ - src, - dest, - siteData, - buildResults: initialResult, - includeMetafileRecord: opts?.metafile !== false, - }) if (siteData.serviceWorker) { // Keep service-worker-only defines and no-policy watch cleanup behavior out of browser bundles. @@ -591,11 +538,7 @@ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = dest, label: 'Service worker', logger, - promote: (result, write) => watchOpts.promoteOutputs - ? watchOpts.promoteOutputs(createEsbuildOutputRecords({ src, dest, siteData, buildResults: result, includeMetafileRecord: false }), 'service-worker', write) - : write(), shouldWriteMetafile: false, - ...(watchOpts.writeDest ? { writeDest: watchOpts.writeDest } : {}), }) contexts.push(serviceWorkerWatch.context) applyBuildOutputMap({ @@ -604,20 +547,11 @@ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = siteData, src, }) - outputs.push(...createEsbuildOutputRecords({ - src, - dest, - siteData, - buildResults: serviceWorkerWatch.initialResult, - includeMetafileRecord: false, - })) } - if (!siteData.serviceWorker) await watchOpts.promoteOutputs?.([], 'service-worker', async () => {}) return { context: createDisposableBuildContext(contexts), outputMap, - outputs, buildResults: initialResult, buildOpts: extendedBuildOpts, } @@ -636,13 +570,10 @@ export async function buildEsbuildWatch (src, dest, siteData, opts, watchOpts = * @param {string} params.label * @param {PinoLogger} params.logger * @param {(result: esbuild.BuildResult) => void | Promise} [params.onEnd] - * @param {(result: esbuild.BuildResult, write: () => Promise) => Promise} [params.promote] * @param {boolean} params.shouldWriteMetafile - * @param {() => string} [params.writeDest] * @returns {Promise<{ context: esbuild.BuildContext, initialResult: esbuild.BuildResult }>} */ -async function createWatchBuild ({ buildOpts, dest, label, logger, onEnd, promote, shouldWriteMetafile, writeDest }) { - validateEsbuildEntryOutputs(buildOpts) +async function createWatchBuild ({ buildOpts, dest, label, logger, onEnd, shouldWriteMetafile }) { const initial = Promise.withResolvers() // Attach a rejection handler before watch() can deliver a failing initial build. initial.promise.catch(() => {}) @@ -658,7 +589,6 @@ async function createWatchBuild ({ buildOpts, dest, label, logger, onEnd, promot isInitialBuild = false try { if (result.errors.length > 0) { - rethrowEsbuildOutputConflict(result.errors, buildOpts) const failure = Object.assign(new Error(`${label} build failed`), { errors: result.errors.map(serializeEsbuildMessage), warnings: result.warnings.map(serializeEsbuildMessage), @@ -672,13 +602,7 @@ async function createWatchBuild ({ buildOpts, dest, label, logger, onEnd, promot if (result.warnings.length) { logger.warn({ warnings: result.warnings.map(serializeEsbuildMessage) }, `${label} build warnings`) } - const write = async () => { - const target = writeDest?.() ?? dest - await writeEsbuildOutputFiles(result, dest, target) - await writeMetafile({ dest: target, result, shouldWrite: shouldWriteMetafile }) - } - if (promote) await promote(result, write) - else await write() + await writeMetafile({ dest, result, shouldWrite: shouldWriteMetafile }) if (first) logger.debug(`${label} initial build complete`) else logger.info(`${label} rebuild complete`) } @@ -709,18 +633,6 @@ async function createWatchBuild ({ buildOpts, dest, label, logger, onEnd, promot } } -/** @param {esbuild.BuildOptions} opts */ -async function buildControlled (opts) { - validateEsbuildEntryOutputs(opts) - try { - return await esbuild.build(opts) - } catch (error) { - const failure = /** @type {esbuild.BuildFailure} */ (error) - rethrowEsbuildOutputConflict(failure.errors ?? [], opts) - throw error - } -} - /** * @param {esbuild.BuildContext[]} contexts * @returns {DisposableBuildContext} @@ -776,12 +688,8 @@ export function createEsbuildOutputRecords ({ src, dest, siteData, buildResults, filepath, outputRelname, kind, - ...(outputMeta.entryPoint - ? { - entryPoint: outputMeta.entryPoint, - sourceRelname: toPosix(relative(src, resolve(outputMeta.entryPoint))), - } - : {}), + entryPoint: outputMeta.entryPoint, + sourceRelname: outputMeta.entryPoint ? toPosix(relative(src, resolve(outputMeta.entryPoint))) : undefined, })) } diff --git a/lib/build-esbuild/output-conflicts.js b/lib/build-esbuild/output-conflicts.js deleted file mode 100644 index b7328fcc..00000000 --- a/lib/build-esbuild/output-conflicts.js +++ /dev/null @@ -1,70 +0,0 @@ -/** @import { BuildOptions, Message } from 'esbuild' */ -import { basename, dirname, extname, join, relative, resolve } from 'node:path' -import { OutputRegistry } from '../output-registry.js' -import { toPosix } from '../helpers/path.js' - -/** Entry output patterns, using esbuild's configured base, aliases and extensions. - * @param {BuildOptions} opts - */ -function entryOutputs (opts) { - const entries = Array.isArray(opts.entryPoints) - ? opts.entryPoints.map(entry => typeof entry === 'string' ? { in: entry, out: undefined } : entry) - : Object.entries(opts.entryPoints ?? {}).map(([out, input]) => ({ in: input, out })) - const cwd = opts.absWorkingDir ?? process.cwd() - const base = resolve(cwd, opts.outbase ?? '.') - return entries.map(entry => { - const extension = extname(entry.in) - const outputExtension = extension === '.css' ? '.css' : '.js' - const name = basename(entry.in, extension) - const dir = toPosix(relative(base, dirname(resolve(cwd, entry.in)))) - const pattern = (entry.out ?? (opts.entryNames ?? '[dir]/[name]') - .replaceAll('[dir]', dir || '.') - .replaceAll('[name]', name) - .replaceAll('[ext]', (opts.outExtension?.[outputExtension] ?? outputExtension).slice(1))) + (opts.outExtension?.[outputExtension] ?? outputExtension) - return { pattern: toPosix(join(pattern)), source: entry.in } - }) -} - -/** Reject statically identifiable entry collisions, including identical contents - * that esbuild would silently coalesce. Hashed names are checked on native errors. - * @param {BuildOptions} opts - */ -export function validateEsbuildEntryOutputs (opts) { - if (opts.outfile || !opts.outbase) return - const registry = new OutputRegistry() - const seen = new Set() - for (const { pattern, source } of entryOutputs(opts)) { - if (pattern.includes('[hash]') || pattern.startsWith('../')) continue - const key = JSON.stringify([pattern, source]) - if (seen.has(key)) continue - seen.add(key) - registry.claim(pattern, { id: source, type: 'esbuild', path: source }) - } -} - -/** Attach domain conflict diagnostics when the native collision can be traced to - * two configured entries. Unidentifiable plugin/chunk errors stay native rather - * than inventing producer attribution or rerunning user plugins. - * @param {Message[]} errors - * @param {BuildOptions} opts - */ -export function rethrowEsbuildOutputConflict (errors, opts) { - if (!opts.outdir || !opts.outbase) return - for (const error of errors) { - const match = /Two output files share the same path but have different contents: (.+)$/.exec(error.text) - if (!match?.[1]) continue - const path = toPosix(relative(resolve(opts.absWorkingDir ?? process.cwd(), opts.outdir), resolve(opts.absWorkingDir ?? process.cwd(), match[1]))) - const candidates = entryOutputs(opts).filter(({ pattern }) => { - const patterns = [pattern] - const jsExtension = opts.outExtension?.['.js'] ?? '.js' - if (pattern.endsWith(jsExtension)) patterns.push(pattern.slice(0, -jsExtension.length) + (opts.outExtension?.['.css'] ?? '.css')) - return patterns.some(pattern => { - const regex = pattern.split('[hash]').map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('[^/]+') - return new RegExp(`^${regex}(?:\\.map)?$`).test(path) - }) - }) - if (new Set(candidates.map(entry => entry.source)).size < 2) continue - const registry = new OutputRegistry() - for (const entry of candidates) registry.claim(path, { id: entry.source, type: 'esbuild', path: entry.source }) - } -} diff --git a/lib/build-pages/index.js b/lib/build-pages/index.js index 67ca76ab..64db0268 100644 --- a/lib/build-pages/index.js +++ b/lib/build-pages/index.js @@ -6,12 +6,10 @@ * @import { ResolvedLayout } from './page-data.js' * @import { DomstackManifestRecord } from '../domstack-manifest/index.js' * @import { WatchDependencyState, WatchConsumer, WatchDependencyTracker } from './watch-dependencies.js' - * @import { OutputClaim, OutputOwner } from '../output-registry.js' */ import { Worker } from 'worker_threads' -import { basename, dirname, isAbsolute, join, normalize, relative, resolve } from 'path' -import { copyFile, mkdir, mkdtemp, rm } from 'node:fs/promises' +import { basename, dirname, isAbsolute, join, normalize, resolve } from 'path' import pMap from 'p-map' import { cpus } from 'os' import { keyBy } from '../helpers/key-by.js' @@ -25,9 +23,7 @@ import { DomStackDataError, DomStackOutputConflictError } from '../helpers/domst import { isAsyncIterable, isPlainObject } from '../helpers/type-guards.js' import { createSubscribedData, resolveDataDeps } from './data-deps.js' import { WatchDependencyTracker as WatchDependencyTrackerClass } from './watch-dependencies.js' -import { OutputRegistry, isCaseInsensitiveDest } from '../output-registry.js' -import { assertInsideDest } from '../helpers/path.js' -import { prepareAdditionalOutputPromotion } from '../helpers/additional-output-promotion.js' +import { outputWarnings } from '../helpers/output-warnings.js' const MAX_CONCURRENCY = Math.min(cpus().length, 24) @@ -49,8 +45,6 @@ const __dirname = import.meta.dirname * @property {TemplateReport[]} templates * @property {WatchDependencyState | undefined} [watchDependencies] * @property {string[] | undefined} [rebuiltPagesFilePaths] - * @property {OutputClaim[] | undefined} [newClaims] - Claims produced by the selected owners. - * @property {string[]} [replacedOwnerIds] */ /** @@ -96,10 +90,6 @@ const __dirname = import.meta.dirname * @property {boolean | undefined} [buildDrafts] - Include generated page definitions marked as drafts. * @property {WatchDependencyState | null | undefined} [previousWatchDependencies] - Dependency state from the previous successful watch build. * @property {boolean | undefined} [trackWatchDependencies] - Collect subscriptions for incremental watch builds. - * @property {OutputClaim[] | null | undefined} [previousOutputClaims] - Output ownership from the latest successful build. - - * @property {boolean} [caseInsensitive] - * @property {(report: PageBuildStepResult, write: () => Promise, preflight: (removablePaths: string[]) => Promise) => Promise} [promoteOutputs] */ /** @@ -328,7 +318,7 @@ function normalizeGeneratedOutputPart (value, { field, allowEmpty = false }) { if (value.split(/[\\/]+/).includes('..')) throw new Error(`Generated page ${field} must not contain ".." segments: ${value}`) if (/[\\/]$/.test(value)) throw new Error(`Generated page ${field} must name a file: ${value}`) - const normalized = normalize(value.replaceAll('\\', '/')) + const normalized = normalize(value) if (!allowEmpty && normalized === '.') throw new Error(`Generated page ${field} must not be empty`) return normalized === '.' ? '' : normalized } @@ -375,12 +365,31 @@ function generatedDefinitionToPageInfo ({ definition, pagesFile, index }) { * @param {Set | null} params.pagesFileFilterSet * @param {boolean | undefined} params.buildDrafts * @param {WatchDependencyTracker} params.watchDependencyTracker - * @param {OutputRegistry} params.outputRegistry * @returns {Promise} */ -async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, pagesFileFilterSet, buildDrafts, watchDependencyTracker, outputRegistry }) { +async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, pagesFileFilterSet, buildDrafts, watchDependencyTracker }) { /** @type {PageInfo[]} */ const generatedPageInfos = [] + /** @type {Map} */ + const pageOutputClaims = new Map() + + for (const pageInfo of siteData.pages) { + pageOutputClaims.set(resolve(pageInfo.outputRelname), { + type: 'page', + path: pageInfo.pageFile.relname, + }) + } + + // Unselected factories keep their outputs. Reserve those paths without + // rerunning the owners, so a targeted build cannot silently overwrite them. + if (pagesFileFilterSet) { + const ownerRelnames = new Map((siteData.pagesFiles ?? []).map(({ pagesFile }) => [pagesFile.filepath, pagesFile.relname])) + for (const consumer of Object.values(watchDependencyTracker.state.consumers)) { + if (consumer.type === 'page' && consumer.ownerPath && !pagesFileFilterSet.has(consumer.ownerPath)) { + pageOutputClaims.set(resolve(consumer.key), { type: 'page', path: ownerRelnames.get(consumer.ownerPath) ?? consumer.key }) + } + } + } for (const pagesFile of siteData.pagesFiles ?? []) { if (pagesFileFilterSet && !pagesFileFilterSet.has(pagesFile.pagesFile.filepath)) continue @@ -417,7 +426,24 @@ async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, p const generatedPageInfo = generatedDefinitionToPageInfo({ definition, pagesFile, index }) if (generatedPageInfo.draft && !buildDrafts) continue - outputRegistry.claim(generatedPageInfo.outputRelname, pageOutputOwner(generatedPageInfo)) + const outputKey = resolve(generatedPageInfo.outputRelname) + const existingClaim = pageOutputClaims.get(outputKey) + const generatedClaim = { + type: /** @type {const} */ ('page'), + path: generatedPageInfo.pageFile.relname, + } + if (existingClaim) { + throw new DomStackOutputConflictError( + `Output path conflict: ${generatedPageInfo.outputRelname} is produced by both ${existingClaim.path} and ${generatedClaim.path}.`, + { + outputPath: generatedPageInfo.outputRelname, + a: existingClaim, + b: generatedClaim, + } + ) + } + + pageOutputClaims.set(outputKey, generatedClaim) generatedPageInfos.push(generatedPageInfo) } } catch (err) { @@ -437,99 +463,50 @@ async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, p * * @type {PageBuildStep} */ -export async function buildPages (src, dest, siteData, opts) { - await mkdir(dest, { recursive: true }) - const stageDest = await mkdtemp(join(dest, '.domstack-pages-')) - try { - // Only page-build filters cross the worker boundary. General build options - // can contain functions (manifest hooks and predicates) or logger instances, - // neither of which can be structured-cloned. - /** @type {BuildPagesFilterOptions} */ - const workerOpts = { - pageFilterPaths: opts?.pageFilterPaths, - templateFilterPaths: opts?.templateFilterPaths, - pagesFileFilterPaths: opts?.pagesFileFilterPaths, - buildDrafts: opts?.buildDrafts, - previousWatchDependencies: opts?.previousWatchDependencies, - trackWatchDependencies: opts?.trackWatchDependencies, - previousOutputClaims: opts?.previousOutputClaims, - caseInsensitive: opts?.caseInsensitive ?? await isCaseInsensitiveDest(dest), - } - - const buildReport = await new Promise((resolve, reject) => { - const worker = new Worker(join(__dirname, 'worker.js'), { - workerData: { src, dest: stageDest, siteData, opts: workerOpts }, - }) - - worker.once('message', message => { - /** @type { WorkerBuildStepResult } */ - const workerReport = message - - /** @type {PageBuildStepResult} */ - const report = { - type: workerReport.type, - report: workerReport.report, - outputs: workerReport.outputs, - errors: [], - warnings: workerReport.warnings ?? [], - } +export function buildPages (src, dest, siteData, opts) { + // Only page-build filters cross the worker boundary. General build options + // can contain functions (manifest hooks and predicates) or logger instances, + // neither of which can be structured-cloned. + /** @type {BuildPagesFilterOptions} */ + const workerOpts = { + pageFilterPaths: opts?.pageFilterPaths, + templateFilterPaths: opts?.templateFilterPaths, + pagesFileFilterPaths: opts?.pagesFileFilterPaths, + buildDrafts: opts?.buildDrafts, + previousWatchDependencies: opts?.previousWatchDependencies, + trackWatchDependencies: opts?.trackWatchDependencies, + } - if (workerReport.errors.length > 0) { - report.errors = workerReport.errors.map(({ error, errorData = {} }) => { - return restoreWorkerError(error, errorData) - }) - } - resolve(report) - }) - worker.once('error', reject) - worker.once('exit', (code) => { - if (code !== 0) reject(new Error(`Worker stopped with exit code ${code}`)) - }) + return new Promise((resolve, reject) => { + const worker = new Worker(join(__dirname, 'worker.js'), { + workerData: { src, dest, siteData, opts: workerOpts }, }) - try { - if (buildReport.errors.length === 0) { - let unchanged = new Set() - const preflight = async (/** @type {string[]} */ removablePaths) => { - unchanged = await prepareAdditionalOutputPromotion(dest, buildReport.outputs, removablePaths) - } - const write = () => promotePageOutputs(dest, buildReport, unchanged) - if (opts?.promoteOutputs) await opts.promoteOutputs(buildReport, write, preflight) - else { - await preflight([]) - await write() - } + worker.once('message', message => { + /** @type { WorkerBuildStepResult } */ + const workerReport = message + + /** @type {PageBuildStepResult} */ + const buildReport = { + type: workerReport.type, + report: workerReport.report, + outputs: workerReport.outputs, + errors: [], + warnings: workerReport.warnings ?? [], } - return buildReport - } finally { - for (const output of buildReport.outputs) output.filepath = resolve(dest, output.outputRelname) - for (const page of buildReport.report.pages) page.pageFilePath = resolve(dest, relative(stageDest, page.pageFilePath)) - } - } finally { - await rm(stageDest, { recursive: true, force: true }) - } -} -/** - * Promote a successful page phase only after every output has rendered and all - * claims have been validated. Render/conflict failures do not alter the real dest; - * an I/O failure during promotion can leave partially updated files. - * - * @param {string} dest - * @param {PageBuildStepResult} buildReport - * @param {Set} unchanged - */ -async function promotePageOutputs (dest, buildReport, unchanged) { - const resolvedDest = resolve(dest) - for (const output of buildReport.outputs) { - const target = resolve(resolvedDest, output.outputRelname) - assertInsideDest(resolvedDest, target) - if (!unchanged.has(output.outputRelname)) { - await mkdir(dirname(target), { recursive: true }) - await copyFile(output.filepath, target) - } - output.filepath = target - } + if (workerReport.errors.length > 0) { + buildReport.errors = workerReport.errors.map(({ error, errorData = {} }) => { + return restoreWorkerError(error, errorData) + }) + } + resolve(buildReport) + }) + worker.once('error', reject) + worker.once('exit', (code) => { + if (code !== 0) { reject(new Error(`Worker stopped with exit code ${code}`)) } + }) + }) } /** @@ -567,10 +544,6 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { enabled: opts?.trackWatchDependencies === true, } ) - const outputRegistry = new OutputRegistry(opts?.previousOutputClaims ?? [], { - - caseInsensitive: opts?.caseInsensitive ?? false, - }) // Note: markdown-it settings are now passed directly to builders through builderOptions @@ -678,29 +651,8 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { }) } - // Select once, after global-data invalidation has expanded the filters. Include - // deleted producers and factories that now emit no files, not just render results. - const replacedOwnerIds = new Set([ - ...(opts?.previousOutputClaims ?? []).filter(({ owner }) => { - if (owner.id.startsWith('page:')) return !pageFilterSet || pageFilterSet.has(owner.id.slice(5)) - if (owner.id.startsWith('template:')) return !templateFilterSet || templateFilterSet.has(owner.id.slice(9)) - if (owner.id.startsWith('pages-file:')) return !pagesFileFilterSet || pagesFileFilterSet.has(owner.id.slice(11)) - return false - }).map(({ owner }) => owner.id), - ...siteData.pages.filter(page => !pageFilterSet || pageFilterSet.has(page.pageFile.filepath)).map(page => pageOutputOwner(page).id), - ...siteData.templates.filter(template => !templateFilterSet || templateFilterSet.has(template.templateFile.filepath)).map(template => `template:${template.templateFile.filepath}`), - ...(siteData.pagesFiles ?? []).filter(file => !pagesFileFilterSet || pagesFileFilterSet.has(file.pagesFile.filepath)).map(file => `pages-file:${file.pagesFile.filepath}`), - ]) - outputRegistry.releaseOwnerIds(replacedOwnerIds) - result.report.replacedOwnerIds = [...replacedOwnerIds] - let generatedPageInfos = /** @type {PageInfo[]} */ ([]) try { - for (const page of siteData.pages) { - if (replacedOwnerIds.has(pageOutputOwner(page).id) || !opts?.previousOutputClaims) { - outputRegistry.claim(page.outputRelname, pageOutputOwner(page)) - } - } generatedPageInfos = await resolveGeneratedPageInfos({ siteData, factoryVars: globalVars, @@ -708,7 +660,6 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { pagesFileFilterSet, buildDrafts: opts?.buildDrafts, watchDependencyTracker, - outputRegistry, }) } catch (err) { const pagesFile = /** @type {PagesFileInfo | undefined} */ (err instanceof Error && 'pagesFile' in err ? err.pagesFile : undefined) @@ -760,32 +711,12 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { : (siteData.pagesFiles ?? []).map(pagesFile => pagesFile.pagesFile.filepath) } - try { - for (const page of pagesToWrite) { - const owner = pageOutputOwner(page.pageInfo) - - if (page.pageInfo.workers && Object.values(page.pageInfo.workers).some(worker => worker.outputRelname)) { - outputRegistry.claim(join(page.pageInfo.path, 'workers.json'), owner) - } - } - } catch (err) { - result.errors.push(serializeBuildError(err)) - return result - } - await Promise.all([ pMap(pagesToWrite, async (page) => { try { const buildResult = await pageWriter({ dest, page, - claimAdditionalOutput: (outputRelname, provenance) => { - const owner = pageOutputOwner(page.pageInfo) - outputRegistry.claim(outputRelname, { - ...owner, - path: provenance ? `${owner.path} (${provenance.kind} additionalOutputs: ${provenance.source})` : owner.path, - }) - }, }) result.report.pages.push({ @@ -809,7 +740,6 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { globalData, template, watchDependencyTracker, - outputRegistry, }) result.report.templates.push(buildResult.report) @@ -820,13 +750,8 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { }, { concurrency: dividedConcurrency[1] }), ]) - result.report.newClaims = outputRegistry.snapshot() - .filter(claim => replacedOwnerIds.has(claim.owner.id)) - .map(claim => claim.owner.id.startsWith('pages-file:') - // Persist the factory's source identity, not its transient definition index. - ? { ...claim, owner: { ...claim.owner, path: claim.owner.path.replace(/#\d+$/, '') } } - : claim) if (opts?.trackWatchDependencies) { + result.warnings.push(...outputWarnings(result.outputs)) watchDependencyTracker.pruneGeneratedPages( new Set(generatedPages.map(page => page.pageInfo.outputRelname)), pagesFileFilterSet @@ -836,17 +761,6 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { return result } -/** - * @param {PageInfo} pageInfo - * @returns {OutputOwner} - */ -function pageOutputOwner (pageInfo) { - const pagesFile = pageInfo.generated?.pagesFile.pagesFile - return pagesFile - ? { id: `pages-file:${pagesFile.filepath}`, type: 'page', path: pageInfo.pageFile.relname } - : { id: `page:${pageInfo.pageFile.filepath}`, type: 'page', path: pageInfo.pageFile.relname } -} - /** * Add invalidated consumers to the mutable filters for a targeted build. * diff --git a/lib/build-pages/page-builders/additional-output-writer.js b/lib/build-pages/page-builders/additional-output-writer.js index 148a7863..74466eca 100644 --- a/lib/build-pages/page-builders/additional-output-writer.js +++ b/lib/build-pages/page-builders/additional-output-writer.js @@ -1,9 +1,9 @@ /** * @import { PageInfo } from '../../identify-pages.js' * @import { DomstackManifestRecord } from '../../domstack-manifest/index.js' - * @import { AdditionalOutputProvenance } from '../additional-outputs.js' + */ -import { lstat, mkdir, writeFile } from 'node:fs/promises' +import { lstat, mkdir, readFile, writeFile } from 'node:fs/promises' import { dirname, isAbsolute, relative, resolve, sep } from 'node:path' import { createDomstackManifestRecord } from '../../domstack-manifest/index.js' @@ -47,8 +47,7 @@ export function resolveAdditionalOutputPath (dest, pageFilePath, outputName) { /** * Check existing components, including the leaf, without following symlinks. - * The caller owns a private stage; this is not a defense against concurrent - * hostile filesystem mutations or a substitute for final promotion checks. + * This is not a defense against concurrent external filesystem mutations. * * @param {string} dest * @param {string} filepath @@ -75,29 +74,35 @@ async function assertWritablePath (dest, filepath) { } /** - * Write only into the supplied build stage; ownership and promotion belong to - * build-pages. Extra hook provenance does not change the owning source page. + * Write sidecars directly, retaining ownership records even for unchanged bytes. * * @param {object} params * @param {string} params.dest * @param {string} params.pageFilePath * @param {PageInfo} params.pageInfo - * @param {Array<{outputName: string, content: string, provenance?: AdditionalOutputProvenance}>} params.additionalOutputs - * @param {(outputRelname: string, provenance?: AdditionalOutputProvenance) => void} [params.claimOutput] + * @param {Array<{outputName: string, content: string}>} params.additionalOutputs * @returns {Promise} */ -export async function writeAdditionalOutputs ({ dest, pageFilePath, pageInfo, additionalOutputs, claimOutput }) { +export async function writeAdditionalOutputs ({ dest, pageFilePath, pageInfo, additionalOutputs }) { const planned = additionalOutputs.map(output => { if (typeof output.content !== 'string') throw new TypeError('Additional output content must be a string') - return { ...resolveAdditionalOutputPath(dest, pageFilePath, output.outputName), content: output.content, provenance: output.provenance } + return { ...resolveAdditionalOutputPath(dest, pageFilePath, output.outputName), content: output.content } }) - // Reserve the whole batch before touching the stage, including duplicate records. - for (const output of planned) claimOutput?.(output.outputRelname, output.provenance) + // Check every path before writing this page's batch. for (const output of planned) await assertWritablePath(dest, output.filepath) const records = [] for (const { filepath, outputRelname, content } of planned) { - await mkdir(dirname(filepath), { recursive: true }) - await writeFile(filepath, content) + const bytes = Buffer.from(content) + let unchanged = false + try { + unchanged = bytes.equals(await readFile(filepath)) + } catch (error) { + if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error + } + if (!unchanged) { + await mkdir(dirname(filepath), { recursive: true }) + await writeFile(filepath, bytes) + } records.push({ ...createDomstackManifestRecord({ dest, diff --git a/lib/build-pages/page-builders/additional-output-writer.test.js b/lib/build-pages/page-builders/additional-output-writer.test.js index 837aed27..c1ced5fa 100644 --- a/lib/build-pages/page-builders/additional-output-writer.test.js +++ b/lib/build-pages/page-builders/additional-output-writer.test.js @@ -47,7 +47,7 @@ test('additional output paths reject escapes, directory names and nonportable fi } }) -test('writer stages sidecars with page ownership and non-navigation JSON records', async t => { +test('writer writes sidecars with page ownership and non-navigation JSON records', async t => { const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-')) t.after(() => rm(dest, { recursive: true, force: true })) const outputs = await writeAdditionalOutputs({ @@ -72,7 +72,7 @@ test('writer stages sidecars with page ownership and non-navigation JSON records test('writer rejects symlink components and existing directories without touching their targets', async t => { const root = await mkdtemp(join(tmpdir(), 'domstack-additional-links-')) t.after(() => rm(root, { recursive: true, force: true })) - const dest = join(root, 'stage') + const dest = join(root, 'dest') const outside = join(root, 'outside') await mkdir(dest) await mkdir(outside) @@ -103,7 +103,7 @@ test('writer validates all sidecar paths before writing the batch', async t => { await assert.rejects(readFile(join(dest, 'valid.json')), { code: 'ENOENT' }) }) -test('writer preserves duplicate records for main to reject through output claims', async t => { +test('writer preserves duplicate records for duplicate-output warnings', async t => { const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-claims-')) t.after(() => rm(dest, { recursive: true, force: true })) const outputs = await writeAdditionalOutputs({ diff --git a/lib/build-pages/page-builders/page-writer.js b/lib/build-pages/page-builders/page-writer.js index 53b52495..a49bbef6 100644 --- a/lib/build-pages/page-builders/page-writer.js +++ b/lib/build-pages/page-builders/page-writer.js @@ -2,7 +2,7 @@ * @import { PageInfo } from '../../identify-pages.js' * @import { PageData as PageDataClass } from '../page-data.js' * @import { DomstackManifestRecord } from '../../domstack-manifest/index.js' - * @import { AdditionalOutputsFunction, AdditionalOutputProvenance } from '../additional-outputs.js' + * @import { AdditionalOutputsFunction } from '../additional-outputs.js' */ import { join } from 'path' @@ -99,19 +99,24 @@ import { writeAdditionalOutputs } from './additional-output-writer.js' * @param {object} params * @param {string} params.dest - The dest folder. * @param {PageData} params.page - The PageInfo object of the current page - * @param {(outputRelname: string, provenance?: AdditionalOutputProvenance) => void} [params.claimAdditionalOutput] + * @returns {Promise<{ pageFilePath: string, outputs: DomstackManifestRecord[] }>} */ export async function pageWriter ({ dest, page, - claimAdditionalOutput, }) { if (!page.pageInfo) throw new Error('Uninitialzied page detected') const pageDir = join(dest, page.pageInfo.path) const pageFilePath = join(pageDir, page.pageInfo.outputName) const formattedPageOutput = await page.renderFullPage() + const additionalOutputs = await writeAdditionalOutputs({ + dest, + pageFilePath, + pageInfo: page.pageInfo, + additionalOutputs: await page.collectAdditionalOutputs(), + }) const vars = page.vars const manifestRole = extractManifestRole(vars) await mkdir(pageDir, { recursive: true }) @@ -168,13 +173,7 @@ export async function pageWriter ({ } } - outputs.push(...await writeAdditionalOutputs({ - dest, - pageFilePath, - pageInfo: page.pageInfo, - additionalOutputs: await page.collectAdditionalOutputs(), - claimOutput: claimAdditionalOutput, - })) + outputs.push(...additionalOutputs) return { pageFilePath, outputs } } diff --git a/lib/build-pages/page-builders/template-builder.js b/lib/build-pages/page-builders/template-builder.js index dc24fbbb..f5d91efb 100644 --- a/lib/build-pages/page-builders/template-builder.js +++ b/lib/build-pages/page-builders/template-builder.js @@ -4,7 +4,6 @@ * @import { WatchDependencyTracker } from '../watch-dependencies.js' */ -import { OutputRegistry } from '../../output-registry.js' import { dirname, join, relative, resolve } from 'node:path' import { writeFile, mkdir } from 'fs/promises' import { createDomstackManifestRecord } from '../../domstack-manifest/index.js' @@ -92,7 +91,6 @@ function isTemplateOutputOverrideArray (value) { * @param {Record} params.globalData - Values returned by global.data. * @param {TemplateInfo} params.template - The TemplateInfo of the template. * @param {WatchDependencyTracker} params.watchDependencyTracker - Declarative watch dependency state. - * @param {OutputRegistry} [params.outputRegistry] - Shared output ownership for this page phase. * @returns {Promise<{ report: TemplateReport, outputs: DomstackManifestRecord[] }>} */ export async function templateBuilder ({ @@ -101,7 +99,6 @@ export async function templateBuilder ({ globalData, template, watchDependencyTracker, - outputRegistry = new OutputRegistry(), }) { const importResults = await import(template.templateFile.filepath) if (!importResults.default || typeof importResults.default !== 'function') { @@ -147,7 +144,6 @@ export async function templateBuilder ({ content: templateResults, template, outputRecords, - outputRegistry, }) } else if (isTemplateOutputOverrideArray(templateResults)) { type = 'array' @@ -159,7 +155,6 @@ export async function templateBuilder ({ content: templateResult.content, template, outputRecords, - outputRegistry, }) } } else if (isTemplateOutputOverride(templateResults)) { @@ -171,7 +166,6 @@ export async function templateBuilder ({ content: templateResults.content, template, outputRecords, - outputRegistry, }) } else if (isAsyncIterable(templateResults)) { type = 'async-iterator' @@ -184,7 +178,6 @@ export async function templateBuilder ({ content: templateResult.content, template, outputRecords, - outputRegistry, }) } else { throw new Error(`Template file returned unknown return type: ${typeof templateResult}`) @@ -212,7 +205,6 @@ export async function templateBuilder ({ * @param {string} params.content * @param {TemplateInfo} params.template * @param {DomstackManifestRecord[]} params.outputRecords - * @param {OutputRegistry} params.outputRegistry */ async function writeTemplateOutput ({ dest, @@ -221,20 +213,14 @@ async function writeTemplateOutput ({ content, template, outputRecords, - outputRegistry, }) { - const filepath = resolve(fileDir, outputName.replaceAll('\\', '/')) + const filepath = resolve(fileDir, outputName) assertInsideDest(dest, filepath, `Template output escapes dest: ${filepath}`) - const outputRelname = toPosix(relative(dest, filepath)) - outputRegistry.claim(outputRelname, { - id: `template:${template.templateFile.filepath}`, - type: 'template', - path: template.templateFile.relname, - }) const filePathDirname = dirname(filepath) await mkdir(filePathDirname, { recursive: true }) await writeFile(filepath, content) + const outputRelname = toPosix(relative(dest, filepath)) outputRecords.push(createDomstackManifestRecord({ dest, filepath, diff --git a/lib/build-static/index.js b/lib/build-static/index.js index f9a17886..3bbdb0e9 100644 --- a/lib/build-static/index.js +++ b/lib/build-static/index.js @@ -1,10 +1,9 @@ /** - * @import { BuildStep, BuildStepResult, DomStackOpts } from '../builder.js' + * @import { BuildStep, BuildStepResult } from '../builder.js' */ import { processedExtensions } from '../file-conventions.js' -/** @import { copy } from 'cpx2' */ -import { stagedCopy } from '../helpers/staged-copy.js' -import { OutputRegistry } from '../output-registry.js' +import { copy } from 'cpx2' +import { createCopiedDomstackManifestRecords } from '../helpers/cpx2-report.js' /** * @typedef {Awaited> | Record} StaticBuilderReport @@ -30,13 +29,9 @@ export function getCopyGlob (src) { /** * run CPX2 on src folder * - * @param {string} src - * @param {string} dest - * @param {unknown} _siteData - * @param {DomStackOpts | null} [opts] - * @param {OutputRegistry} [registry] + * @type {StaticBuildStep} */ -export async function buildStatic (src, dest, _siteData, opts, registry = new OutputRegistry()) { +export async function buildStatic (src, dest, _siteData, opts) { /** @type {StaticBuildStepResult} */ const results = { type: 'static', @@ -47,10 +42,14 @@ export async function buildStatic (src, dest, _siteData, opts, registry = new Ou } try { - if (opts?.static === false) return results - const copied = await stagedCopy(getCopyGlob(src), src, dest, 'static', registry, opts?.ignore) - results.report = copied.report - results.outputs = copied.outputs + const report = await copy(getCopyGlob(src), dest, ...(opts?.ignore ? [{ ignore: opts.ignore }] : [])) + results.report = report + results.outputs = createCopiedDomstackManifestRecords({ + src, + dest, + report, + kind: 'static', + }) } catch (err) { const buildError = new Error('Error copying static files', { cause: err }) results.errors.push(buildError) diff --git a/lib/builder.js b/lib/builder.js index baa30887..57dea202 100644 --- a/lib/builder.js +++ b/lib/builder.js @@ -6,7 +6,6 @@ * @import { PageBuildStepResult } from './build-pages/index.js' * @import { StaticBuildStepResult } from './build-static/index.js' * @import { CopyBuildStepResult } from './build-copy/index.js' - * @import { OutputClaim } from './output-registry.js' * @import { DomstackManifest, DomstackManifestConfig, DomstackManifestRecord } from './domstack-manifest/index.js' */ @@ -15,17 +14,10 @@ import { identifyPages } from './identify-pages.js' import { buildStatic } from './build-static/index.js' import { buildCopy } from './build-copy/index.js' import { buildEsbuild, buildServiceWorkerEsbuild } from './build-esbuild/index.js' -import { cp, mkdir, mkdtemp, realpath, rm } from 'node:fs/promises' -import { join, relative, resolve } from 'node:path' -import { toPosix } from './helpers/path.js' -import { prepareAdditionalOutputPromotion } from './helpers/additional-output-promotion.js' import { DomStackAggregateError } from './helpers/domstack-aggregate-error.js' - -import { OutputRegistry, isCaseInsensitiveDest } from './output-registry.js' - -import { remapCopyReport } from './helpers/staged-copy.js' +import { ensureDest } from './helpers/ensure-dest.js' +import { outputWarnings } from './helpers/output-warnings.js' import { - DEFAULT_DOMSTACK_MANIFEST_FILENAME, isDomstackManifestEnabled, reconcileDomstackManifest, resolveDomstackManifestOptions, @@ -99,7 +91,6 @@ import { * @property {PageBuildStepResult} [pageBuildResults] * @property {DomstackManifest} [domstackManifest] * @property {BuildStepWarnings} warnings - * @property {OutputClaim[]} [outputClaims] - Internal full-watch ownership snapshot. */ /** @@ -111,7 +102,6 @@ import { * @param {string} src - The source directory from which the site should be built. * @param {string} dest - The destination directory where the built site should be placed. * @param {DomStackOpts} opts - Options for the build process. - * @param {{ watch?: boolean, caseInsensitive?: boolean, promoteOutputs?: (claims: OutputClaim[], write: () => Promise, preflight: (removablePaths: string[]) => Promise) => Promise }} [internal] * @returns {Promise} * * @example @@ -127,61 +117,11 @@ import { * console.error(error) * } */ -export async function builder (src, dest, opts, internal = {}) { - if (!internal.watch) { - const results = await buildInto(src, dest, opts, dest, false, internal.caseInsensitive) - delete results.outputClaims - return results - } - await mkdir(dest, { recursive: true }) - const stageDest = await mkdtemp(join(resolve(dest), '.domstack-stage-')) - try { - const results = await buildInto(src, stageDest, { ...opts, ignore: [...(opts.ignore ?? []), '.domstack-stage-*'] }, dest, internal.watch ?? false, internal.caseInsensitive) - let unchanged = new Set() - const preflight = async (/** @type {string[]} */ removablePaths) => { - unchanged = await prepareAdditionalOutputPromotion(dest, results.pageBuildResults?.outputs ?? [], removablePaths) - } - - const write = async () => { - await mkdir(dest, { recursive: true }) - await cp(stageDest, await realpath(dest), { - recursive: true, - force: true, - filter: source => !unchanged.has(toPosix(relative(stageDest, source))), - }) - } - if (internal.promoteOutputs) await internal.promoteOutputs(results.outputClaims ?? [], write, preflight) - else { - await preflight([]) - await write() - } - remapBuildResults(results, stageDest, dest) - delete results.outputClaims - return results - } catch (error) { - if (error instanceof DomStackAggregateError && error.results?.esbuildResults) remapBuildResults(error.results, stageDest, dest) - throw error - } finally { - await rm(stageDest, { recursive: true, force: true }) - } -} - -/** - * Build into the requested destination (or the caller's full-watch stage). - * - * @param {string} src - * @param {string} dest - * @param {DomStackOpts} opts - * @param {string} publicDest - * @param {boolean} watch - * @param {boolean} [casePolicy] - * @returns {Promise} - */ -async function buildInto (src, dest, opts, publicDest, watch, casePolicy) { +export async function builder (src, dest, opts) { const errors = [] /** @type {BuildStepErrors} */ const warnings = [] /** @type {BuildStepWarnings} */ - const siteData = await identifyPages(src, { ...(opts.ignore ? { ignore: opts.ignore } : {}), ...(opts.buildDrafts !== undefined ? { buildDrafts: opts.buildDrafts } : {}) }) /** @type {SiteData} */ + const siteData = await identifyPages(src, opts) /** @type {SiteData} */ errors.push(...siteData.errors) warnings.push(...siteData.warnings) @@ -191,7 +131,7 @@ async function buildInto (src, dest, opts, publicDest, watch, casePolicy) { throw pageWalkErrors } - await mkdir(dest, { recursive: true }) + await ensureDest(dest, siteData) const domstackManifestSettingsPath = siteData?.domstackManifestSettings?.filepath const domstackManifestEnabled = isDomstackManifestEnabled({ @@ -203,18 +143,16 @@ async function buildInto (src, dest, opts, publicDest, watch, casePolicy) { opts, }) - const caseInsensitive = casePolicy ?? await isCaseInsensitiveDest(publicDest) - const outputRegistry = new OutputRegistry([], { caseInsensitive }) const [ esbuildResults, staticResults, copyResults, ] = await Promise.all([ - buildEsbuild(src, dest, siteData, opts, outputRegistry, watch, publicDest), - opts.static !== false - ? buildStatic(src, dest, siteData, opts, outputRegistry) + buildEsbuild(src, dest, siteData, opts), + opts.static + ? buildStatic(src, dest, siteData, opts) : Promise.resolve(null), - buildCopy(src, dest, siteData, opts, outputRegistry), + buildCopy(src, dest, siteData, opts), ]) /** @type {Results} */ @@ -242,12 +180,7 @@ async function buildInto (src, dest, opts, publicDest, watch, casePolicy) { throw preBuildError } - const pageBuildResults = await buildPages(src, dest, siteData, { - ...opts, - previousOutputClaims: outputRegistry.snapshot(), - caseInsensitive, - trackWatchDependencies: watch, - }) + const pageBuildResults = await buildPages(src, dest, siteData, opts) errors.push(...pageBuildResults.errors) warnings.push(...pageBuildResults.warnings) @@ -258,11 +191,6 @@ async function buildInto (src, dest, opts, publicDest, watch, casePolicy) { throw buildError } - outputRegistry.releaseOwnerIds(pageBuildResults.report.replacedOwnerIds ?? []) - for (const claim of pageBuildResults.report.newClaims ?? []) outputRegistry.claim(claim.outputRelname, claim.owner) - delete pageBuildResults.report.newClaims - delete pageBuildResults.report.replacedOwnerIds - const baseOutputRecords = collectOutputRecords( esbuildResults, staticResults, @@ -270,6 +198,8 @@ async function buildInto (src, dest, opts, publicDest, watch, casePolicy) { pageBuildResults ) + warnings.push(...outputWarnings(baseOutputRecords)) + const domstackManifestReconciliation = domstackManifestEnabled ? await reconcileDomstackManifest({ dest, @@ -284,14 +214,7 @@ async function buildInto (src, dest, opts, publicDest, watch, casePolicy) { const domstackManifest = domstackManifestReconciliation?.manifest const domstackManifestBuiltHookResult = domstackManifest - ? await runDomstackManifestBuiltHooks(dest, domstackManifest, domstackManifestOptions, { - publicDest, - claimOutput: (outputRelname, hookIndex) => outputRegistry.claim(outputRelname, { - id: `manifest-hook:${hookIndex}`, - type: 'manifest hook', - path: `manifestBuilt hook #${hookIndex + 1}`, - }), - }) + ? await runDomstackManifestBuiltHooks(dest, domstackManifest, domstackManifestOptions) : { serviceWorkerDefines: {} } const serviceWorkerBuildDefines = { @@ -304,8 +227,7 @@ async function buildInto (src, dest, opts, publicDest, watch, casePolicy) { dest, siteData, esbuildResults.report.buildOpts, - serviceWorkerBuildDefines, - outputRegistry + serviceWorkerBuildDefines ) errors.push(...serviceWorkerEsbuildResults.errors) @@ -319,48 +241,16 @@ async function buildInto (src, dest, opts, publicDest, watch, casePolicy) { if (domstackManifest) results.domstackManifest = domstackManifest if (domstackManifest && shouldWriteDomstackManifest(opts)) { - outputRegistry.claim(DEFAULT_DOMSTACK_MANIFEST_FILENAME, { - id: 'domstack-manifest', - type: 'metadata', - path: 'generated domstack manifest', - }) await writeDomstackManifest(dest, domstackManifest) } - results.outputClaims = outputRegistry.snapshot() return results } /** - * @param {...(BuildOutputStepResult | null | undefined)} results + * @param {...(BuildOutputStepResult | null)} results * @returns {DomstackManifestRecord[]} */ function collectOutputRecords (...results) { return results.flatMap(result => result?.outputs ?? []) } - -/** - * Staging is internal; public reports continue to describe the requested dest. - * - * @param {Results} results - * @param {string} stageDest - * @param {string} dest - */ -function remapBuildResults (results, stageDest, dest) { - const steps = [ - results.esbuildResults, - results.staticResults, - results.copyResults, - results.pageBuildResults, - ] - for (const record of collectOutputRecords(...steps)) { - record.filepath = resolve(dest, record.outputRelname) - } - - for (const page of results.pageBuildResults?.report.pages ?? []) { - page.pageFilePath = resolve(dest, page.pageFilePath.slice(resolve(stageDest).length + 1)) - } - - if (results.staticResults) remapCopyReport(results.staticResults.report, dest) - for (const report of Object.values(results.copyResults?.report ?? {})) remapCopyReport(report, dest) -} diff --git a/lib/domstack-manifest/hooks.js b/lib/domstack-manifest/hooks.js index 2faa7ffc..009e14dd 100644 --- a/lib/domstack-manifest/hooks.js +++ b/lib/domstack-manifest/hooks.js @@ -23,7 +23,7 @@ export async function writeDomstackManifest (dest, domstackManifest) { * @param {string | Uint8Array} contents */ async function writeGeneratedManifestFile (dest, outputRelname, contents) { - const filepath = resolve(dest, outputRelname.replaceAll('\\', '/')) + const filepath = resolve(dest, outputRelname) assertInsideDest(dest, filepath) await mkdir(dirname(filepath), { recursive: true }) await writeFile(filepath, contents) @@ -35,16 +35,15 @@ async function writeGeneratedManifestFile (dest, outputRelname, contents) { * @param {string} dest * @param {DomstackManifest} manifest * @param {DomstackManifestOptions} options - * @param {{ claimOutput?: (outputRelname: string, hookIndex: number) => void, publicDest?: string }} [buildOptions] * @returns {Promise} */ -export async function runDomstackManifestBuiltHooks (dest, manifest, options, buildOptions = {}) { +export async function runDomstackManifestBuiltHooks (dest, manifest, options) { const serviceWorkerDefines = /** @type {Record} */ ({}) const hooks = options.hooks?.manifestBuilt ?? [] - for (const [hookIndex, hook] of hooks.entries()) { + for (const hook of hooks) { await hook({ - dest: buildOptions.publicDest ?? dest, + dest, manifest, defineServiceWorkerConstant: (identifier, value) => { const serializedValue = JSON.stringify(value) @@ -53,10 +52,7 @@ export async function runDomstackManifestBuiltHooks (dest, manifest, options, bu } serviceWorkerDefines[identifier] = serializedValue }, - writeFile: (outputRelname, contents) => { - buildOptions.claimOutput?.(outputRelname, hookIndex) - return writeGeneratedManifestFile(dest, outputRelname, contents) - }, + writeFile: (outputRelname, contents) => writeGeneratedManifestFile(dest, outputRelname, contents), }) } diff --git a/lib/helpers/additional-output-promotion.js b/lib/helpers/additional-output-promotion.js deleted file mode 100644 index a49187da..00000000 --- a/lib/helpers/additional-output-promotion.js +++ /dev/null @@ -1,68 +0,0 @@ -/** - * @import { DomstackManifestRecord } from '../domstack-manifest/index.js' - */ -import { lstat, readFile, realpath } from 'node:fs/promises' -import { isAbsolute, relative, resolve, sep } from 'node:path' - -/** - * Validate every stale removal and sidecar destination before cleanup or writing. - * The root may resolve through symlinks, but no component below it may do so. - * Only intermediate regular files in this transaction's removal set may block a - * new destination. This preflight does not guard against external filesystem races. - * - * @param {string} dest - * @param {DomstackManifestRecord[]} outputs - * @param {Iterable} [removablePaths] Destination-relative stale claims. - * @returns {Promise>} Unchanged page-additional outputRelnames only. - */ -export async function prepareAdditionalOutputPromotion (dest, outputs, removablePaths = []) { - const sidecars = outputs.filter(output => output.kind === 'page-additional') - const removals = [...removablePaths] - const unchanged = new Set() - if (sidecars.length === 0 && removals.length === 0) return unchanged - let root = resolve(dest) - try { root = await realpath(root) } catch (error) { - if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error - } - const removable = new Set(removals.map(name => resolve(root, name))) - /** @param {string} name @param {boolean} allowRemoval */ - async function inspect (name, allowRemoval) { - const target = resolve(root, name) - const rel = relative(root, target) - if (!rel || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { - throw new Error(`Additional output path escapes dest: ${name}`) - } - let current = root - const components = rel.split(sep) - for (const [index, component] of components.entries()) { - current = resolve(current, component) - let info - try { info = await lstat(current) } catch (error) { - if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error - return { target, comparable: false } - } - if (info.isSymbolicLink()) throw new Error(`Additional output path contains a symlink: ${name}`) - if (index < components.length - 1 && !info.isDirectory()) { - if (allowRemoval && info.isFile() && removable.has(current)) return { target, comparable: false } - throw Object.assign(new Error(`Output path has a non-directory ancestor: ${name}`), { code: 'ENOTDIR' }) - } - if (index === components.length - 1) return { target, comparable: !info.isDirectory() && !removable.has(target) } - } - return { target, comparable: false } - } - // Validate the entire removal set even when no sidecars are being emitted. - for (const name of removals) await inspect(name, false) - const targets = [] - for (const output of sidecars) targets.push({ output, ...await inspect(output.outputRelname, true) }) - for (const { output, target, comparable } of targets) { - // Missing staged sources are errors, not evidence that a destination changed. - const source = await readFile(output.filepath) - if (!comparable) continue - try { - if (source.equals(await readFile(target))) unchanged.add(output.outputRelname) - } catch (error) { - if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error - } - } - return unchanged -} diff --git a/lib/helpers/additional-output-promotion.test.js b/lib/helpers/additional-output-promotion.test.js deleted file mode 100644 index 747058c4..00000000 --- a/lib/helpers/additional-output-promotion.test.js +++ /dev/null @@ -1,83 +0,0 @@ -/** @import { TestContext } from 'node:test' */ -import { test } from 'node:test' -import assert from 'node:assert/strict' -import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { prepareAdditionalOutputPromotion } from './additional-output-promotion.js' -import { assertInsideDest } from './path.js' - -/** @param {TestContext} t */ -async function fixture (t) { - const root = await mkdtemp(join(tmpdir(), 'additional-promotion-')) - t.after(() => rm(root, { recursive: true, force: true })) - const dest = join(root, 'dest') - await mkdir(dest) - const filepath = join(root, 'staged') - await writeFile(filepath, Buffer.from([0, 255, 1])) - const output = { kind: /** @type {const} */ ('page-additional'), outputRelname: 'feed.bin', filepath } - return { root, dest, filepath, output } -} - -test('validates stale paths without sidecars and permits only owned regular blockers', async t => { - const { root, dest, output } = await fixture(t) - await writeFile(join(dest, 'raw'), 'old') - const nested = { ...output, outputRelname: 'raw/article.md' } - await assert.rejects(prepareAdditionalOutputPromotion(dest, [nested]), { code: 'ENOTDIR' }) - await assert.rejects(prepareAdditionalOutputPromotion(dest, [nested], ['other']), { code: 'ENOTDIR' }) - assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [nested], ['raw']), new Set()) - await symlink(root, join(dest, 'unsafe')) - await assert.rejects(prepareAdditionalOutputPromotion(dest, [], ['raw', 'unsafe/stale']), /symlink/) - await assert.rejects(prepareAdditionalOutputPromotion(dest, [], ['../escape']), /escapes dest/) - await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: 'unsafe/new' }], ['unsafe']), /symlink/) - const alias = join(root, 'alias') - await symlink(dest, alias) - assert.deepEqual(await prepareAdditionalOutputPromotion(alias, [], ['raw']), new Set()) - assert.doesNotThrow(() => assertInsideDest(dest, join(dest, '..hidden'))) - assert.throws(() => assertInsideDest(dest, join(dest, '../hidden')), /escapes dest/) -}) - -test('compares exact bytes only for sidecars, allowing missing destinations', async t => { - const { dest, output } = await fixture(t) - assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [output]), new Set()) - await writeFile(join(dest, 'feed.bin'), Buffer.from([0, 255, 1])) - assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [output]), new Set(['feed.bin'])) - assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [{ ...output, kind: 'page' }]), new Set()) - await writeFile(join(dest, 'feed.bin'), Buffer.from([0, 254, 1])) - assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [output]), new Set()) -}) - -test('allows realpath destination root but rejects leaf, ancestor and dangling symlinks', async t => { - const { root, dest, output } = await fixture(t) - const alias = join(root, 'alias') - await symlink(dest, alias) - await writeFile(join(dest, 'feed.bin'), Buffer.from([0, 255, 1])) - assert.deepEqual(await prepareAdditionalOutputPromotion(alias, [output]), new Set(['feed.bin'])) - for (const [name, target] of /** @type {[string, string][]} */ ([['leaf', output.filepath], ['inside', dest], ['outside', root], ['dangling', join(root, 'absent')]])) { - await symlink(target, join(dest, name)) - await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: name }]), /symlink/) - await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: `${name}/child` }]), /symlink/) - } -}) - -test('validates all paths before comparison and rejects escapes', async t => { - const { root, dest, output } = await fixture(t) - await symlink(root, join(dest, 'unsafe')) - await assert.rejects(prepareAdditionalOutputPromotion(dest, [ - { ...output, filepath: join(root, 'missing-source') }, - { ...output, outputRelname: 'unsafe/file' }, - ]), /symlink/) - for (const outputRelname of ['../escape', dest, '.']) { - await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname }]), /escapes dest/) - } -}) - -test('directory targets remain changed for owned-descendant cleanup; other errors propagate', async t => { - const { root, dest, output } = await fixture(t) - await mkdir(join(dest, 'feed.bin')) - assert.deepEqual(await prepareAdditionalOutputPromotion(dest, [output]), new Set()) - await writeFile(join(dest, 'parent'), '') - await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: 'parent/file' }]), { code: 'ENOTDIR' }) - await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: 'missing', filepath: join(root, 'absent') }]), { code: 'ENOENT' }) - await assert.rejects(prepareAdditionalOutputPromotion(dest, [{ ...output, outputRelname: 'missing', filepath: dest }]), { code: 'EISDIR' }) -}) diff --git a/lib/helpers/domstack-error.js b/lib/helpers/domstack-error.js index 6ce7877e..9a03915e 100644 --- a/lib/helpers/domstack-error.js +++ b/lib/helpers/domstack-error.js @@ -30,7 +30,7 @@ export class DomStackDataError extends Error { /** * @typedef DomStackOutputConflictErrorClaim - * @property {string} type - The kind of output producer. + * @property {'page'} type - The kind of output producer. * @property {string} path - Human-readable source or output path for the producer. */ diff --git a/lib/helpers/domstack-warning.js b/lib/helpers/domstack-warning.js index 15d5d1a1..e3641de8 100644 --- a/lib/helpers/domstack-warning.js +++ b/lib/helpers/domstack-warning.js @@ -12,6 +12,7 @@ * 'DOM_STACK_WARNING_DUPLICATE_MARKDOWN_IT_SETTINGS' | * 'DOM_STACK_WARNING_DUPLICATE_DOMSTACK_MANIFEST_SETTINGS' | * 'DOM_STACK_WARNING_CONFLICTING_MANIFEST_OUTPUT' | + * 'DOM_STACK_WARNING_DUPLICATE_OUTPUT' | * 'DOM_STACK_WARNING_DUPLICATE_GLOBAL_VARS' | * 'DOM_STACK_WARNING_DUPLICATE_GLOBAL_DATA' | * 'DOM_STACK_WARNING_PAGE_MD_SHADOWS_README' diff --git a/lib/helpers/output-warnings.js b/lib/helpers/output-warnings.js new file mode 100644 index 00000000..17b11a5d --- /dev/null +++ b/lib/helpers/output-warnings.js @@ -0,0 +1,27 @@ +/** + * @import { DomstackManifestRecord } from '../domstack-manifest/index.js' + * @import { DomStackWarning } from './domstack-warning.js' + */ +import { resolve } from 'node:path' + +/** + * Report duplicate destinations observed in build reports. This does not reserve + * paths or control writes; watch phases can only report the outputs they see. + * @param {DomstackManifestRecord[]} outputs + * @returns {DomStackWarning[]} + */ +export function outputWarnings (outputs) { + const destinations = new Map() + const warnings = /** @type {DomStackWarning[]} */ ([]) + for (const output of outputs) { + const path = resolve(output.filepath) + const previous = destinations.get(path) + if (previous) { + warnings.push({ + code: 'DOM_STACK_WARNING_DUPLICATE_OUTPUT', + message: `Duplicate output "${output.outputRelname}": ${previous.sourceRelname ?? previous.kind} (${previous.kind}) and ${output.sourceRelname ?? output.kind} (${output.kind}) write the same destination; output may be overwritten.`, + }) + } else destinations.set(path, output) + } + return warnings +} diff --git a/lib/helpers/staged-copy.js b/lib/helpers/staged-copy.js deleted file mode 100644 index e66c3a7a..00000000 --- a/lib/helpers/staged-copy.js +++ /dev/null @@ -1,61 +0,0 @@ -/** - * @import { OutputRegistry } from '../output-registry.js' - * @import { NormalizedOptions } from 'cpx2' - */ -import normalizeOptions from 'cpx2/lib/utils/normalize-options.js' -import applyAction from 'cpx2/lib/utils/apply-action.js' -import copy from 'cpx2/lib/utils/copy-file.js' -import { copyFile, mkdir, mkdtemp, rm } from 'node:fs/promises' -import { dirname, join, resolve } from 'node:path' -import { createCopiedDomstackManifestRecords } from './cpx2-report.js' - -/** - * Keep cpx's matching and mapping semantics, but inventory before writing so - * case aliases and file/directory aliases cannot collapse in the staging tree. - * These cpx internals are intentionally confined to this adapter. - * @param {string} source - * @param {string} src - * @param {string} dest - * @param {'static' | 'copy'} kind - * @param {OutputRegistry} registry - * @param {string[]} [ignore] - * @param {string} [ownerPrefix] - Identity of this configured copy-root occurrence. - */ -export async function stagedCopy (source, src, dest, kind, registry, ignore = [], ownerPrefix = '') { - const options = normalizeOptions(source, dest, { ignore }) - const sources = /** @type {string[]} */ (await applyAction(options.source, options, path => path)) - const report = { - cleaned: [], - copied: sources.map(source => ({ source, output: options.toDestination(source), skipped: false })), - options, - } - const outputs = createCopiedDomstackManifestRecords({ src, dest, report, kind }) - registry.claimRecords(outputs, ownerPrefix) - await mkdir(dest, { recursive: true }) - const stage = await mkdtemp(join(dest, '.domstack-copy-')) - try { - for (const [index, entry] of report.copied.entries()) { - const stagedPath = join(stage, String(index)) - await copy(entry.source, stagedPath, options) - } - for (const [index, output] of outputs.entries()) { - const target = resolve(dest, output.outputRelname) - await mkdir(dirname(target), { recursive: true }) - await copyFile(join(stage, String(index)), target) - } - return { report, outputs } - } finally { - await rm(stage, { recursive: true, force: true }) - } -} - -/** Rebuild the mapper as well as the known report paths after full-watch staging. - * @param {object} value - * @param {string} dest - */ -export function remapCopyReport (value, dest) { - if (!('options' in value)) return - const report = /** @type {{ options: NormalizedOptions, copied: { source: string, output: string }[] }} */ (value) - report.options = normalizeOptions(report.options.source, dest, { ignore: report.options.ignore ?? [] }) - for (const entry of report.copied) entry.output = report.options.toDestination(entry.source) -} diff --git a/lib/output-registry.js b/lib/output-registry.js deleted file mode 100644 index ba92f84c..00000000 --- a/lib/output-registry.js +++ /dev/null @@ -1,174 +0,0 @@ -/** - * @import { DomstackManifestRecord } from './domstack-manifest/index.js' - * @import { DomStackOutputConflictErrorClaim } from './helpers/domstack-error.js' - */ - -import { join, posix } from 'node:path' -import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' -import { DomStackOutputConflictError } from './helpers/domstack-error.js' -import { toPosix } from './helpers/path.js' - -/** - * @typedef OutputOwner - * @property {string} id - Stable identity used to replace an owner's outputs in watch mode. - * @property {string} type - Human-readable producer type. - * @property {string} path - Human-readable source or build-step path. - */ - -/** - * @typedef OutputClaim - * @property {string} outputRelname - * @property {OutputOwner} owner - */ - -/** - * Track destination-relative output ownership for one successful build state. - */ -export class OutputRegistry { - /** @type {Map} */ - #claims = new Map() - #caseInsensitive - - /** - * @param {OutputClaim[]} [previousClaims] - * @param {{ replaceOwnerIds?: Iterable, caseInsensitive?: boolean }} [options] - */ - constructor (previousClaims = [], options = {}) { - this.#caseInsensitive = options.caseInsensitive ?? false - const replaceOwnerIds = new Set(options.replaceOwnerIds ?? []) - for (const claim of previousClaims) { - if (replaceOwnerIds.has(claim.owner.id)) continue - this.#claims.set(this.#key(claim.outputRelname), structuredClone(claim)) - } - } - - /** - * @param {string} outputRelname - * @param {OutputOwner} owner - */ - claim (outputRelname, owner) { - const normalized = normalizeOutputRelname(outputRelname) - const conflict = this.#findConflict(normalized) - if (conflict) throw createConflictError(normalized, conflict.owner, owner) - this.#claims.set(this.#key(normalized), { outputRelname: normalized, owner: { ...owner } }) - } - - /** @param {string} path */ - #key (path) { - const normalized = normalizeOutputRelname(path) - return this.#caseInsensitive ? normalized.normalize('NFC').toLowerCase() : normalized - } - - /** @param {Iterable} ownerIds */ - releaseOwnerIds (ownerIds) { - const released = new Set(ownerIds) - for (const [outputRelname, claim] of this.#claims) { - if (released.has(claim.owner.id)) this.#claims.delete(outputRelname) - } - } - - /** - * @param {DomstackManifestRecord[]} records - * @param {string} [ownerPrefix] - */ - claimRecords (records, ownerPrefix = '') { - const seen = new Set() - for (const record of records) { - // Reporting the same record twice is not a second write. Dynamic writers - // use claim() directly, where repeated outputs always conflict. - const key = JSON.stringify(record) - if (seen.has(key)) continue - seen.add(key) - const owner = outputOwnerForRecord(record) - this.claim(record.outputRelname, { ...owner, id: `${ownerPrefix}${owner.id}` }) - } - } - - /** @returns {OutputClaim[]} */ - snapshot () { - return Array.from(this.#claims.values(), claim => structuredClone(claim)) - } - - /** - * @param {string} outputRelname - * @returns {OutputClaim | undefined} - */ - #findConflict (outputRelname) { - outputRelname = this.#key(outputRelname) - const exact = this.#claims.get(outputRelname) - if (exact) return exact - - for (const [key, claim] of this.#claims) { - if (outputRelname.startsWith(`${key}/`) || key.startsWith(`${outputRelname}/`)) { - return claim - } - } - } -} - -/** Probe the destination volume rather than assuming case behavior from the OS. - * @param {string} dest - */ -export async function isCaseInsensitiveDest (dest) { - await mkdir(dest, { recursive: true }) - const probe = await mkdtemp(join(dest, '.domstack-case-')) - try { - await writeFile(join(probe, 'probe'), '') - try { - await access(join(probe, 'PROBE')) - return true - } catch (error) { - if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error - return false - } - } finally { - await rm(probe, { recursive: true, force: true }) - } -} - -/** - * @param {DomstackManifestRecord} record - * @returns {OutputOwner} - */ -export function outputOwnerForRecord (record) { - const source = record.sourceRelname ?? record.entryPoint - const path = source ?? describeBuildStep(record.kind) - return { - id: `${record.kind}:${source ?? record.outputRelname}`, - type: record.kind, - path, - } -} - -/** - * @param {string} outputRelname - * @returns {string} - */ -export function normalizeOutputRelname (outputRelname) { - const normalized = posix.normalize(toPosix(outputRelname).replaceAll('\\', '/')) - if (!normalized || normalized === '.' || normalized === '..' || normalized.startsWith('../') || normalized.startsWith('/') || /^[a-z]:/i.test(normalized)) { - throw new Error(`Output path must be destination-relative: ${outputRelname}`) - } - return normalized -} - -/** - * @param {string} outputPath - * @param {OutputOwner} a - * @param {OutputOwner} b - */ -function createConflictError (outputPath, a, b) { - const first = /** @type {DomStackOutputConflictErrorClaim} */ ({ type: a.type, path: a.path }) - const second = /** @type {DomStackOutputConflictErrorClaim} */ ({ type: b.type, path: b.path }) - return new DomStackOutputConflictError( - `Output path conflict: ${outputPath} is produced by both ${first.path} and ${second.path}.`, - { outputPath, a: first, b: second } - ) -} - -/** @param {DomstackManifestRecord['kind']} kind */ -function describeBuildStep (kind) { - if (kind === 'metadata') return 'domstack esbuild metadata' - if (kind === 'service-worker') return 'service worker build' - return `${kind} build step` -} diff --git a/test-cases/output-conflicts/index.test.js b/test-cases/output-conflicts/index.test.js deleted file mode 100644 index 668f79c0..00000000 --- a/test-cases/output-conflicts/index.test.js +++ /dev/null @@ -1,614 +0,0 @@ -/** - * @import { TestContext } from 'node:test' - * @import { DomStackOpts } from '../../lib/builder.js' - */ -import { test } from 'node:test' -import assert from 'node:assert/strict' -import { mkdir, mkdtemp, readFile, readdir, rename, rm, stat, symlink, writeFile } from 'node:fs/promises' -import { dirname, join, resolve } from 'node:path' -import { createHash } from 'node:crypto' -import { builder } from '../../lib/builder.js' -import { stagedCopy } from '../../lib/helpers/staged-copy.js' -import { buildPages } from '../../lib/build-pages/index.js' -import { identifyPages } from '../../lib/identify-pages.js' -import { DomStackAggregateError } from '../../lib/helpers/domstack-aggregate-error.js' -import pino from 'pino' -import { DomStack } from '../../index.js' -import { OutputRegistry, isCaseInsensitiveDest } from '../../lib/output-registry.js' - -/** @param {string} root @param {Record} files */ -async function writeFiles (root, files) { - for (const [name, content] of Object.entries(files)) { - const path = join(root, name) - await mkdir(dirname(path), { recursive: true }) - await writeFile(path, content) - } -} - -/** @param {TestContext} t @param {Record} files @param {DomStackOpts} [opts] */ -async function setup (t, files, opts = {}) { - const tmp = await mkdtemp(join(import.meta.dirname, 'tmp-')) - const src = join(tmp, 'src') - const dest = join(tmp, 'public') - await writeFiles(src, { - 'global.vars.js': "export default { layout: 'root' }", - 'root.layout.js': 'export default ({ children }) => children', - ...files, - }) - const logs = /** @type {string[]} */ ([]) - const site = new DomStack(src, dest, { ...opts, logger: pino({ level: 'debug' }, { write: line => logs.push(line) }) }) - const sites = [site] - t.after(async () => { - for (const site of sites) if (site.watching) await site.stopWatching() - await rm(tmp, { recursive: true, force: true }) - }) - return { src, dest, tmp, site, sites, logs } -} - -/** @param {unknown} error @returns {Error & { code?: string, contextData?: { outputPath?: string } } | undefined} */ -function conflict (error) { - if (!(error instanceof Error)) return - const value = /** @type {Error & { code?: string, errors?: unknown[], contextData?: { outputPath?: string } }} */ (error) - if (value.code === 'DOM_STACK_ERROR_OUTPUT_CONFLICT') return value - return conflict(value.cause) ?? value.errors?.map(conflict).find(Boolean) -} - -/** @param {string} output @param {string} [content] */ -const template = (output, content = 'template') => `export default () => ({ outputName: ${JSON.stringify(output)}, content: ${JSON.stringify(content)} })` - -for (const scenario of [ - { name: 'regular page/template', files: { 'page.html': 'page', 'a.template.js': template('index.html') }, output: 'index.html', sources: ['page.html', 'a.template.js'] }, - { name: 'generated page/template', files: { 'a.pages.js': "export default { outputName: 'generated.html', children: 'page' }", 'b.template.js': template('generated.html') }, output: 'generated.html', sources: ['a.pages.js', 'b.template.js'] }, - { name: 'template objects', files: { 'a.template.js': template('feed.xml'), 'b.template.js': template('feed.xml') }, output: 'feed.xml', sources: ['a.template.js', 'b.template.js'] }, - { name: 'template array', files: { 'a.template.js': "export default () => [{ outputName: 'feed.xml', content: 'one' }, { outputName: 'feed.xml', content: 'two' }]" }, output: 'feed.xml', sources: ['a.template.js'] }, - { name: 'template async iterator', files: { 'a.template.js': "export default async function * () { yield { outputName: 'feed.xml', content: 'one' }; yield { outputName: 'feed.xml', content: 'two' } }" }, output: 'feed.xml', sources: ['a.template.js'] }, - { name: 'file/directory templates', files: { 'a.template.js': template('feed'), 'b.template.js': template('feed/index.xml') }, output: 'feed', sources: ['a.template.js', 'b.template.js'] }, - { name: 'static/template', files: { 'feed.xml': 'static', 'a.template.js': template('feed.xml') }, output: 'feed.xml', sources: ['feed.xml', 'a.template.js'] }, - - { name: 'esbuild/template', files: { 'page.html': 'page', 'client.js': 'console.log(1)', 'esbuild.settings.js': "export default opts => ({ ...opts, entryNames: '[dir]/[name]' })", 'a.template.js': template('client.js') }, output: 'client.js', sources: ['client.js', 'a.template.js'] }, - { name: 'esbuild settings cannot bypass claims', files: { 'page.html': 'page', 'client.js': 'console.log(1)', 'esbuild.settings.js': "export default opts => ({ ...opts, entryNames: '[dir]/[name]', metafile: false, write: true })", 'a.template.js': template('client.js') }, output: 'client.js', sources: ['client.js', 'a.template.js'] }, - { name: 'service worker/template', files: { 'service-worker.js': 'console.log(1)', 'a.template.js': template('service-worker.js') }, output: 'service-worker.js', sources: ['service-worker.js', 'a.template.js'] }, - { name: 'workers.json/template', files: { 'page.html': 'page', 'test.worker.js': 'console.log(1)', 'a.template.js': template('workers.json') }, output: 'workers.json', sources: ['page.html', 'a.template.js'] }, - { name: 'metadata/template', files: { 'a.template.js': template('domstack-esbuild-meta.json') }, output: 'domstack-esbuild-meta.json', sources: ['domstack esbuild metadata', 'a.template.js'] }, - { name: 'manifest/template', files: { 'a.template.js': template('domstack-manifest.json') }, output: 'domstack-manifest.json', sources: ['generated domstack manifest', 'a.template.js'] }, - { name: 'normalized separators', files: { 'a.template.js': template('feed/index.xml'), 'b.template.js': template('feed\\index.xml') }, output: 'feed/index.xml', sources: ['a.template.js', 'b.template.js'] }, -]) { - test(`one-shot rejects ${scenario.name} without overwriting the first producer`, async t => { - const { site, dest } = await setup(t, scenario.files, { domstackManifest: true }) - await writeFiles(dest, { 'sentinel.txt': 'last successful build' }) - await assert.rejects(site.build(), error => { - const found = conflict(error) - assert.ok(found, String(error)) - assert.ok(found.message.includes(scenario.output), found.message) - for (const source of scenario.sources) assert.ok(found.message.includes(source), found.message) - return true - }) - assert.ok(!(await readdir(dest)).some(name => name.startsWith('.domstack-'))) - if (scenario.name === 'static/template') assert.equal(await readFile(join(dest, 'feed.xml'), 'utf8'), 'static') - if (scenario.name.startsWith('esbuild/')) assert.match(await readFile(join(dest, 'client.js'), 'utf8'), /console.log/) - if (scenario.name === 'manifest/template') assert.equal(await readFile(join(dest, 'domstack-manifest.json'), 'utf8'), 'template') - if (scenario.name === 'service worker/template') assert.equal(await readFile(join(dest, 'service-worker.js'), 'utf8'), 'template') - assert.equal(await readFile(join(dest, 'sentinel.txt'), 'utf8'), 'last successful build') - }) -} - -test('copy producers are isolated before file/directory and cross-step checks', async t => { - for (const mode of ['copy-copy', 'copy-static', 'copy-page', 'copy-esbuild']) { - await t.test(mode, async t => { - const { tmp, src, dest } = await setup(t, { - ...(mode === 'copy-static' ? { feed: 'static' } : {}), - ...(mode === 'copy-page' ? { 'feed/page.html': 'page' } : {}), - ...(mode === 'copy-esbuild' ? { 'page.html': 'page', 'client.js': 'console.log(1)', 'esbuild.settings.js': "export default opts => ({ ...opts, entryNames: '[dir]/[name]' })" } : {}), - }) - const a = join(tmp, 'a') - const b = join(tmp, 'b') - await writeFiles(a, { [mode === 'copy-static' ? 'feed/index.xml' : mode === 'copy-esbuild' ? 'client.js' : 'feed']: 'copy a' }) - await writeFiles(b, { 'feed/index.xml': 'copy b' }) - const site = new DomStack(src, dest, { copy: mode === 'copy-copy' ? [a, b] : [a] }) - await assert.rejects(site.build(), error => { - assert.ok(conflict(error), String(error)) - return true - }) - assert.ok(!(await readdir(dest)).some(name => name.startsWith('.domstack-'))) - if (mode === 'copy-page') assert.equal(await readFile(join(dest, 'feed'), 'utf8'), 'copy a') - }) - } -}) - -test('stages are unique, do not delete lookalike user directories, and stay out of metadata', async t => { - const { site, dest, tmp } = await setup(t, { 'page.html': 'page', 'client.js': 'console.log(1)' }, { domstackManifest: true }) - await writeFiles(`${dest}.domstack-stage`, { 'keep.txt': 'user data' }) - const [a, b] = await Promise.all([site.build(), site.build()]) - assert.equal(a.domstackManifest?.version, b.domstackManifest?.version) - assert.equal(await readFile(join(`${dest}.domstack-stage`, 'keep.txt'), 'utf8'), 'user data') - assert.ok(!(await readFile(join(dest, 'domstack-esbuild-meta.json'), 'utf8')).includes('.domstack-stage')) - assert.doesNotMatch(JSON.stringify(a), /\.domstack-stage-[a-zA-Z0-9]{6}/) - assert.ok(!(await readdir(tmp)).some(name => name.startsWith('.domstack-stage-'))) -}) - -test('case-insensitive destination collisions follow the destination filesystem', async t => { - const { site, dest } = await setup(t, { 'a.template.js': template('Feed.xml'), 'b.template.js': template('feed.xml') }) - if (await isCaseInsensitiveDest(dest)) await assert.rejects(site.build(), error => !!conflict(error)) - else { - await site.build() - assert.equal(await readFile(join(dest, 'Feed.xml'), 'utf8'), 'template') - assert.equal(await readFile(join(dest, 'feed.xml'), 'utf8'), 'template') - } -}) - -test('registry distinguishes duplicate records from duplicate writes and bounds replacement state', () => { - const owner = { id: 'template:a', type: 'template', path: 'a.template.js' } - let registry = new OutputRegistry([], { caseInsensitive: true }) - registry.claim('Feed\\index.xml', owner) - assert.throws(() => registry.claim('feed/index.xml', owner), error => !!conflict(error)) - assert.throws(() => registry.claim('FEED', { ...owner, id: 'b' }), error => !!conflict(error)) - for (let i = 0; i < 100; i++) { - registry = new OutputRegistry(registry.snapshot(), { replaceOwnerIds: [owner.id] }) - registry.claim(`feed-${i}.xml`, owner) - assert.equal(registry.snapshot().length, 1) - } - const record = { filepath: '/dest/a', outputRelname: 'a', kind: /** @type {const} */ ('static'), sourceRelname: 'a' } - registry.claimRecords([record, { ...record }]) - assert.equal(registry.snapshot().length, 2) - for (const path of ['../bad', '/bad', 'C:\\bad', '.']) assert.throws(() => registry.claim(path, owner)) -}) - -/** @param {() => boolean | Promise} predicate */ -async function waitFor (predicate) { - const deadline = Date.now() + 5000 - while (!await predicate()) { - assert.ok(Date.now() < deadline, 'Timed out waiting for the watch result') - await new Promise(resolve => setTimeout(resolve, 25)) - } -} - -/** @param {DomStack} site */ -async function settle (site) { - await new Promise(resolve => setTimeout(resolve, 850)) - await site.settled() -} - -test('filtered watch conflicts retain successful outputs and recover after template renames and removals', { timeout: 30_000 }, async t => { - const { site, src, dest, logs } = await setup(t, { - 'a.template.js': template('a.txt', 'A'), - 'b.template.js': template('b.txt', 'B'), - }) - await site.watch({ serve: false }) - await writeFile(join(src, 'a.template.js'), template('b.txt', 'conflict')) - await settle(site) - assert.ok(logs.some(line => line.includes('Output path conflict'))) - assert.equal(await readFile(join(dest, 'a.txt'), 'utf8'), 'A') - assert.equal(await readFile(join(dest, 'b.txt'), 'utf8'), 'B') - await writeFile(join(src, 'a.template.js'), template('c.txt', 'C')) - await settle(site) - await assert.rejects(stat(join(dest, 'a.txt')), { code: 'ENOENT' }) - assert.equal(await readFile(join(dest, 'c.txt'), 'utf8'), 'C') - await rename(join(src, 'a.template.js'), join(src, 'renamed.template.js')) - await settle(site) - assert.equal(await readFile(join(dest, 'c.txt'), 'utf8'), 'C') - await rm(join(src, 'renamed.template.js')) - await settle(site) - await assert.rejects(stat(join(dest, 'c.txt')), { code: 'ENOENT' }) - await writeFile(join(src, 'b.template.js'), template('c.txt', 'reclaimed')) - await settle(site) - assert.equal(await readFile(join(dest, 'c.txt'), 'utf8'), 'reclaimed') - await assert.rejects(stat(join(dest, 'b.txt')), { code: 'ENOENT' }) -}) - -test('copy watch rejects page-owned paths and releases removed copy outputs', { timeout: 30_000 }, async t => { - const { tmp, src, dest, logs, sites } = await setup(t, { 'page.html': 'page' }) - const copy = join(tmp, 'copy') - await writeFiles(copy, { 'copy.txt': 'old' }) - const site = new DomStack(src, dest, { copy: [copy], logger: pino({}, { write: line => logs.push(line) }) }) - sites.push(site) - await site.watch({ serve: false }) - const page = await readFile(join(dest, 'index.html'), 'utf8') - await writeFiles(copy, { 'index.html': 'collision' }) - await waitFor(() => logs.some(line => line.includes('Output path conflict'))) - await settle(site) - assert.equal(await readFile(join(dest, 'index.html'), 'utf8'), page) - assert.ok(logs.some(line => line.includes('Output path conflict'))) - await rm(join(copy, 'index.html')) - await rename(join(copy, 'copy.txt'), join(copy, 'renamed.txt')) - await settle(site) - await assert.rejects(stat(join(dest, 'copy.txt')), { code: 'ENOENT' }) - assert.equal(await readFile(join(dest, 'renamed.txt'), 'utf8'), 'old') -}) - -test('service worker sourcemaps retain phase ownership across rebuilds and removal', { timeout: 30_000 }, async t => { - const { site, src, dest, logs } = await setup(t, { - 'page.html': 'page', - 'client.js': 'console.log(1)', - 'service-worker.js': 'console.log(1)', - 'a.template.js': template('a.txt'), - }) - await site.watch({ serve: false }) - await writeFile(join(src, 'service-worker.js'), 'console.log(2)') - await writeFile(join(src, 'client.js'), 'console.log(2)') - await settle(site) - assert.ok(!logs.some(line => line.includes('Output path conflict'))) - await writeFile(join(src, 'a.template.js'), template('service-worker.js.map')) - await settle(site) - assert.ok(logs.some(line => line.includes('Output path conflict'))) - assert.ok((await readFile(join(dest, 'service-worker.js.map'), 'utf8')).includes('sources')) - await rm(join(src, 'service-worker.js')) - await settle(site) - await writeFile(join(src, 'a.template.js'), template('service-worker.js.map', 'reclaimed')) - await settle(site) - assert.equal(await readFile(join(dest, 'service-worker.js.map'), 'utf8'), 'reclaimed') -}) - -test('initial watch conflicts leave the previous destination untouched and recover', { timeout: 30_000 }, async t => { - const { site, src, dest, logs } = await setup(t, { - 'page.html': 'new page', - 'client.js': 'console.log(1)', - 'a.template.js': template('client.js', 'collision'), - }) - await writeFiles(dest, { 'index.html': 'old page', 'client.js': 'old bundle' }) - await site.watch({ serve: false }) - assert.ok(logs.some(line => line.includes('Output path conflict'))) - assert.equal(await readFile(join(dest, 'index.html'), 'utf8'), 'old page') - assert.equal(await readFile(join(dest, 'client.js'), 'utf8'), 'old bundle') - await writeFile(join(src, 'a.template.js'), template('safe.txt', 'recovered')) - await settle(site) - assert.equal(await readFile(join(dest, 'index.html'), 'utf8'), 'new page') - assert.equal(await readFile(join(dest, 'safe.txt'), 'utf8'), 'recovered') - assert.match(await readFile(join(dest, 'client.js'), 'utf8'), /console.log/) -}) - -test('page promotion revalidates ownership acquired by esbuild during rendering', { timeout: 30_000 }, async t => { - const { site, src, dest, logs } = await setup(t, { - 'page.html': 'page', - 'client.js': 'console.log(1)', - 'payload.bin': 'esbuild asset', - 'a.template.js': template('a.txt', 'old template'), - 'esbuild.settings.js': "export default opts => ({ ...opts, assetNames: '[name]', loader: { ...opts.loader, '.bin': 'file' } })", - }, { static: false }) - await site.watch({ serve: false }) - await writeFile(join(src, 'a.template.js'), `import { writeFile } from 'node:fs/promises' -export default async () => { - await writeFile(new URL('./.rendering', import.meta.url), '') - await new Promise(resolve => setTimeout(resolve, 1200)) - return { outputName: 'payload.bin', content: 'template collision' } -}`) - for (let i = 0; i < 100; i++) { - if (await stat(join(src, '.rendering')).then(() => true, () => false)) break - await new Promise(resolve => setTimeout(resolve, 20)) - } - await stat(join(src, '.rendering')) - await writeFile(join(src, 'client.js'), "import asset from './payload.bin'; console.log(asset)") - await settle(site) - assert.equal(await readFile(join(dest, 'payload.bin'), 'utf8'), 'esbuild asset') - assert.equal(await readFile(join(dest, 'a.txt'), 'utf8'), 'old template') - assert.ok(logs.some(line => line.includes('Output path conflict'))) - await writeFile(join(src, 'a.template.js'), template('a.txt', 'recovered')) - await settle(site) - assert.equal(await readFile(join(dest, 'a.txt'), 'utf8'), 'recovered') - assert.equal(await readFile(join(dest, 'payload.bin'), 'utf8'), 'esbuild asset') -}) - -test('a producer can replace its own file with a directory and back', { timeout: 30_000 }, async t => { - const { site, src, dest } = await setup(t, { 'a.template.js': template('feed', 'file') }) - await site.watch({ serve: false }) - await writeFile(join(src, 'a.template.js'), template('feed/index.xml', 'nested')) - await settle(site) - assert.equal(await readFile(join(dest, 'feed/index.xml'), 'utf8'), 'nested') - await writeFile(join(src, 'a.template.js'), template('feed', 'file again')) - await settle(site) - assert.equal(await readFile(join(dest, 'feed'), 'utf8'), 'file again') -}) - -test('manifest hooks claim outputs and receive the public destination', async t => { - const { src, dest } = await setup(t, { 'page.html': 'page' }) - const site = new DomStack(src, dest, { - domstackManifest: { - hooks: { - manifestBuilt: [async ({ dest: hookDest, writeFile }) => { - assert.equal(hookDest, dest) - await writeFile('index.html', 'conflict') - }], - }, - }, - }) - await assert.rejects(site.build(), error => !!conflict(error)) - assert.equal(await readFile(join(dest, 'index.html'), 'utf8'), 'page') -}) - -test('one-shot hooks can immediately read their writes at the public destination', async t => { - const { src, dest } = await setup(t, { 'page.html': 'page' }) - await writeFiles(dest, { 'hook.txt': 'old' }) - const site = new DomStack(src, dest, { - domstackManifest: { - hooks: { - manifestBuilt: [async ({ dest: hookDest, writeFile }) => { - await writeFile('hook.txt', 'new') - assert.equal(await readFile(join(hookDest, 'hook.txt'), 'utf8'), 'new') - await writeFile('brand-new.txt', 'first write') - assert.equal(await readFile(join(hookDest, 'brand-new.txt'), 'utf8'), 'first write') - }], - }, - }, - }) - await site.build() -}) - -for (const watch of [false, true]) { - test(`copy reports keep live public mappers (full-watch staging: ${watch})`, async t => { - const { src, dest, tmp } = await setup(t, { 'asset.txt': 'static', 'client.js': 'console.log(1)', 'service-worker.js': 'console.log(2)' }) - const copyDir = join(tmp, 'copy') - await writeFiles(copyDir, { 'copied.txt': 'copy' }) - const result = await builder(src, dest, { copy: [copyDir] }, { watch }) - for (const value of [result.staticResults?.report, ...Object.values(result.copyResults?.report ?? {})]) { - const report = /** @type {{ options: { outputDir: string, toDestination: (source: string) => string }, copied: { source: string, output: string }[] }} */ (value) - assert.equal(report.options.outputDir, dest) - for (const file of report.copied) { - assert.equal(report.options.toDestination(file.source), file.output) - assert.equal(await readFile(file.output, 'utf8'), await readFile(file.source, 'utf8')) - } - } - assert.doesNotMatch(JSON.stringify(result), /\.domstack-(stage|copy|pages)-[a-zA-Z0-9]{6}/) - for (const name of (await readdir(dest)).filter(name => name.endsWith('.map'))) { - const map = JSON.parse(await readFile(join(dest, name), 'utf8')) - for (const source of map.sources) await stat(resolve(dest, source)) - } - }) -} - -for (const sameContents of [false, true]) { - test(`esbuild entry aliases identify both sources (identical contents: ${sameContents})`, async t => { - const { site } = await setup(t, { - 'a.js': 'console.log(1)', - 'b.js': sameContents ? 'console.log(1)' : 'console.log(2)', - 'esbuild.settings.js': 'export default opts => ({ ...opts, entryPoints: [new URL(\'./a.js\', import.meta.url).pathname, new URL(\'./b.js\', import.meta.url).pathname], entryNames: \'shared\' })', - }) - await assert.rejects(site.build(), error => { - const found = conflict(error) - assert.ok(found) - assert.match(found.message, /a\.js/) - assert.match(found.message, /b\.js/) - assert.match(found.message, /shared\.js/) - return true - }) - }) -} - -for (const paths of [['Feed.xml', 'feed.xml'], ['Feed', 'feed/index.xml']]) { - test(`copy inventory checks case aliases before staging: ${paths.join(', ')}`, async t => { - const { src, dest } = await setup(t, {}) - if (await isCaseInsensitiveDest(src)) return t.skip('Source filesystem cannot represent both aliases') - await writeFiles(src, Object.fromEntries(paths.map((path, i) => [path, String(i)]))) - const registry = new OutputRegistry([], { caseInsensitive: true }) - await assert.rejects(stagedCopy(join(src, '**'), src, dest, 'copy', registry), error => !!conflict(error)) - assert.ok(paths[0]) - await assert.rejects(stat(join(dest, paths[0])), { code: 'ENOENT' }) - }) -} - -test('worker returns a replacement delta including empty factories and deleted producers without dependency tracking', async t => { - const { src, dest } = await setup(t, { 'empty.pages.js': 'export default []' }) - const emptyOwner = `pages-file:${join(src, 'empty.pages.js')}` - const deletedOwner = `template:${join(src, 'deleted.template.js')}` - const previousOutputClaims = [ - { outputRelname: 'old.html', owner: { id: emptyOwner, type: 'page', path: 'empty.pages.js' } }, - { outputRelname: 'deleted.txt', owner: { id: deletedOwner, type: 'template', path: 'deleted.template.js' } }, - { outputRelname: 'asset.txt', owner: { id: 'static:asset.txt', type: 'static', path: 'asset.txt' } }, - ] - const result = await buildPages(src, dest, await identifyPages(src), { previousOutputClaims }) - assert.deepEqual(result.errors, []) - assert.deepEqual(result.report.newClaims, []) - assert.deepEqual(new Set(result.report.replacedOwnerIds), new Set([emptyOwner, deletedOwner])) - assert.ok(!('outputClaims' in result.report)) -}) - -test('watch uses its live copy inventory when files are renamed before initial rendering', { timeout: 30_000 }, async t => { - const { site, src, dest, logs } = await setup(t, { - 'old.txt': 'copied', - 'a.template.js': template('old.txt', 'page now owns old path'), - 'esbuild.settings.js': `import { access, writeFile } from 'node:fs/promises' -export default async opts => { - await writeFile(new URL('./.esbuild-started', import.meta.url), '') - while (!await access(new URL('./.continue', import.meta.url)).then(() => true, () => false)) await new Promise(resolve => setTimeout(resolve, 20)) - return opts -}`, - }, { ignore: ['.esbuild-started', '.continue'] }) - const startup = site.watch({ serve: false }) - await waitFor(() => stat(join(src, '.esbuild-started')).then(() => true, () => false)) - await rename(join(src, 'old.txt'), join(src, 'new.txt')) - const stageName = (await readdir(dest)).find(name => name.startsWith('.domstack-copy-watch-')) - assert.ok(stageName) - const stagedOld = join(dest, stageName, createHash('sha256').update(join(src, 'old.txt')).digest('hex')) - await waitFor(() => stat(stagedOld).then(() => false, () => true)) - await waitFor(() => logs.some(line => line.includes('Copy ') && line.includes('new.txt'))) - await writeFile(join(src, '.continue'), '') - await startup - assert.equal(await readFile(join(dest, 'old.txt'), 'utf8'), 'page now owns old path') - assert.equal(await readFile(join(dest, 'new.txt'), 'utf8'), 'copied') - assert.ok(!logs.some(line => line.includes('Output path conflict'))) - assert.equal(logs.filter(line => line.includes('Copy ') && line.includes('old.txt')).length, 1) -}) - -test('copy changes during initial rendering and onInitialBuild are drained', { timeout: 30_000 }, async t => { - const { site, src, dest, logs } = await setup(t, { - 'asset.txt': 'initial', - 'a.template.js': `import { writeFile } from 'node:fs/promises' -export default async () => { - await writeFile(new URL('./asset.txt', import.meta.url), 'during render') - await new Promise(resolve => setTimeout(resolve, 500)) - return { outputName: 'page.txt', content: 'page' } -}`, - }) - await site.watch({ - serve: false, - onInitialBuild: async () => { - assert.equal(await readFile(join(dest, 'asset.txt'), 'utf8'), 'during render') - const before = logs.filter(line => line.includes('Copy ') && line.includes('asset.txt')).length - await writeFile(join(src, 'asset.txt'), 'during callback') - await waitFor(() => logs.filter(line => line.includes('Copy ') && line.includes('asset.txt')).length > before) - } - }) - await site.settled() - assert.equal(await readFile(join(dest, 'asset.txt'), 'utf8'), 'during callback') -}) - -test('watch recovers after a page promotion I/O failure once the obstruction is removed', { timeout: 30_000 }, async t => { - const { site, src, dest, logs } = await setup(t, { 'a.template.js': template('old.txt', 'old') }) - await site.watch({ serve: false }) - await writeFiles(dest, { 'blocked.txt/unowned.txt': 'keep' }) - await writeFile(join(src, 'a.template.js'), template('blocked.txt', 'new')) - await settle(site) - assert.equal(await readFile(join(dest, 'blocked.txt/unowned.txt'), 'utf8'), 'keep') - assert.ok(logs.some(line => line.includes('EISDIR'))) - await rm(join(dest, 'blocked.txt'), { recursive: true }) - await writeFile(join(src, 'a.template.js'), template('blocked.txt', 'recovered')) - await settle(site) - assert.equal(await readFile(join(dest, 'blocked.txt'), 'utf8'), 'recovered') - await assert.rejects(stat(join(dest, 'old.txt')), { code: 'ENOENT' }) -}) - -test('native esbuild CSS bundle collisions identify both entry sources', async t => { - const { site } = await setup(t, { - 'a/client.js': "import './imported.css'", - 'a/imported.css': 'body { color: red }', - 'b/client.css': 'body { color: blue }', - 'esbuild.settings.js': 'export default opts => ({ ...opts, entryPoints: [new URL(\'./a/client.js\', import.meta.url).pathname, new URL(\'./b/client.css\', import.meta.url).pathname], entryNames: \'[name]\' })', - }) - await assert.rejects(site.build(), error => { - const found = conflict(error) - assert.ok(found, String(error)) - assert.match(found.message, /a\/client\.js/) - assert.match(found.message, /b\/client\.css/) - assert.match(found.message, /client\.css/) - return true - }) -}) - -test('one-shot and full-watch stages support a symlinked destination', async t => { - const { src, dest, tmp, site } = await setup(t, { 'page.html': 'page', 'asset.txt': 'static' }) - const actualDest = join(tmp, 'actual') - await mkdir(actualDest) - await symlink(actualDest, dest, 'dir') - for (const watch of [false, true]) { - const result = await builder(src, dest, {}, { watch }) - assert.equal(result.pageBuildResults?.outputs[0]?.filepath, join(dest, 'index.html')) - assert.equal(await readFile(join(actualDest, 'index.html'), 'utf8'), 'page') - assert.ok(!(await readdir(actualDest)).some(name => name.startsWith('.domstack-'))) - } - await site.watch({ serve: false }) - assert.equal(await readFile(join(actualDest, 'asset.txt'), 'utf8'), 'static') - await site.stopWatching() - assert.ok(!(await readdir(actualDest)).some(name => name.startsWith('.domstack-'))) -}) - -for (const watch of [false, true]) { - test(`failed page reports describe public paths (full-watch staging: ${watch})`, async t => { - const { src, dest } = await setup(t, { - 'page.html': 'successful render', - 'a.template.js': `export default async () => { - await new Promise(resolve => setTimeout(resolve, 100)) - throw new Error('render failed') -}`, - }) - await assert.rejects(builder(src, dest, {}, { watch }), error => { - assert.ok(error instanceof DomStackAggregateError) - assert.equal(error.results.pageBuildResults.report.pages[0].pageFilePath, join(dest, 'index.html')) - assert.doesNotMatch(JSON.stringify(error.results), /\.domstack-(stage|copy|pages)-[a-zA-Z0-9]{6}/) - return true - }) - await assert.rejects(stat(join(dest, 'index.html')), { code: 'ENOENT' }) - assert.ok(!(await readdir(dest)).some(name => name.startsWith('.domstack-'))) - }) -} - -for (const phase of ['copy', 'esbuild']) { - test(`initial ${phase} conflicts abort watch and clean up startup resources`, async t => { - const { src, dest, tmp, sites } = await setup(t, { - ...(phase === 'copy' - ? { 'asset.txt': 'static' } - : { - 'a.js': 'console.log(1)', - 'b.js': 'console.log(2)', - 'esbuild.settings.js': 'export default opts => ({ ...opts, entryPoints: [new URL(\'./a.js\', import.meta.url).pathname, new URL(\'./b.js\', import.meta.url).pathname], entryNames: \'shared\' })', - }), - }) - const copyDir = join(tmp, 'copy') - await writeFiles(copyDir, { 'asset.txt': 'copy' }) - const site = new DomStack(src, dest, { copy: phase === 'copy' ? [copyDir] : [], logger: pino({ level: 'silent' }) }) - sites.push(site) - await writeFiles(dest, { 'sentinel.txt': 'old destination' }) - await assert.rejects(site.watch({ serve: false }), error => !!conflict(error)) - assert.equal(site.watching, false) - assert.deepEqual(await readdir(dest), ['sentinel.txt']) - }) -} - -test('overlapping copy roots retain both mappings through watch startup, edits, full rebuilds, renames and removal', { timeout: 30_000 }, async t => { - const { src, dest, tmp, sites, logs } = await setup(t, { 'page.html': 'page' }) - const assets = join(tmp, 'assets') - const nested = join(assets, 'nested') - await writeFiles(nested, { 'asset.txt': 'initial' }) - const opts = { copy: [assets, nested], logger: pino({ level: 'debug' }, { write: line => logs.push(line) }) } - const site = new DomStack(src, dest, opts) - sites.push(site) - const assertMappings = async (/** @type {string} */ name, /** @type {string} */ content) => { - for (const path of [name, join('nested', name)]) assert.equal(await readFile(join(dest, path), 'utf8'), content) - } - const assertRemoved = async (/** @type {string} */ name) => { - for (const path of [name, join('nested', name)]) await assert.rejects(stat(join(dest, path)), { code: 'ENOENT' }) - } - await site.build() - await assertMappings('asset.txt', 'initial') - await rm(dest, { recursive: true }) - await site.watch({ serve: false }) - await assertMappings('asset.txt', 'initial') - await writeFile(join(nested, 'asset.txt'), 'edited') - await settle(site) - await assertMappings('asset.txt', 'edited') - - await writeFile(join(src, 'global.vars.js'), "export default { layout: 'root', rebuilt: true }") - await settle(site) - assert.ok(logs.some(line => line.includes('Triggering full rebuild'))) - await assertMappings('asset.txt', 'edited') - await writeFile(join(nested, 'asset.txt'), 'after full rebuild') - await settle(site) - await assertMappings('asset.txt', 'after full rebuild') - - await rename(join(nested, 'asset.txt'), join(nested, 'renamed.txt')) - await settle(site) - await assertRemoved('asset.txt') - await assertMappings('renamed.txt', 'after full rebuild') - await rm(join(nested, 'renamed.txt')) - await settle(site) - await assertRemoved('renamed.txt') - await writeFile(join(nested, 'asset.txt'), 'recreated') - await settle(site) - await assertMappings('asset.txt', 'recreated') - assert.ok(!logs.some(line => line.includes('Output path conflict'))) - - // A new file in the outer root cannot take the inner mapping's output, and - // removing that rejected producer must not remove the successful mapping. - await writeFile(join(assets, 'asset.txt'), 'conflicting outer file') - await settle(site) - assert.ok(logs.some(line => line.includes('Output path conflict'))) - await assertMappings('asset.txt', 'recreated') - await rm(join(assets, 'asset.txt')) - await settle(site) - await assertMappings('asset.txt', 'recreated') -}) - -for (const repeatedRoot of [false, true]) { - test(`copy root mappings that emit the same output still conflict (repeated root: ${repeatedRoot})`, async t => { - const { src, dest, tmp, sites } = await setup(t, {}) - const assets = join(tmp, 'assets') - const nested = join(assets, 'nested') - await writeFiles(assets, { 'asset.txt': 'root', 'nested/asset.txt': 'nested' }) - const opts = { copy: [assets, repeatedRoot ? assets : nested], logger: pino({ level: 'silent' }) } - for (const watch of [false, true]) { - await assert.rejects(builder(src, dest, opts, { watch }), error => !!conflict(error)) - } - await rm(dest, { recursive: true, force: true }) - const site = new DomStack(src, dest, opts) - sites.push(site) - await assert.rejects(site.watch({ serve: false }), error => !!conflict(error)) - assert.equal(site.watching, false) - }) -} diff --git a/test-cases/page-additional-outputs/helpers.js b/test-cases/page-additional-outputs/helpers.js index 0cf8d7e1..75bb200a 100644 --- a/test-cases/page-additional-outputs/helpers.js +++ b/test-cases/page-additional-outputs/helpers.js @@ -28,7 +28,7 @@ export async function setup (t, files) { ...files, }) const logs = /** @type {string[]} */ ([]) - const options = { domstackManifest: false, logger: pino({ level: 'debug' }, { write: line => logs.push(line) }) } + const options = { static: true, domstackManifest: false, logger: pino({ level: 'debug' }, { write: line => logs.push(line) }) } const site = new DomStack(src, dest, options) t.after(async () => { if (site.watching) await site.stopWatching() diff --git a/test-cases/page-additional-outputs/index.test.js b/test-cases/page-additional-outputs/index.test.js index 17aa3d02..cb5fdcdb 100644 --- a/test-cases/page-additional-outputs/index.test.js +++ b/test-cases/page-additional-outputs/index.test.js @@ -139,26 +139,23 @@ for (const scenario of [ { name: 'layout hook', output: 'shared.txt', files: { 'root.layout.js': 'export default ({ children }) => children; ' + hook('shared.txt') } }, { name: 'other page hook', output: 'shared.txt', files: { 'other/page.js': "export default () => 'other'; " + hook('/shared.txt') } }, ]) { - test(`builder rejects sidecar collision with ${scenario.name}`, async t => { - const { build, dest, read } = await setup(t, { + test(`builder warns about a duplicate sidecar destination with ${scenario.name}`, async t => { + const { build } = await setup(t, { 'page.js': "export default () => 'new main'; " + hook(scenario.output), ...scenario.files, }) - await writeFiles(dest, { 'index.html': 'previous main', 'sentinel.txt': 'keep' }) - await assert.rejects(build(), error => { - assert.match(errorText(error), /Output path conflict/) - assert.ok(errorText(error).includes(scenario.output), errorText(error)) - return true - }) - assert.equal(await read('index.html'), 'previous main', 'failed page phase never publishes main HTML') - assert.equal(await read('sentinel.txt'), 'keep') + const result = await build() + assert.ok(result.warnings.some(warning => { + const message = errorText(warning) + return /duplicate|conflict/i.test(message) && message.includes(scenario.output) + }), `expected a duplicate destination warning for ${scenario.output}: ${errorText(result.warnings)}`) }) } for (const result of [ "'bare string'", "{ outputName: 'bad.txt', content: 42 }", - "[{ outputName: 'same.txt', content: 'same' }, { outputName: './same.txt', content: 'same' }]", + "{ outputName: '../escape.txt', content: 'bad' }", "{ outputName: '/', content: 'bad' }", ]) { @@ -173,7 +170,7 @@ for (const result of [ }) } -test('iterator failure after a yield preserves all live page-phase outputs', async t => { +test('iterator failure after a yield leaves the owning page unchanged', async t => { const { build, dest, read } = await setup(t, { 'a/page.js': "export default () => 'new sibling'; " + hook('sibling.txt', 'new sibling sidecar'), 'z/page.js': `export default () => 'new main'; export async function* additionalOutputs () { @@ -182,7 +179,7 @@ test('iterator failure after a yield preserves all live page-phase outputs', asy throw Error('iterator exploded') }`, }) - const previous = { 'a/index.html': 'old sibling', 'a/sibling.txt': 'old sibling sidecar', 'z/index.html': 'old main', 'z/old.txt': 'old sidecar', 'z/stale.txt': 'retain on failure' } + const previous = { 'z/index.html': 'old main', 'z/old.txt': 'old sidecar', 'z/stale.txt': 'retain on failure' } await writeFiles(dest, previous) await assert.rejects(build(), error => { assert.match(errorText(error), /iterator exploded/) @@ -191,3 +188,32 @@ test('iterator failure after a yield preserves all live page-phase outputs', asy for (const [name, content] of Object.entries(previous)) assert.equal(await read(name), content) await assert.rejects(stat(join(dest, 'z/partial.txt')), { code: 'ENOENT' }) }) + +test('identical duplicate records from one hook warn rather than reject the build', async t => { + const { build, read } = await setup(t, { + 'page.js': `export default () => 'main'; export const additionalOutputs = () => [ + { outputName: 'same.txt', content: 'same' }, + { outputName: './same.txt', content: 'same' }, + ]`, + }) + const result = await build() + assert.equal(await read('index.html'), 'main') + assert.equal(await read('same.txt'), 'same') + assert.ok(result.warnings.some(warning => { + const message = errorText(warning) + return /duplicate|conflict/i.test(message) && message.includes('same.txt') + }), `expected a duplicate destination warning: ${errorText(result.warnings)}`) +}) + +test('render failure leaves the owning page HTML and sidecars unchanged', async t => { + const { build, dest, read } = await setup(t, { + 'page.js': "export default () => { throw Error('render exploded') }; " + hook('old.txt', 'replacement'), + }) + await writeFiles(dest, { 'index.html': 'old main', 'old.txt': 'old sidecar' }) + await assert.rejects(build(), error => { + assert.match(errorText(error), /render exploded/) + return true + }) + assert.equal(await read('index.html'), 'old main') + assert.equal(await read('old.txt'), 'old sidecar') +}) diff --git a/test-cases/page-additional-outputs/ownership.test.js b/test-cases/page-additional-outputs/ownership.test.js new file mode 100644 index 00000000..14739bf4 --- /dev/null +++ b/test-cases/page-additional-outputs/ownership.test.js @@ -0,0 +1,70 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { mkdir, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { hook, setup, settle } from './helpers.js' + +test('data-invalidated pages replace ownership using actual reports', { timeout: 15_000 }, async t => { + const { site, src, dest, read, logs } = await setup(t, { + 'global.data.js': "export default { name: 'old.txt' }", + 'page.js': "export const vars = { dataDeps: ['name'] }; export default () => 'main'; export const additionalOutputs = ({ data }) => ({ outputName: data.name, content: 'sidecar' })", + }) + await site.watch({ serve: false }) + await settle(site, logs, async () => { + await writeFile(join(src, 'global.data.js'), "export default { name: 'new.txt' }") + }) + assert.equal(await read('new.txt'), 'sidecar') + await assert.rejects(stat(join(dest, 'old.txt')), { code: 'ENOENT' }) +}) + +for (const owner of ['page', 'template']) { + test(`targeted cleanup protects an untouched ${owner} claim`, { timeout: 15_000 }, async t => { + const { site, src, read, logs } = await setup(t, { + 'page.js': "export default () => 'main'; " + hook('shared.html'), + ...(owner === 'page' + ? { 'shared.md': 'other page' } + : { 'shared.template.js': "export default () => ({ outputName: 'shared.html', content: 'template' })" }), + }) + await site.watch({ serve: false }) + const shared = await read('shared.html') + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), "export default () => 'updated main'") + }) + assert.equal(await read('shared.html'), shared) + }) +} + +test('failed direct builds retain successful ownership for recovery', { timeout: 15_000 }, async t => { + const { site, src, dest, read, logs } = await setup(t, { + 'page.js': "export default () => 'old main'; " + hook('old.txt'), + }) + await site.watch({ serve: false }) + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* additionalOutputs () { + yield { outputName: 'partial.txt', content: 'partial' } + throw Error('ownership failure') + }`) + }, 'ownership failure') + assert.equal(await read('old.txt'), 'sidecar', 'failure must not clean up the previous successful output') + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), "export default () => 'recovered'; " + hook('new.txt')) + }) + assert.equal(await read('new.txt'), 'sidecar') + await assert.rejects(stat(join(dest, 'old.txt')), { code: 'ENOENT' }) +}) + +test('stale cleanup does not follow symlink ancestors outside dest', { timeout: 15_000 }, async t => { + const { site, src, dest, logs } = await setup(t, { + 'page.js': "export default () => 'main'; " + hook('nested/owned.txt'), + }) + await site.watch({ serve: false }) + const outside = join(dest, '..', 'outside') + await mkdir(outside) + await writeFile(join(outside, 'owned.txt'), 'keep') + await rm(join(dest, 'nested'), { recursive: true }) + await symlink(outside, join(dest, 'nested'), 'dir') + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), "export default () => 'main without hook'") + }) + assert.equal(await readFile(join(outside, 'owned.txt'), 'utf8'), 'keep') +}) diff --git a/test-cases/page-additional-outputs/promotion.test.js b/test-cases/page-additional-outputs/promotion.test.js deleted file mode 100644 index 54b40ec2..00000000 --- a/test-cases/page-additional-outputs/promotion.test.js +++ /dev/null @@ -1,73 +0,0 @@ -import { test } from 'node:test' -import assert from 'node:assert/strict' -import { mkdir, readFile, rename, rm, stat, symlink, writeFile } from 'node:fs/promises' -import { join } from 'node:path' -import { hook, setup, settle } from './helpers.js' - -for (const full of [false, true]) { - const mode = full ? 'full' : 'targeted' - test(`${mode} watch converges owned file to directory and back`, { timeout: 30_000 }, async t => { - const { site, src, dest, read, mtime, logs } = await setup(t, { - 'page.js': "export default () => 'main'; export const additionalOutputs = ({ vars }) => ({ outputName: vars.rawName, content: 'raw bytes' })", - 'global.vars.js': "export default { layout: 'root', rawName: '/raw' }", - 'page.vars.js': 'export default {}', - }) - await site.watch({ serve: false }) - const change = async (/** @type {string} */ name) => { - await settle(site, logs, async () => { - await writeFile(join(src, full ? 'global.vars.js' : 'page.vars.js'), `export default { layout: 'root', rawName: ${JSON.stringify(name)} }`) - }) - } - await change('/raw/article.md') - assert.equal(await read('raw/article.md'), 'raw bytes') - const time = await mtime('raw/article.md') - await change('/raw/article.md') - assert.equal(await mtime('raw/article.md'), time) - await change('/raw') - assert.equal(await read('raw'), 'raw bytes') - assert.equal((await stat(join(dest, 'raw'))).isFile(), true) - }) - - test(`${mode} watch validates all stale paths before removing any output`, { timeout: 30_000 }, async t => { - const { site, src, dest, read, logs } = await setup(t, { - 'page.js': "export default () => 'old main'; export const additionalOutputs = () => [{ outputName: '/first.txt', content: 'first' }, { outputName: '/raw/article.md', content: 'owned' }]", - }) - await site.watch({ serve: false }) - const outside = join(src, '..', 'outside') - await mkdir(outside) - await writeFile(join(outside, 'article.md'), 'external sentinel') - await rename(join(dest, 'raw'), join(dest, 'saved-raw')) - await symlink(outside, join(dest, 'raw')) - await settle(site, logs, async () => { - if (full) await rm(join(src, 'page.js')) - else await writeFile(join(src, 'page.js'), "export default () => 'new main'") - }, 'symlink') - assert.ok(logs.some(line => line.includes('symlink'))) - assert.equal(await readFile(join(outside, 'article.md'), 'utf8'), 'external sentinel') - assert.equal(await read('first.txt'), 'first') - assert.equal(await read('index.html'), 'old main') - await rm(join(dest, 'raw')) - await rename(join(dest, 'saved-raw'), join(dest, 'raw')) - await settle(site, logs, async () => { - await writeFile(join(src, 'page.js'), "export default () => 'recovered'; " + hook('/new.txt')) - }) - assert.equal(await read('new.txt'), 'sidecar') - for (const name of ['first.txt', 'raw/article.md']) await assert.rejects(stat(join(dest, name)), { code: 'ENOENT' }) - }) - - test(`${mode} watch rejects an unowned blocking file before stale cleanup`, { timeout: 30_000 }, async t => { - const { site, src, dest, read, logs } = await setup(t, { - 'page.js': "export default () => 'main'; export const additionalOutputs = ({ vars }) => ({ outputName: vars.rawName, content: 'owned' })", - 'global.vars.js': "export default { layout: 'root', rawName: '/old.txt' }", - 'page.vars.js': 'export default {}', - }) - await site.watch({ serve: false }) - await writeFile(join(dest, 'raw'), 'unowned sentinel') - await settle(site, logs, async () => { - await writeFile(join(src, full ? 'global.vars.js' : 'page.vars.js'), "export default { layout: 'root', rawName: '/raw/article.md' }") - }, 'non-directory ancestor') - assert.ok(logs.some(line => line.includes('non-directory ancestor'))) - assert.equal(await read('raw'), 'unowned sentinel') - assert.equal(await read('old.txt'), 'owned') - }) -} diff --git a/test-cases/page-additional-outputs/watch.test.js b/test-cases/page-additional-outputs/watch.test.js index cb8c2e10..62100e87 100644 --- a/test-cases/page-additional-outputs/watch.test.js +++ b/test-cases/page-additional-outputs/watch.test.js @@ -102,12 +102,13 @@ test('watch removes sidecars on source rename and draft exclusion', { timeout: 3 assert.equal(await read('renamed.html.txt'), '# Article\n') }) -test('watch iterator failure preserves ownership and recovery removes stale outputs', { timeout: 30_000 }, async t => { +test('watch hook failure leaves its owning page unchanged and recovery removes stale outputs', { timeout: 30_000 }, async t => { const { site, src, dest, read, mtime, logs } = await setup(t, { 'page.js': "export default () => 'old main'; " + hook('old.txt', 'old sidecar'), }) await site.watch({ serve: false }) const oldTime = await mtime('old.txt') + const oldHtmlTime = await mtime('index.html') await settle(site, logs, async () => { await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* additionalOutputs () { yield { outputName: 'old.txt', content: 'failed replacement' } @@ -117,6 +118,7 @@ test('watch iterator failure preserves ownership and recovery removes stale outp }, 'watch iterator exploded') assert.ok(logs.some(line => line.includes('watch iterator exploded')), 'watch reports the hook failure') assert.equal(await read('index.html'), 'old main') + assert.equal(await mtime('index.html'), oldHtmlTime) assert.equal(await read('old.txt'), 'old sidecar') assert.equal(await mtime('old.txt'), oldTime) await assert.rejects(stat(join(dest, 'partial.txt')), { code: 'ENOENT' }) From e5176c7f6b729132d3b144c354d97a0c972ac638 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Sun, 13 Sep 2026 20:04:32 -0700 Subject: [PATCH 4/8] Remove redundant JavaScript page builder result type --- lib/build-pages/page-builders/js/index.js | 7 +------ lib/build-pages/page-data.js | 3 +-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/build-pages/page-builders/js/index.js b/lib/build-pages/page-builders/js/index.js index cf0a3b45..6e4c3eb9 100644 --- a/lib/build-pages/page-builders/js/index.js +++ b/lib/build-pages/page-builders/js/index.js @@ -1,11 +1,6 @@ /** * @import { PageInfo } from '../../../identify-pages.js' * @import { PageBuilderResult } from '../page-writer.js' - * @import { AdditionalOutputsFunction } from '../../additional-outputs.js' - * - * @template {Record} T - * @template [U=any] - * @typedef {PageBuilderResult & { additionalOutputs?: AdditionalOutputsFunction }} JsPageBuilderResult */ import assert from 'node:assert' @@ -17,7 +12,7 @@ import { validateAdditionalOutputsHook } from '../../additional-outputs.js' * @template [U=any] U - The return type of the page function * @param {object} params * @param {PageInfo} params.pageInfo - * @returns {Promise>} + * @returns {Promise>} */ export async function jsBuilder ({ pageInfo }) { assert(pageInfo.type === 'js', 'js page builder requires "js" page type') diff --git a/lib/build-pages/page-data.js b/lib/build-pages/page-data.js index bc521818..1f1956e3 100644 --- a/lib/build-pages/page-data.js +++ b/lib/build-pages/page-data.js @@ -2,7 +2,6 @@ * @import { PageInfo } from '../identify-pages.js' * @import { BuilderOptions, InternalPageFunction } from './page-builders/page-writer.js' * @import { AdditionalOutputsFunction, AdditionalOutputProvenance, CollectedAdditionalOutput } from './additional-outputs.js' - * @import { JsPageBuilderResult } from './page-builders/js/index.js' */ import { readFile } from 'node:fs/promises' @@ -321,7 +320,7 @@ export class PageData { const built = await builder({ pageInfo, options: this.builderOptions }) const { vars: builderVars } = built if (!pageInfo.generated) { - const moduleHook = type === 'js' ? /** @type {JsPageBuilderResult} */ (built).additionalOutputs : undefined + const moduleHook = type === 'js' ? built.additionalOutputs : undefined const companionHook = pageVars?.filepath ? validateAdditionalOutputsHook((await import(pageVars.filepath)).additionalOutputs, pageVars.filepath) : undefined From f4a26c6f1f8da99105507db9254de10da67bb726 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Sun, 13 Sep 2026 20:36:44 -0700 Subject: [PATCH 5/8] Allow undefined additional output hooks without conditional spreads --- lib/build-pages/page-builders/js/index.js | 2 +- lib/build-pages/page-builders/page-writer.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/build-pages/page-builders/js/index.js b/lib/build-pages/page-builders/js/index.js index 6e4c3eb9..94f69bc4 100644 --- a/lib/build-pages/page-builders/js/index.js +++ b/lib/build-pages/page-builders/js/index.js @@ -33,5 +33,5 @@ export async function jsBuilder ({ pageInfo }) { assert(typeof pageLayout === 'function', 'js pages pageLayout must be a function') const hook = validateAdditionalOutputsHook(additionalOutputs, pageInfo.pageFile.filepath) - return { vars, pageLayout, ...(hook ? { additionalOutputs: hook } : {}) } + return { vars, pageLayout, additionalOutputs: hook } } diff --git a/lib/build-pages/page-builders/page-writer.js b/lib/build-pages/page-builders/page-writer.js index a49bbef6..69121336 100644 --- a/lib/build-pages/page-builders/page-writer.js +++ b/lib/build-pages/page-builders/page-writer.js @@ -76,7 +76,7 @@ import { writeAdditionalOutputs } from './additional-output-writer.js' * @typedef PageBuilderResult * @property {Partial} vars - Any variables resolved by the builder * @property {InternalPageFunction} pageLayout - The function that returns the rendered page - * @property {AdditionalOutputsFunction} [additionalOutputs] - Optional build-only additional-output hook. + * @property {AdditionalOutputsFunction | undefined} [additionalOutputs] - Optional build-only additional-output hook. */ /** From f5cfb0afeaaf6c784383570547d61227e381e011 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Sun, 13 Sep 2026 21:59:11 -0700 Subject: [PATCH 6/8] Stream additional outputs and generated pages as they are yielded --- docs/generation/README.md | 28 ++ docs/layouts/README.md | 4 + docs/pages/README.md | 16 +- index.js | 15 + .../additional-outputs-types.test.ts | 9 +- lib/build-pages/additional-outputs.js | 28 +- lib/build-pages/additional-outputs.test.js | 91 ++++- lib/build-pages/index.js | 160 ++++----- .../page-builders/additional-output-writer.js | 25 +- .../additional-output-writer.test.js | 6 +- lib/build-pages/page-builders/page-writer.js | 6 +- .../page-data-additional-outputs.test.js | 152 ++++++++- lib/build-pages/page-data.js | 23 +- test-cases/generated-pages/index.test.js | 3 +- test-cases/generated-pages/streaming.test.js | 323 ++++++++++++++++++ .../page-additional-outputs/index.test.js | 104 +++++- .../page-additional-outputs/ownership.test.js | 56 ++- .../page-additional-outputs/watch.test.js | 18 +- test-cases/watch/index.test.js | 4 +- 19 files changed, 907 insertions(+), 164 deletions(-) create mode 100644 test-cases/generated-pages/streaming.test.js diff --git a/docs/generation/README.md b/docs/generation/README.md index 03831595..484361eb 100644 --- a/docs/generation/README.md +++ b/docs/generation/README.md @@ -48,6 +48,8 @@ A generated-pages module can default-export: | An async iterable, usually returned by `async function*` | Pages are discovered incrementally or the total is not known in advance | Static objects and arrays do not receive factory parameters. +A `null` or `undefined` default export or factory result produces no pages, as does an empty array or async iterable. +Each array entry or yielded value must still be a page-definition object; `null` entries are not skipped. #### One page definition @@ -138,6 +140,29 @@ export default async function * archivePages ({ data }) { } ``` +#### Streaming and failures + +DOMStack consumes generated pages one at a time rather than collecting all definitions before building them. +For each definition, it validates the definition and output path, initializes the page's variables, layouts, and declared data, then renders and writes the HTML before requesting the next definition. +An async generator resumes after `yield` only once that page has been written, so it can release resources associated with the completed page before preparing the next one. +Arrays and single-object results use the same per-page pipeline, although an array-producing factory necessarily creates its array before returning it. + +Definitions marked `draft: true` are skipped unless drafts are enabled. +Skipped drafts retain their position in source identifiers: after a skipped first definition, the next page is identified as `archive.pages.ts#1`, not `archive.pages.ts#0`. +Output paths must be unique across generated pages and must not collide with source-backed pages. +Do not rely on the processing order of sibling factory modules. + +A factory, validation, initialization, or render failure stops the active iterator without requesting later definitions. +Async generators can use `try`/`finally` to release resources when iteration stops. +Earlier completed pages remain written; generated-page builds are not transactional, and a late failure does not roll back earlier HTML. +Conflict errors still identify both sources even when one page has already been written. +Build results retain output metadata and the owning pages-file path for completed pages, including when the build fails. + +In watch mode, failed builds retain both previously owned outputs and any newly written partial outputs without stale-output cleanup. +A later successful rebuild removes obsolete outputs, including partial pages from repeated failures or an initially failed watch build. +Removing the factory or changing it to return no pages also cleans up its tracked outputs after a successful rebuild. +This ownership tracking does not require a public DOMStack manifest and does not remove outputs still owned by sibling factories. + ### Generated-pages factory parameters Functions receive one object with: @@ -150,6 +175,8 @@ Functions receive one object with: Factories do not receive raw source or generated `PageData` collections. Put page-collection logic in [`global.data.ts`](../data/#global-data), return a focused serializable value, and subscribe to its key from the factory. +Source-backed pages remain fully initialized before `global.data.ts` runs, so it can inspect their resolved variables and use their rendering methods subject to the normal data-dependency rules. +Generated pages are not included in that collection, even after earlier yielded pages have been written. This keeps factories downstream of source discovery without exposing generation order or creating page-generation cycles. ### Generated page definitions @@ -163,6 +190,7 @@ This keeps factories downstream of source discovery without exposing generation Generated pages use [global bundles](../global-bundles/) and [layout assets](../layouts/#layout-styles). They do not have page-local `style.css`, `client.js`, or worker entries because they do not have their own source-page directory. +Generated pages skip [additional-output hooks](../pages/#additional-outputs), including hooks inherited from layouts; streaming does not enable these hooks. ### Generated-pages types diff --git a/docs/layouts/README.md b/docs/layouts/README.md index 9a6cfd07..4889c69f 100644 --- a/docs/layouts/README.md +++ b/docs/layouts/README.md @@ -128,6 +128,10 @@ The filename is relative to the current page's output directory, not the layout Using the page's HTML filename helps keep destinations unique when several loose Markdown pages share a directory. Exact duplicate destinations produce best-effort build warnings, not an override contract; avoid sharing output paths between pages or hooks. Nested hooks run outermost layout → innermost layout → page, and each layout hook shares only that layout renderer's `vars.dataDeps` subscriptions. +Hooks may return a record, an array, or an async iterable, directly or through a promise. +Each record is validated and written (or checked for identical content) before the next record is requested, and a later hook runs only after the preceding hook's files have been processed. +If a later record or provider fails, earlier sidecar writes remain in the destination; outputs are not staged or rolled back. +Watch mode retains these partial outputs in the page's ownership for cleanup after recovery or removal. Generated pages skip these hooks, including inherited layout hooks. See [Additional outputs](../pages/#additional-outputs) for the complete API, companion modules, path rules, and watch behavior. diff --git a/docs/pages/README.md b/docs/pages/README.md index d9486c88..de3315be 100644 --- a/docs/pages/README.md +++ b/docs/pages/README.md @@ -387,6 +387,9 @@ Bare strings are invalid because there is no implicit filename. An empty array or an async iterator that yields nothing declares no files for that hook. Applicable hooks execute in outermost layout → innermost layout → page order, and all their outputs are additive. +Records are consumed sequentially, without collecting all hook results first. +DOMStack validates each record and its destination, then writes its content (or checks that the existing bytes are identical) before requesting the next record. +A later layout or page hook starts only after the preceding hook's files have been processed. Returning `[]` from the page hook does not suppress layout outputs. For an application-specific opt-out, have the layout inspect a resolved variable such as `rawExport: false` and return `[]` itself. Hooks run only in the owning page's output-build phase, not when collection or global-data code calls `renderInnerPage()` or `renderFullPage()`. @@ -413,16 +416,19 @@ They do not reject the build, even when content differs. Watch warnings only compare outputs observed in the current page/template phase; this is not a persistent cross-build conflict registry or a case-alias check. Choose unique destinations; do not rely on write order or cleanup behavior for conflicting outputs. -DOMStack renders a page and collects and validates all its hook results before writing that page's HTML and additional files directly to the destination. -If rendering, a hook, an iterator, or output validation fails, that owning page's existing HTML and sidecars remain unchanged. -An iterator that throws after yielding records therefore does not write those earlier yields. -Other pages and build phases may already have written their outputs; there is no whole-build or page-phase isolation, and filesystem write failures can leave partial updates. +Additional files are written directly to the destination as their records arrive, without staging or rollback. +If a later hook, iterator step, output validation, or write fails, earlier sidecar writes remain, including updates to existing files and newly created files. +Processing stops at the failure rather than requesting subsequent records or invoking later hooks. +The page's HTML is currently rendered before sidecar processing and written only after its hooks succeed, so a hook failure leaves the previous HTML in place. +This is not a transactional guarantee: other pages and build phases may already have written their outputs, and filesystem write failures can leave partial updates. ### Watch behavior and ownership Additional files belong to the source page and appear in page build reports and the build manifest. Ownership tracking and cleanup also work when public build-manifest generation is disabled. -After a successful rebuild, DOMStack removes previously owned files no longer returned, including renamed outputs and files from removed hooks. +A failed build retains prior ownership and adds any sidecar paths already emitted before the failure; it does not clean up the page's old outputs. +After a successful rebuild, DOMStack removes previously owned files no longer returned, including renamed outputs, files from removed hooks, and partial outputs retained from failed attempts. +Partial ownership is tracked even when the initial watch build fails, so recovery, hook removal, or source deletion can clean up those files. Source deletion, source rename, or draft exclusion also removes the page's old outputs. Adding, editing, removing, or renaming a companion updates the owning page's hook and output set. diff --git a/index.js b/index.js index d99488e3..0a5c2659 100644 --- a/index.js +++ b/index.js @@ -281,6 +281,7 @@ export class DomStack { trackWatchDependencies: true, }) if (pageBuildResults.errors.length > 0) { + this.#rememberPartialPageOutputs(pageBuildResults) throw new DomStackAggregateError(pageBuildResults.errors, 'Page build finished but there were errors.', { siteData, pageBuildResults, @@ -536,6 +537,7 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) trackWatchDependencies: true, }) if (pageBuildResults.errors.length > 0) { + this.#rememberPartialPageOutputs(pageBuildResults) throw new DomStackAggregateError(pageBuildResults.errors, 'Page build finished but there were errors.', { siteData, pageBuildResults, @@ -566,6 +568,19 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) } } + /** + * Failed direct builds can leave new files. Keep their paths alongside prior + * ownership without cleaning anything up until a successful rebuild. + * @param {Pick} results + */ + #rememberPartialPageOutputs (results) { + for (const [owner, outputs] of getPageOutputMap(resolve(this.#dest), results.report.pages)) { + const previous = this.#pageOutputMap.get(owner) ?? new Set() + for (const path of outputs) previous.add(path) + this.#pageOutputMap.set(owner, previous) + } + } + /** * Reconcile page ownership only after a successful page phase. Untouched page * and template owners still protect their outputs during targeted builds. diff --git a/lib/build-pages/additional-outputs-types.test.ts b/lib/build-pages/additional-outputs-types.test.ts index 56e62c76..8452dc99 100644 --- a/lib/build-pages/additional-outputs-types.test.ts +++ b/lib/build-pages/additional-outputs-types.test.ts @@ -8,6 +8,7 @@ import type { CollectedAdditionalOutput, PageData, } from '../../types.ts' +import { normalizeAdditionalOutputs } from './additional-outputs.js' // Compile-only assertions for the public type entry and the narrow hook contract. export function checkAdditionalOutputsTypes (pageData: PageData<{ title: string }>, page: AdditionalOutputsPage) { @@ -19,7 +20,11 @@ export function checkAdditionalOutputsTypes (pageData: PageData<{ title: string outputName: 'feed.json', content: vars.title + page.url + data.posts.join(','), }) const params: AdditionalOutputsFunctionParams<{ title: string }, { posts: string[] }> = { page, vars: { title: 'title' }, data: { posts: [] } } - const promise: Promise = pageData.collectAdditionalOutputs() + const outputs: AsyncGenerator = pageData.collectAdditionalOutputs() + const normalized: AsyncGenerator = normalizeAdditionalOutputs(result, provenance) + const iterable: AsyncIterable = outputs + // @ts-expect-error Collection is streamed, not a promise of buffered records. + const buffered: Promise = pageData.collectAdditionalOutputs() const iterator: AdditionalOutputsFunction = async function * () { yield output } const promisedIterator: AdditionalOutputsFunction = async () => (async function * () { yield output })() // @ts-expect-error Bare strings are not hook results. @@ -40,5 +45,5 @@ export function checkAdditionalOutputsTypes (pageData: PageData<{ title: string params.vars.title = 'other' // @ts-expect-error Only declared data is available. const secret = params.data.secret - return { hook, params, promise, collected, result, iterator, promisedIterator, badHook, badOutput, bypass, globalData, secret } + return { hook, params, outputs, normalized, iterable, buffered, collected, result, iterator, promisedIterator, badHook, badOutput, bypass, globalData, secret } } diff --git a/lib/build-pages/additional-outputs.js b/lib/build-pages/additional-outputs.js index a33e351f..4bb89873 100644 --- a/lib/build-pages/additional-outputs.js +++ b/lib/build-pages/additional-outputs.js @@ -45,33 +45,35 @@ export function validateAdditionalOutputsHook (hook, source) { /** * Normalize one provider's result, preserving order and diagnostic context. - * Output destination containment and collisions are enforced by the output writer. + * The writer checks each destination and writes it before requesting another record. * @param {unknown} result * @param {AdditionalOutputProvenance} provenance - * @returns {Promise} + * @returns {AsyncGenerator} */ -export async function normalizeAdditionalOutputs (result, provenance) { - /** @type {CollectedAdditionalOutput[]} */ - const outputs = [] - /** @param {unknown} record */ - const append = (record) => { +export async function * normalizeAdditionalOutputs (result, provenance) { + let recordNumber = 0 + /** + * @param {unknown} record + * @returns {CollectedAdditionalOutput} + */ + const validate = (record) => { + recordNumber++ if (!record || typeof record !== 'object' || !('outputName' in record) || typeof record.outputName !== 'string' || !record.outputName.trim() || !('content' in record) || typeof record.content !== 'string') { - throw new TypeError(`Record ${outputs.length + 1} must be { outputName: non-empty string, content: string }`) + throw new TypeError(`Record ${recordNumber} must be { outputName: non-empty string, content: string }`) } - outputs.push({ outputName: record.outputName, content: record.content, provenance: { ...provenance } }) + return { outputName: record.outputName, content: record.content, provenance: { ...provenance } } } try { const resolved = await result if (Array.isArray(resolved)) { - for (const record of resolved) append(record) + for (const record of resolved) yield validate(record) } else if (resolved && typeof resolved === 'object' && Symbol.asyncIterator in resolved) { - for await (const record of /** @type {AsyncIterable} */ (resolved)) append(record) + for await (const record of /** @type {AsyncIterable} */ (resolved)) yield validate(record) } else { - append(resolved) + yield validate(resolved) } - return outputs } catch (cause) { throw new Error(`Invalid additionalOutputs from ${provenance.kind} "${provenance.source}": ${cause instanceof Error ? cause.message : String(cause)}`, { cause }) } diff --git a/lib/build-pages/additional-outputs.test.js b/lib/build-pages/additional-outputs.test.js index 06b25bf4..d3f83f23 100644 --- a/lib/build-pages/additional-outputs.test.js +++ b/lib/build-pages/additional-outputs.test.js @@ -9,13 +9,13 @@ const record = { outputName: 'feed.json', content: '' } test('additionalOutputs normalizes records, arrays, promises and async iterables', async () => { const expected = [{ ...record, provenance }] - assert.deepEqual(await normalizeAdditionalOutputs(record, provenance), expected) - assert.deepEqual(await normalizeAdditionalOutputs(Promise.resolve([record]), provenance), expected) + assert.deepEqual(await Array.fromAsync(normalizeAdditionalOutputs(record, provenance)), expected) + assert.deepEqual(await Array.fromAsync(normalizeAdditionalOutputs(Promise.resolve([record]), provenance)), expected) async function * records () { yield record; yield { ...record, outputName: 'second.json' } } - assert.deepEqual(await normalizeAdditionalOutputs(Promise.resolve(records()), provenance), [...expected, { ...record, outputName: 'second.json', provenance }]) - assert.deepEqual(await normalizeAdditionalOutputs([], provenance), []) + assert.deepEqual(await Array.fromAsync(normalizeAdditionalOutputs(Promise.resolve(records()), provenance)), [...expected, { ...record, outputName: 'second.json', provenance }]) + assert.deepEqual(await Array.fromAsync(normalizeAdditionalOutputs([], provenance)), []) async function * empty () {} - assert.deepEqual(await normalizeAdditionalOutputs(empty(), provenance), []) + assert.deepEqual(await Array.fromAsync(normalizeAdditionalOutputs(empty(), provenance)), []) }) test('additionalOutputs rejects invalid hooks and records with source context', async () => { @@ -23,9 +23,84 @@ test('additionalOutputs rejects invalid hooks and records with source context', assert.throws(() => validateAdditionalOutputsHook(hook, provenance.source), /additionalOutputs.*\/src\/page.ts.*function/) } for (const result of ['bare string', undefined, null, {}, { outputName: '', content: '' }, { outputName: 'x', content: 1 }, [record, 'bad'], new Set([record])]) { - await assert.rejects(normalizeAdditionalOutputs(result, provenance), /Invalid additionalOutputs.*\/src\/page.ts.*Record/) + await assert.rejects(Array.fromAsync(normalizeAdditionalOutputs(result, provenance)), /Invalid additionalOutputs.*\/src\/page.ts.*Record/) } async function * broken () { yield record; throw new Error('iterator failed') } - await assert.rejects(normalizeAdditionalOutputs(broken(), provenance), /\/src\/page.ts.*iterator failed/) - await assert.rejects(normalizeAdditionalOutputs(Promise.reject(new Error('promise failed')), provenance), /\/src\/page.ts.*promise failed/) + await assert.rejects(Array.fromAsync(normalizeAdditionalOutputs(broken(), provenance)), /\/src\/page.ts.*iterator failed/) + await assert.rejects(Array.fromAsync(normalizeAdditionalOutputs(Promise.reject(new Error('promise failed')), provenance)), /\/src\/page.ts.*promise failed/) +}) + +test('normalization pulls one record at a time and closes the provider on return or break', async () => { + for (const close of ['return', 'break']) { + /** @type {string[]} */ + const events = [] + async function * records () { + try { + events.push('first') + yield record + events.push('second') + yield { ...record, outputName: 'second.json' } + } finally { + events.push('closed') + } + } + const outputs = normalizeAdditionalOutputs(records(), provenance) + assert.equal(outputs[Symbol.asyncIterator](), outputs) + assert.deepEqual(events, []) + if (close === 'return') { + const first = await outputs.next() + assert.deepEqual(first, { value: { ...record, provenance }, done: false }) + assert.notEqual(first.value?.provenance, provenance) + assert.deepEqual(events, ['first']) + assert.deepEqual(await outputs.return(), { value: undefined, done: true }) + } else { + // eslint-disable-next-line no-unreachable-loop -- Exercise iterator cleanup on an early break. + for await (const output of outputs) { + assert.deepEqual(output, { ...record, provenance }) + assert.deepEqual(events, ['first']) + break + } + } + assert.deepEqual(events, ['first', 'closed']) + assert.deepEqual(await outputs.next(), { value: undefined, done: true }) + } +}) + +test('normalization validates each record only when requested with its record number', async () => { + let closed = false + async function * records () { + try { + yield record + yield { outputName: 'invalid.json', content: 1 } + } finally { + closed = true + } + } + for (const result of [[record, { outputName: 'invalid.json', content: 1 }], records()]) { + const outputs = normalizeAdditionalOutputs(result, provenance) + assert.deepEqual(await outputs.next(), { value: { ...record, provenance }, done: false }) + await assert.rejects(outputs.next(), error => { + assert.ok(error instanceof Error) + assert.match(error.message, /Invalid additionalOutputs from page "\/src\/page.ts": Record 2/) + assert.ok(error.cause instanceof TypeError) + return true + }) + } + assert.equal(closed, true) +}) + +test('normalization preserves iterator failure causes after yielding valid records', async () => { + const cause = new Error('iterator failed') + async function * records () { + yield record + throw cause + } + const outputs = normalizeAdditionalOutputs(records(), provenance) + assert.deepEqual(await outputs.next(), { value: { ...record, provenance }, done: false }) + await assert.rejects(outputs.next(), error => { + assert.ok(error instanceof Error) + assert.match(error.message, /\/src\/page.ts.*iterator failed/) + assert.equal(error.cause, cause) + return true + }) }) diff --git a/lib/build-pages/index.js b/lib/build-pages/index.js index 64db0268..b7cb17f8 100644 --- a/lib/build-pages/index.js +++ b/lib/build-pages/index.js @@ -283,25 +283,16 @@ function validateGeneratedPageDefinition (value) { /** * @param {unknown} value - * @returns {Promise} + * @returns {AsyncGenerator} */ -async function collectGeneratedPageDefinitions (value) { - if (value == null) return [] +async function * iterateGeneratedPageDefinitions (value) { + if (value == null) return - if (Array.isArray(value)) { - return value.map(validateGeneratedPageDefinition) + if (Array.isArray(value) || isAsyncIterable(value)) { + for await (const definition of value) yield validateGeneratedPageDefinition(definition) + } else { + yield validateGeneratedPageDefinition(value) } - - if (isAsyncIterable(value)) { - /** @type {GeneratedPageDefinition[]} */ - const definitions = [] - for await (const definition of value) { - definitions.push(validateGeneratedPageDefinition(definition)) - } - return definitions - } - - return [validateGeneratedPageDefinition(value)] } /** @@ -365,11 +356,9 @@ function generatedDefinitionToPageInfo ({ definition, pagesFile, index }) { * @param {Set | null} params.pagesFileFilterSet * @param {boolean | undefined} params.buildDrafts * @param {WatchDependencyTracker} params.watchDependencyTracker - * @returns {Promise} + * @returns {AsyncGenerator} */ -async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, pagesFileFilterSet, buildDrafts, watchDependencyTracker }) { - /** @type {PageInfo[]} */ - const generatedPageInfos = [] +async function * resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, pagesFileFilterSet, buildDrafts, watchDependencyTracker }) { /** @type {Map} */ const pageOutputClaims = new Map() @@ -420,10 +409,9 @@ async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, p }) : pagesExport - const definitions = await collectGeneratedPageDefinitions(pagesResults) - - for (const [index, definition] of definitions.entries()) { - const generatedPageInfo = generatedDefinitionToPageInfo({ definition, pagesFile, index }) + let index = 0 + for await (const definition of iterateGeneratedPageDefinitions(pagesResults)) { + const generatedPageInfo = generatedDefinitionToPageInfo({ definition, pagesFile, index: index++ }) if (generatedPageInfo.draft && !buildDrafts) continue const outputKey = resolve(generatedPageInfo.outputRelname) @@ -444,7 +432,7 @@ async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, p } pageOutputClaims.set(outputKey, generatedClaim) - generatedPageInfos.push(generatedPageInfo) + yield generatedPageInfo } } catch (err) { const error = err instanceof Error @@ -454,8 +442,6 @@ async function resolveGeneratedPageInfos ({ siteData, factoryVars, globalData, p throw error } } - - return generatedPageInfos } /** @@ -651,39 +637,6 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { }) } - let generatedPageInfos = /** @type {PageInfo[]} */ ([]) - try { - generatedPageInfos = await resolveGeneratedPageInfos({ - siteData, - factoryVars: globalVars, - globalData, - pagesFileFilterSet, - buildDrafts: opts?.buildDrafts, - watchDependencyTracker, - }) - } catch (err) { - const pagesFile = /** @type {PagesFileInfo | undefined} */ (err instanceof Error && 'pagesFile' in err ? err.pagesFile : undefined) - result.errors.push(serializeBuildError(err, { pagesFile }, `Error resolving generated pages: ${err instanceof Error ? err.message : String(err)}`)) - } - - if (result.errors.length > 0) return result - - const generatedPages = await pMap(generatedPageInfos, initPageData, { concurrency: MAX_CONCURRENCY }) - if (result.errors.length > 0) return result - - for (const page of generatedPages) { - page.setGlobalData(globalData) - const generatedOwnerPath = page.pageInfo.generated?.pagesFile.pagesFile.filepath - watchDependencyTracker.registerConsumer( - 'page', - page.pageInfo.outputRelname, - page.dataDeps, - generatedOwnerPath ? { ownerPath: generatedOwnerPath } : {} - ) - } - - if (result.errors.length > 0) return result - /** @type {PageData[]} */ const pagesToWrite = [] @@ -693,10 +646,6 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { } } - for (const page of generatedPages) { - pagesToWrite.push(page) - } - /** @type {[number, number]} Divided concurrency values */ const dividedConcurrency = MAX_CONCURRENCY % 2 ? [((MAX_CONCURRENCY - 1) / 2) + 1, (MAX_CONCURRENCY - 1) / 2] // odd @@ -711,27 +660,80 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { : (siteData.pagesFiles ?? []).map(pagesFile => pagesFile.pagesFile.filepath) } - await Promise.all([ - pMap(pagesToWrite, async (page) => { - try { - const buildResult = await pageWriter({ - dest, - page, - }) - + /** @param {PageData} page */ + const writePage = async (page) => { + /** @type {DomstackManifestRecord[]} */ + const emittedOutputs = [] + try { + const buildResult = await pageWriter({ + dest, + page, + onAdditionalOutput: output => emittedOutputs.push(output), + }) + + result.report.pages.push({ + pageFilePath: buildResult.pageFilePath, + sourcePageFilePath: page.pageInfo.generated ? undefined : page.pageInfo.pageFile.filepath, + pagesFilePath: page.pageInfo.generated?.pagesFile.pagesFile.filepath, + layoutName: page.layout?.name, + layoutNames: page.layoutChain.map(layout => layout.name), + outputs: buildResult.outputs, + }) + result.outputs.push(...buildResult.outputs) + return true + } catch (err) { + // Direct writes already emitted by a failed iterator still need ownership + // so a later successful watch rebuild can remove them. + if (emittedOutputs.length > 0) { result.report.pages.push({ - pageFilePath: buildResult.pageFilePath, + pageFilePath: join(dest, page.pageInfo.outputRelname), sourcePageFilePath: page.pageInfo.generated ? undefined : page.pageInfo.pageFile.filepath, pagesFilePath: page.pageInfo.generated?.pagesFile.pagesFile.filepath, layoutName: page.layout?.name, layoutNames: page.layoutChain.map(layout => layout.name), - outputs: buildResult.outputs, + outputs: emittedOutputs, }) - result.outputs.push(...buildResult.outputs) - } catch (err) { - result.errors.push(serializeBuildError(err, { page: pageInfoForWorker(page.pageInfo) }, 'Error building page')) + result.outputs.push(...emittedOutputs) } - }, { concurrency: dividedConcurrency[0] }), + result.errors.push(serializeBuildError(err, { page: pageInfoForWorker(page.pageInfo) }, `Error building page "${page.pageInfo.pageFile.relname}"`)) + return false + } + } + + // Keep output names for dependency pruning, not generated definitions or PageData instances. + const generatedOutputRelnames = new Set() + const writeGeneratedPages = async () => { + try { + for await (const pageInfo of resolveGeneratedPageInfos({ + siteData, + factoryVars: globalVars, + globalData, + pagesFileFilterSet, + buildDrafts: opts?.buildDrafts, + watchDependencyTracker, + })) { + const errorCount = result.errors.length + const page = await initPageData(pageInfo) + if (result.errors.length > errorCount) break + page.setGlobalData(globalData) + watchDependencyTracker.registerConsumer( + 'page', + pageInfo.outputRelname, + page.dataDeps, + { ownerPath: pageInfo.pageFile.filepath } + ) + if (!await writePage(page)) break + generatedOutputRelnames.add(pageInfo.outputRelname) + } + } catch (err) { + const pagesFile = /** @type {PagesFileInfo | undefined} */ (err instanceof Error && 'pagesFile' in err ? err.pagesFile : undefined) + result.errors.push(serializeBuildError(err, { pagesFile }, `Error building generated pages: ${err instanceof Error ? err.message : String(err)}`)) + } + } + + await Promise.all([ + pMap(pagesToWrite, writePage, { concurrency: dividedConcurrency[0] }), + writeGeneratedPages(), pMap(templatesToRender, async (template) => { try { const buildResult = await templateBuilder({ @@ -753,7 +755,7 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { if (opts?.trackWatchDependencies) { result.warnings.push(...outputWarnings(result.outputs)) watchDependencyTracker.pruneGeneratedPages( - new Set(generatedPages.map(page => page.pageInfo.outputRelname)), + generatedOutputRelnames, pagesFileFilterSet ) result.report.watchDependencies = watchDependencyTracker.state diff --git a/lib/build-pages/page-builders/additional-output-writer.js b/lib/build-pages/page-builders/additional-output-writer.js index 74466eca..2a9f263b 100644 --- a/lib/build-pages/page-builders/additional-output-writer.js +++ b/lib/build-pages/page-builders/additional-output-writer.js @@ -1,7 +1,6 @@ /** * @import { PageInfo } from '../../identify-pages.js' * @import { DomstackManifestRecord } from '../../domstack-manifest/index.js' - */ import { lstat, mkdir, readFile, writeFile } from 'node:fs/promises' import { dirname, isAbsolute, relative, resolve, sep } from 'node:path' @@ -80,19 +79,17 @@ async function assertWritablePath (dest, filepath) { * @param {string} params.dest * @param {string} params.pageFilePath * @param {PageInfo} params.pageInfo - * @param {Array<{outputName: string, content: string}>} params.additionalOutputs + * @param {Iterable<{outputName: string, content: string}> | AsyncIterable<{outputName: string, content: string}>} params.additionalOutputs + * @param {((output: DomstackManifestRecord) => void) | undefined} [params.onOutput] * @returns {Promise} */ -export async function writeAdditionalOutputs ({ dest, pageFilePath, pageInfo, additionalOutputs }) { - const planned = additionalOutputs.map(output => { - if (typeof output.content !== 'string') throw new TypeError('Additional output content must be a string') - return { ...resolveAdditionalOutputPath(dest, pageFilePath, output.outputName), content: output.content } - }) - // Check every path before writing this page's batch. - for (const output of planned) await assertWritablePath(dest, output.filepath) +export async function writeAdditionalOutputs ({ dest, pageFilePath, pageInfo, additionalOutputs, onOutput }) { const records = [] - for (const { filepath, outputRelname, content } of planned) { - const bytes = Buffer.from(content) + for await (const output of additionalOutputs) { + if (typeof output.content !== 'string') throw new TypeError('Additional output content must be a string') + const { filepath, outputRelname } = resolveAdditionalOutputPath(dest, pageFilePath, output.outputName) + await assertWritablePath(dest, filepath) + const bytes = Buffer.from(output.content) let unchanged = false try { unchanged = bytes.equals(await readFile(filepath)) @@ -103,7 +100,7 @@ export async function writeAdditionalOutputs ({ dest, pageFilePath, pageInfo, ad await mkdir(dirname(filepath), { recursive: true }) await writeFile(filepath, bytes) } - records.push({ + const record = { ...createDomstackManifestRecord({ dest, filepath, @@ -114,7 +111,9 @@ export async function writeAdditionalOutputs ({ dest, pageFilePath, pageInfo, ad pageUrl: pageInfo.url, }), pagePath: pageInfo.path, - }) + } + records.push(record) + onOutput?.(record) } return records } diff --git a/lib/build-pages/page-builders/additional-output-writer.test.js b/lib/build-pages/page-builders/additional-output-writer.test.js index c1ced5fa..d713b93c 100644 --- a/lib/build-pages/page-builders/additional-output-writer.test.js +++ b/lib/build-pages/page-builders/additional-output-writer.test.js @@ -91,7 +91,7 @@ test('writer rejects symlink components and existing directories without touchin assert.equal(await readFile(join(outside, 'data.json'), 'utf8'), 'unchanged') }) -test('writer validates all sidecar paths before writing the batch', async t => { +test('writer validates each path before writing it, retaining earlier writes', async t => { const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-invalid-')) t.after(() => rm(dest, { recursive: true, force: true })) await assert.rejects(writeAdditionalOutputs({ @@ -100,7 +100,7 @@ test('writer validates all sidecar paths before writing the batch', async t => { pageInfo, additionalOutputs: [{ outputName: 'valid.json', content: '{}' }, { outputName: '../escape.json', content: '{}' }], }), /escapes dest/) - await assert.rejects(readFile(join(dest, 'valid.json')), { code: 'ENOENT' }) + assert.equal(await readFile(join(dest, 'valid.json'), 'utf8'), '{}') }) test('writer preserves duplicate records for duplicate-output warnings', async t => { @@ -128,7 +128,7 @@ test('page writer collects once after rendering and reports HTML and sidecars to pageInfo, vars: {}, async renderFullPage () { events.push('render'); return '

Page

' }, - async collectAdditionalOutputs () { events.push('collect'); return [{ outputName: 'data.json', content: '{}' }] }, + async * collectAdditionalOutputs () { events.push('collect'); yield { outputName: 'data.json', content: '{}' } }, })) const result = await pageWriter({ dest, page }) assert.deepEqual(events, ['render', 'collect']) diff --git a/lib/build-pages/page-builders/page-writer.js b/lib/build-pages/page-builders/page-writer.js index 69121336..279a72b6 100644 --- a/lib/build-pages/page-builders/page-writer.js +++ b/lib/build-pages/page-builders/page-writer.js @@ -99,12 +99,13 @@ import { writeAdditionalOutputs } from './additional-output-writer.js' * @param {object} params * @param {string} params.dest - The dest folder. * @param {PageData} params.page - The PageInfo object of the current page - + * @param {((output: DomstackManifestRecord) => void) | undefined} [params.onAdditionalOutput] - Record emitted sidecars even if a later yield fails. * @returns {Promise<{ pageFilePath: string, outputs: DomstackManifestRecord[] }>} */ export async function pageWriter ({ dest, page, + onAdditionalOutput, }) { if (!page.pageInfo) throw new Error('Uninitialzied page detected') const pageDir = join(dest, page.pageInfo.path) @@ -115,7 +116,8 @@ export async function pageWriter ({ dest, pageFilePath, pageInfo: page.pageInfo, - additionalOutputs: await page.collectAdditionalOutputs(), + additionalOutputs: page.collectAdditionalOutputs(), + onOutput: onAdditionalOutput, }) const vars = page.vars const manifestRole = extractManifestRole(vars) diff --git a/lib/build-pages/page-data-additional-outputs.test.js b/lib/build-pages/page-data-additional-outputs.test.js index c5e145b4..a8a7e26e 100644 --- a/lib/build-pages/page-data-additional-outputs.test.js +++ b/lib/build-pages/page-data-additional-outputs.test.js @@ -74,9 +74,9 @@ test('explicit collection runs outer -> inner -> page with renderer subscription return hook(params) } } - await assert.rejects(pd.collectAdditionalOutputs(), /initialized/) + await assert.rejects(Array.fromAsync(pd.collectAdditionalOutputs()), /initialized/) await pd.init({ layouts }) - await assert.rejects(pd.collectAdditionalOutputs(), /outer.*not available/) + await assert.rejects(Array.fromAsync(pd.collectAdditionalOutputs()), /outer.*not available/) assert.deepEqual(pd.dataDeps, ['companionKey', 'innerKey', 'outerKey', 'pageKey']) pd.setGlobalData({ pageKey: 'p', companionKey: 'c', outerKey: 'o', innerKey: 'i', secret: 'hidden' }) await pd.renderInnerPage() @@ -84,7 +84,7 @@ test('explicit collection runs outer -> inner -> page with renderer subscription assert.deepEqual(calls, []) const pageModule = await import(pd.pageInfo.pageFile.filepath) assert.equal(pageModule.hookCalls, 0) - const outputs = await pd.collectAdditionalOutputs() + const outputs = await Array.fromAsync(pd.collectAdditionalOutputs()) assert.equal(pageModule.hookCalls, 1) assert.deepEqual(calls, ['outer', 'inner']) assert.deepEqual(outputs.map(({ outputName, content }) => ({ outputName, content })), [ @@ -94,6 +94,136 @@ test('explicit collection runs outer -> inner -> page with renderer subscription assert.deepEqual(outputs[2]?.provenance, { kind: 'page', source: join(dir, 'page.mjs') }) }) +test('collection invokes providers lazily in outer -> inner -> page order', async t => { + const { pd, layouts } = await fixture(t, { + module: `export default () => '' + export let hookCalls = 0 + export function additionalOutputs () { + hookCalls++ + return [{ outputName: 'page.txt', content: 'page' }] + }`, + }) + /** @type {string[]} */ + const events = [] + for (const name of ['outer', 'inner']) { + const layout = layouts[name] + assert.ok(layout) + layout.additionalOutputs = () => { + events.push(`${name}:called`) + return (async function * () { + try { + for (const index of [1, 2]) { + events.push(`${name}:${index}`) + yield { outputName: `${name}-${index}.txt`, content: `${index}` } + } + } finally { + events.push(`${name}:closed`) + } + })() + } + } + await pd.init({ layouts }) + const pageModule = await import(pd.pageInfo.pageFile.filepath) + const outputs = pd.collectAdditionalOutputs() + assert.equal(outputs[Symbol.asyncIterator](), outputs) + assert.deepEqual(events, []) + assert.equal(pageModule.hookCalls, 0) + assert.equal((await outputs.next()).value?.outputName, 'outer-1.txt') + assert.deepEqual(events, ['outer:called', 'outer:1']) + assert.equal((await outputs.next()).value?.outputName, 'outer-2.txt') + assert.deepEqual(events, ['outer:called', 'outer:1', 'outer:2']) + assert.equal(pageModule.hookCalls, 0) + assert.equal((await outputs.next()).value?.outputName, 'inner-1.txt') + assert.deepEqual(events, ['outer:called', 'outer:1', 'outer:2', 'outer:closed', 'inner:called', 'inner:1']) + assert.equal((await outputs.next()).value?.outputName, 'inner-2.txt') + assert.equal(pageModule.hookCalls, 0) + assert.equal((await outputs.next()).value?.outputName, 'page.txt') + assert.deepEqual(events, ['outer:called', 'outer:1', 'outer:2', 'outer:closed', 'inner:called', 'inner:1', 'inner:2', 'inner:closed']) + assert.equal(pageModule.hookCalls, 1) + assert.deepEqual(await outputs.next(), { value: undefined, done: true }) +}) + +test('closing collection runs the active provider finally and never invokes later providers', async t => { + for (const close of ['return', 'break']) { + const { pd, layouts } = await fixture(t, { + module: `export default () => '' + export let hookCalls = 0 + export function additionalOutputs () { hookCalls++; return [] }`, + }) + /** @type {string[]} */ + const events = [] + const { outer, inner } = layouts + assert.ok(outer) + assert.ok(inner) + outer.additionalOutputs = () => { + events.push('outer:called') + return (async function * () { + try { + yield { outputName: 'first.txt', content: 'first' } + events.push('outer:second') + yield { outputName: 'second.txt', content: 'second' } + } finally { + await Promise.resolve() + events.push('outer:closed') + } + })() + } + inner.vars = { dataDeps: ['unready'] } + inner.additionalOutputs = () => { events.push('inner:called'); return [] } + await pd.init({ layouts }) + const pageModule = await import(pd.pageInfo.pageFile.filepath) + const unopened = pd.collectAdditionalOutputs() + await unopened.return() + assert.deepEqual(events, []) + const outputs = pd.collectAdditionalOutputs() + if (close === 'return') { + assert.equal((await outputs.next()).value?.outputName, 'first.txt') + assert.deepEqual(events, ['outer:called']) + assert.deepEqual(await outputs.return(), { value: undefined, done: true }) + } else { + // eslint-disable-next-line no-unreachable-loop -- Exercise iterator cleanup on an early break. + for await (const output of outputs) { + assert.equal(output.outputName, 'first.txt') + assert.deepEqual(events, ['outer:called']) + break + } + } + assert.deepEqual(events, ['outer:called', 'outer:closed']) + assert.equal(pageModule.hookCalls, 0) + assert.deepEqual(await outputs.next(), { value: undefined, done: true }) + } +}) + +test('streamed provider errors retain page context and causes after earlier records', async t => { + const { pd, layouts } = await fixture(t, { + module: "export default () => ''; export const additionalOutputs = () => { throw new Error('page must not run') }", + }) + const cause = new Error('iterator failed') + let closed = false + const { outer } = layouts + assert.ok(outer) + outer.additionalOutputs = async function * () { + try { + yield { outputName: 'first.txt', content: 'first' } + throw cause + } finally { + closed = true + } + } + await pd.init({ layouts }) + const outputs = pd.collectAdditionalOutputs() + assert.equal((await outputs.next()).value?.outputName, 'first.txt') + assert.equal(closed, false) + await assert.rejects(outputs.next(), error => { + assert.ok(error instanceof Error) + assert.match(error.message, /additionalOutputs for page "page.mjs" from layout "outer" failed: Invalid additionalOutputs.*iterator failed/) + assert.ok(error.cause instanceof Error) + assert.equal(error.cause.cause, cause) + return true + }) + assert.equal(closed, true) +}) + test('markdown companion gets a frozen narrow source handle and subscribed data', async t => { const { pd, layouts, dir } = await fixture(t, { companion: ` @@ -112,9 +242,9 @@ test('markdown companion gets a frozen narrow source handle and subscribed data' }` }) await pd.init({ layouts }) - await assert.rejects(pd.collectAdditionalOutputs(), /companion.*not available/) + await assert.rejects(Array.fromAsync(pd.collectAdditionalOutputs()), /companion.*not available/) pd.setGlobalData({ selected: 'selected:', secret: 'hidden' }) - assert.deepEqual(await pd.collectAdditionalOutputs(), [{ outputName: 'body.md', content: 'selected:\n# Body\n', provenance: { kind: 'companion', source: join(dir, 'page.vars.mjs') } }]) + assert.deepEqual(await Array.fromAsync(pd.collectAdditionalOutputs()), [{ outputName: 'body.md', content: 'selected:\n# Body\n', provenance: { kind: 'companion', source: join(dir, 'page.vars.mjs') } }]) }) test('JS and TS page providers work and cannot read markdown or undeclared data', async t => { @@ -131,7 +261,7 @@ test('JS and TS page providers work and cannot read markdown or undeclared data' }) await pd.init({ layouts }) pd.setGlobalData({ secret: 'hidden' }) - assert.deepEqual(await pd.collectAdditionalOutputs(), []) + assert.deepEqual(await Array.fromAsync(pd.collectAdditionalOutputs()), []) } }) @@ -148,7 +278,7 @@ test('companions also provide outputs for JS and HTML pages', async t => { } await pd.init({ layouts }) pd.setGlobalData({ selected: type }) - const outputs = await pd.collectAdditionalOutputs() + const outputs = await Array.fromAsync(pd.collectAdditionalOutputs()) assert.equal(outputs[0]?.content, type) assert.equal(outputs[0]?.provenance.kind, 'companion') } @@ -175,7 +305,7 @@ test('generated pages skip page, companion and all layout hooks', async t => { layout.additionalOutputs = () => { throw new Error('generated hook ran') } } await pd.init({ layouts }) - assert.deepEqual(await pd.collectAdditionalOutputs(), []) + assert.deepEqual(await pd.collectAdditionalOutputs().next(), { value: undefined, done: true }) }) test('discovery excludes ignored providers and disabled drafts but enabled drafts collect outputs', async t => { @@ -195,16 +325,16 @@ test('discovery excludes ignored providers and disabled drafts but enabled draft pd.pageInfo = pageInfo await pd.init({ layouts }) pd.setGlobalData({}) - assert.equal((await pd.collectAdditionalOutputs())[0]?.content, 'draft output') + assert.equal((await Array.fromAsync(pd.collectAdditionalOutputs()))[0]?.content, 'draft output') }) test('page hook failures include page and provider context, and no providers is valid', async t => { for (const body of ["throw new Error('hook failed')", "return 'bare string'", "return (async function * () { throw new Error('iterator failed') })()"]) { const { pd, layouts } = await fixture(t, { companion: `export function additionalOutputs () { ${body} }` }) await pd.init({ layouts }) - await assert.rejects(pd.collectAdditionalOutputs(), /additionalOutputs for page "page.md" from companion.*page.vars.mjs.*(failed|Record)/) + await assert.rejects(Array.fromAsync(pd.collectAdditionalOutputs()), /additionalOutputs for page "page.md" from companion.*page.vars.mjs.*(failed|Record)/) } const { pd, layouts } = await fixture(t) await pd.init({ layouts }) - assert.deepEqual(await pd.collectAdditionalOutputs(), []) + assert.deepEqual(await Array.fromAsync(pd.collectAdditionalOutputs()), []) }) diff --git a/lib/build-pages/page-data.js b/lib/build-pages/page-data.js index 1f1956e3..cf98d73c 100644 --- a/lib/build-pages/page-data.js +++ b/lib/build-pages/page-data.js @@ -394,38 +394,38 @@ export class PageData { } /** - * Run hooks only when explicitly requested by the output phase. + * Run hooks lazily as the output phase consumes each record. * Layouts run outermost first, followed by the single page-level provider. - * @returns {Promise} + * @returns {AsyncGenerator} */ - async collectAdditionalOutputs () { + async * collectAdditionalOutputs () { if (!this.#initialized) throw new Error('Must be initialized before collecting additionalOutputs') - if (this.pageInfo.generated) return [] - /** @type {CollectedAdditionalOutput[]} */ - const outputs = [] + if (this.pageInfo.generated) return // Capture source metadata so rebinding the reader cannot change its source. const sourceInfo = { ...this.pageInfo, pageFile: { ...this.pageInfo.pageFile } } const page = createAdditionalOutputsPage(sourceInfo, async () => { if (sourceInfo.type !== 'md') throw new Error('Markdown content can only be read from markdown pages') return parseMdFileContents(await readFile(sourceInfo.pageFile.filepath, 'utf8')).markdownContent }) + const pageData = this /** * @param {AdditionalOutputsFunction} hook * @param {AdditionalOutputProvenance} provenance * @param {() => object} getData + * @returns {AsyncGenerator} */ - const collect = async (hook, provenance, getData) => { + const collect = async function * (hook, provenance, getData) { try { - outputs.push(...await normalizeAdditionalOutputs(hook({ page, vars: this.vars, data: getData() }), provenance)) + yield * normalizeAdditionalOutputs(hook({ page, vars: pageData.vars, data: getData() }), provenance) } catch (cause) { - throw new Error(`additionalOutputs for page "${this.pageInfo.pageFile.relname}" from ${provenance.kind} "${provenance.source}" failed: ${cause instanceof Error ? cause.message : String(cause)}`, { cause }) + throw new Error(`additionalOutputs for page "${pageData.pageInfo.pageFile.relname}" from ${provenance.kind} "${provenance.source}" failed: ${cause instanceof Error ? cause.message : String(cause)}`, { cause }) } } for (const layout of this.layoutChain) { const source = layout.source ?? layout.name const hook = validateAdditionalOutputsHook(layout.additionalOutputs, source) if (!hook) continue - await collect(hook, { kind: 'layout', source, layoutName: layout.name }, () => { + yield * collect(hook, { kind: 'layout', source, layoutName: layout.name }, () => { const subscription = this.#layoutSubscriptions.get(layout.name) if (!this.#dataReady && subscription?.keys.length) throw this.#dataNotReadyError() return subscription?.data ?? Object.freeze({}) @@ -433,9 +433,8 @@ export class PageData { } if (this.#pageAdditionalOutputs) { const { hook, provenance } = this.#pageAdditionalOutputs - await collect(hook, provenance, () => this.data) + yield * collect(hook, provenance, () => this.data) } - return outputs } /** diff --git a/test-cases/generated-pages/index.test.js b/test-cases/generated-pages/index.test.js index 009246ce..270b293b 100644 --- a/test-cases/generated-pages/index.test.js +++ b/test-cases/generated-pages/index.test.js @@ -529,7 +529,7 @@ export default function indexesPages ({ data }) { await withTempFixture({ 'root.layout.js': minimalRootLayout, 'global.vars.js': minimalGlobalVars, - 'invalid.pages.js': 'export default [{ outputName: "valid/index.html" }, 42]\n', + 'invalid.pages.js': 'export default [{ outputName: "valid/index.html", children: "Published before validation failure" }, 42]\n', }, async ({ src, dest }) => { const domstack = new DomStack(src, dest) await assert.rejects( @@ -544,6 +544,7 @@ export default function indexesPages ({ data }) { return true } ) + assert.match(await readFile(join(dest, 'valid/index.html'), 'utf8'), /Published before validation failure/) }) }) diff --git a/test-cases/generated-pages/streaming.test.js b/test-cases/generated-pages/streaming.test.js new file mode 100644 index 00000000..5a99a7c7 --- /dev/null +++ b/test-cases/generated-pages/streaming.test.js @@ -0,0 +1,323 @@ +/** + * @import { TestContext } from 'node:test' + * @import { Results } from '../../lib/builder.js' + */ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import pino from 'pino' +import { DomStack } from '../../index.js' +import { builder } from '../../lib/builder.js' +import { DomStackAggregateError } from '../../lib/helpers/domstack-aggregate-error.js' +import { errorText, settle, writeFiles } from '../page-additional-outputs/helpers.js' + +/** @param {TestContext} t @param {Record} files @param {boolean} [buildDrafts] */ +async function setup (t, files, buildDrafts = false) { + // index.test.js sweeps .tmp-* directories; these fixtures must survive parallel test files. + const root = await mkdtemp(join(import.meta.dirname, '.streaming-')) + const src = join(root, 'src') + const dest = join(root, 'custom-output') + const logs = /** @type {string[]} */ ([]) + const options = { static: true, domstackManifest: false, buildDrafts, logger: pino({ level: 'debug' }, { write: line => logs.push(line) }) } + const site = new DomStack(src, dest, options) + t.after(async () => { + if (site.watching) await site.stopWatching() + await rm(root, { recursive: true, force: true }) + }) + await writeFiles(src, { + 'global.vars.js': `export default { layout: 'root', title: 'Global', testRoot: ${JSON.stringify(root)}, testDest: ${JSON.stringify(dest)} }`, + 'root.layout.js': "export default ({ children }) => '
' + children + '
'", + ...files, + }) + return { + src, + dest, + root, + site, + logs, + build: () => builder(src, dest, options), + /** @param {string} name */ + read: name => readFile(join(dest, name), 'utf8'), + } +} + +/** + * @param {Results['pageBuildResults']} result + * @param {string} src + * @param {string} dest + * @param {string} outputRelname + * @param {string} owner + * @param {number} index + */ +function assertReported (result, src, dest, outputRelname, owner, index) { + assert.ok(result, 'page build results survive worker transport') + const output = result.outputs.find(output => output.outputRelname === outputRelname) + assert.ok(output, `${outputRelname} retains output metadata`) + assert.equal(output.kind, 'page') + assert.equal(output.filepath, join(dest, outputRelname)) + assert.equal(output.sourceRelname, `${owner}#${index}`) + const report = result.report.pages.find(page => page.outputs.some(output => output.outputRelname === outputRelname)) + assert.ok(report, `${outputRelname} retains an ownership report`) + assert.equal(report.pagesFilePath, join(src, owner)) + assert.equal(report.sourcePageFilePath, undefined) + assert.equal(report.pageFilePath, join(dest, outputRelname)) + assert.deepEqual(report.outputs.find(record => record.outputRelname === outputRelname), output) +} + +for (const [form, factoryExport] of Object.entries({ + 'generator function': 'export default pages', + 'static async iterable': 'export default pages()', + 'async function returning an iterable': 'export default async () => pages()', +})) { + test(`${form} renders and writes each page before requesting the next definition`, async t => { + const { build, src, dest, read } = await setup(t, { + 'concrete/page.js': 'export default ({ vars }) => vars.title', + 'concrete/page.vars.js': "export default async () => ({ title: 'Initialized concrete' })", + 'global.data.js': `import assert from 'node:assert/strict' + export default async ({ pages }) => { + assert.equal(pages.length, 1) + assert.equal(pages[0].pageInfo.generated, undefined) + assert.equal(pages[0].vars.title, 'Initialized concrete') + assert.equal(await pages[0].renderInnerPage(), 'Initialized concrete') + return { collection: pages.map(page => page.vars.title), pageValue: 'Page data', layoutValue: 'Layout data' } + }`, + 'generated.layout.js': `import assert from 'node:assert/strict' + export const vars = { title: 'Layout', layoutOnly: 'Resolved layout', dataDeps: ['layoutValue'] } + export default ({ vars, children, data }) => { + assert.throws(() => data.pageValue, /undeclared/) + return '
' + vars.title + ':' + vars.layoutOnly + ':' + data.layoutValue + ':' + children + '
' + } + export const additionalOutputs = () => { throw Error('generated layout output hook must be skipped') }`, + 'stream.pages.js': `import assert from 'node:assert/strict' + import { readFile } from 'node:fs/promises' + import { join } from 'node:path' + import globals from './global.vars.js' + export const dataDeps = ['collection'] + export const additionalOutputs = () => { throw Error('pages-file output hook must be skipped') } + async function* pages (context) { + if (context) { + assert.deepEqual(context.data.collection, ['Initialized concrete']) + assert.equal(context.pagesFile.pagesFile.relname, 'stream.pages.js') + assert.throws(() => context.data.pageValue, /undeclared/) + } + for (const name of ['first', 'second']) { + yield { + outputName: name + '/index.html', + vars: { layout: 'generated', title: name, dataDeps: ['pageValue'] }, + children: async ({ vars, data }) => { + assert.equal(vars.layoutOnly, 'Resolved layout') + assert.throws(() => data.layoutValue, /undeclared/) + return data.pageValue + }, + } + assert.equal(await readFile(join(globals.testDest, name, 'index.html'), 'utf8'), + '
' + name + ':Resolved layout:Layout data:Page data
') + } + } + ${factoryExport}`, + }) + await writeFiles(dest, { 'first/index.html': 'stale HTML must be replaced before the next pull' }) + const result = await build().catch(error => { + t.diagnostic(errorText(error)) + throw error + }) + for (const [index, name] of ['first', 'second'].entries()) { + assert.equal(await read(`${name}/index.html`), `
${name}:Resolved layout:Layout data:Page data
`) + assertReported(result.pageBuildResults, src, dest, `${name}/index.html`, 'stream.pages.js', index) + } + assert.equal(await read('concrete/index.html'), '
Initialized concrete
') + assert.equal(result.pageBuildResults?.outputs.length, 3, 'generated hooks produce no extra files') + }) +} + +test('single, array and function exports preserve defaults, empty children and nullish results', async t => { + const { build, read } = await setup(t, { + 'nested/single.pages.js': 'export default {}', + 'array.pages.js': `export default [ + { children: undefined, vars: undefined, outputName: undefined }, + { outputName: 'null-child.html', children: null }, + { outputName: 'inline.html', children: async () => 'Inline' }, + ]`, + 'sync.pages.js': 'export default ({ vars }) => ({ children: vars.title })', + 'async.pages.js': "export default async () => [{ outputName: 'async.html', children: 'Async' }]", + 'null.pages.js': 'export default null', + 'undefined.pages.js': 'export default undefined', + 'null-function.pages.js': 'export default () => null', + 'undefined-function.pages.js': 'export default async () => undefined', + 'empty.pages.js': 'export default []', + 'empty-iterator.pages.js': 'export default async function* () {}', + }) + const result = await build() + const expected = { + 'nested/single/index.html': '', + 'array/index.html': '', + 'null-child.html': '', + 'inline.html': 'Inline', + 'sync/index.html': 'Global', + 'async.html': 'Async', + } + assert.deepEqual(result.pageBuildResults?.outputs.map(output => output.outputRelname).sort(), Object.keys(expected).sort()) + for (const [name, content] of Object.entries(expected)) assert.equal(await read(name), `
${content}
`) +}) + +for (const buildDrafts of [false, true]) { + test(`draft yields retain their source indices with buildDrafts=${buildDrafts}`, async t => { + const { build, src, dest, read } = await setup(t, { + 'drafts.pages.js': `export default async function* () { + yield { outputName: 'draft.html', draft: true, children: 'Draft' } + yield { outputName: 'published.html', children: 'Published' } + yield { outputName: 'another-draft.html', draft: true, children: 'Another draft' } + yield { outputName: 'last.html', children: 'Last' } + }`, + }, buildDrafts) + const result = await build() + assertReported(result.pageBuildResults, src, dest, 'published.html', 'drafts.pages.js', 1) + assertReported(result.pageBuildResults, src, dest, 'last.html', 'drafts.pages.js', 3) + assert.equal(result.pageBuildResults?.outputs.length, buildDrafts ? 4 : 2) + for (const [name, index, content] of /** @type {[string, number, string][]} */ ([['draft.html', 0, 'Draft'], ['another-draft.html', 2, 'Another draft']])) { + if (buildDrafts) { + assertReported(result.pageBuildResults, src, dest, name, 'drafts.pages.js', index) + assert.equal(await read(name), `
${content}
`) + } else { + await assert.rejects(stat(join(dest, name)), { code: 'ENOENT' }) + } + } + }) +} + +for (const scenario of [ + { name: 'invalid definition', operation: 'yield 42', message: /Generated page definition must be an object/ }, + { name: 'null yielded definition', operation: 'yield null', message: /Generated page definition must be an object/ }, + { name: 'invalid path', operation: "yield { outputName: '../escape.html' }", message: /must not contain "\.\." segments/ }, + { name: 'collision', operation: "yield { outputName: 'first.html', children: 'Must not overwrite' }", message: /Output path conflict/ }, + { name: 'page render', operation: "yield { outputName: 'broken.html', children: () => { throw Error('stream render failed') } }", message: /stream render failed/ }, + { name: 'layout render', operation: "yield { outputName: 'broken.html', vars: { layout: 'broken' } }", message: /stream layout failed/ }, + { name: 'vars initialization', operation: "yield { outputName: 'broken.html', vars: { dataDeps: false } }", message: /dataDeps/ }, + { name: 'data binding', operation: "yield { outputName: 'broken.html', vars: { dataDeps: ['missing'] } }", message: /missing/ }, + { name: 'factory', operation: "throw Error('stream factory failed')", message: /stream factory failed/ }, +]) { + test(`${scenario.name} failure closes the generator without pulling later pages and retains earlier reports`, async t => { + const { build, src, dest, root, read } = await setup(t, { + 'broken.layout.js': "export default () => { throw Error('stream layout failed') }", + 'stream.pages.js': String.raw`import { appendFile } from 'node:fs/promises' + import { join } from 'node:path' + export default async function* ({ vars }) { + const trace = join(vars.testRoot, 'trace.txt') + try { + await appendFile(trace, 'first\n') + yield { outputName: 'first.html', vars: { title: 'Published' }, children: 'Published' } + await appendFile(trace, 'bad\n') + ${scenario.operation} + await appendFile(trace, 'later\n') + yield { outputName: 'later.html', children: 'Must not render' } + } finally { + await appendFile(trace, 'closed\n') + } + }`, + }) + await writeFiles(dest, { 'broken.html': 'previous HTML' }) + await assert.rejects(build(), error => { + assert.ok(error instanceof DomStackAggregateError) + assert.match(errorText(error), scenario.message) + assert.match(errorText(error), /stream\.pages\.js/) + if (scenario.name === 'collision') { + assert.deepEqual(error.errors[0].conflict, { + outputPath: 'first.html', + a: { type: 'page', path: 'stream.pages.js#0' }, + b: { type: 'page', path: 'stream.pages.js#1' }, + }) + } + const results = /** @type {Results} */ (error.results) + assertReported(results.pageBuildResults, src, dest, 'first.html', 'stream.pages.js', 0) + assert.equal(results.pageBuildResults?.outputs.length, 1) + assert.equal(results.pageBuildResults?.outputs[0]?.pageVars?.['title'], 'Published') + return true + }) + assert.equal(await readFile(join(root, 'trace.txt'), 'utf8'), 'first\nbad\nclosed\n', 'finally is awaited and no following yield is requested') + assert.equal(await read('first.html'), '
Published
') + assert.equal(await read('broken.html'), 'previous HTML') + await assert.rejects(stat(join(dest, 'later.html')), { code: 'ENOENT' }) + await assert.rejects(stat(join(root, 'escape.html')), { code: 'ENOENT' }) + }) +} + +test('sibling factories publish unique outputs with independent owner metadata', async t => { + const { build, src, dest, read } = await setup(t, Object.fromEntries(['a', 'b'].map(name => [ + `${name}.pages.js`, + `export default async function* () { + yield { outputName: '${name}/one.html', children: '${name} one' } + yield { outputName: '${name}/two.html', children: '${name} two' } + }`, + ]))) + const result = await build() + assert.equal(result.pageBuildResults?.outputs.length, 4) + for (const owner of ['a', 'b']) { + for (const [index, name] of ['one', 'two'].entries()) { + const output = `${owner}/${name}.html` + assert.equal(await read(output), `
${owner} ${name}
`) + assertReported(result.pageBuildResults, src, dest, output, `${owner}.pages.js`, index) + } + } +}) + +/** @param {string[]} names @param {string} [failure] */ +function watchFactory (names, failure) { + return `export default async function* () { + ${names.map(name => `yield { outputName: '${name}.html', children: '${name}' }`).join('\n')} + ${failure ? `throw Error('${failure}')` : ''} + }` +} + +for (const change of ['recovery', 'deletion', 'empty result']) { + test(`watch ${change} cleans successful and repeated partial factory ownership`, { timeout: 30_000 }, async t => { + const { site, src, dest, read, logs } = await setup(t, { + 'stream.pages.js': watchFactory(['old', 'stale']), + 'sibling.pages.js': watchFactory(['sibling']), + }) + await site.watch({ serve: false }) + const sibling = await read('sibling.html') + for (const name of ['partial', 'second-partial']) { + await settle(site, logs, async () => { + await writeFile(join(src, 'stream.pages.js'), watchFactory([name], `${name} failure`)) + }, `${name} failure`) + assert.equal(await read(`${name}.html`), `
${name}
`) + assert.equal(await read('old.html'), '
old
') + assert.equal(await read('stale.html'), '
stale
') + assert.equal(await read('partial.html'), '
partial
', 'repeated failure keeps earlier partial ownership') + } + await settle(site, logs, async () => { + if (change === 'deletion') await rm(join(src, 'stream.pages.js')) + else await writeFile(join(src, 'stream.pages.js'), change === 'empty result' ? 'export default null' : watchFactory(['recovered'])) + }) + for (const name of ['old', 'stale', 'partial', 'second-partial']) { + await assert.rejects(stat(join(dest, `${name}.html`)), { code: 'ENOENT' }) + } + if (change === 'recovery') assert.equal(await read('recovered.html'), '
recovered
') + assert.equal(await read('sibling.html'), sibling, 'cleanup preserves sibling factory output') + await assert.rejects(stat(join(dest, 'domstack-manifest.json')), { code: 'ENOENT' }) + }) +} + +for (const change of ['recovery', 'deletion', 'empty result']) { + test(`initial failed watch retains partial generated page reports for ${change}`, { timeout: 30_000 }, async t => { + const { site, src, dest, read, logs } = await setup(t, { + 'stream.pages.js': watchFactory(['partial', 'nested/partial'], 'initial stream failure'), + }) + const result = await site.watch({ serve: false }) + assert.ok(logs.some(line => JSON.parse(line).msg === 'Build Failed!')) + assert.ok(logs.some(line => line.includes('initial stream failure'))) + for (const [index, name] of ['partial', 'nested/partial'].entries()) { + assert.equal(await read(`${name}.html`), `
${name}
`) + assertReported(result.pageBuildResults, src, dest, `${name}.html`, 'stream.pages.js', index) + } + await settle(site, logs, async () => { + if (change === 'deletion') await rm(join(src, 'stream.pages.js')) + else await writeFile(join(src, 'stream.pages.js'), change === 'empty result' ? 'export default []' : watchFactory(['recovered'])) + }) + for (const name of ['partial', 'nested/partial']) { + await assert.rejects(stat(join(dest, `${name}.html`)), { code: 'ENOENT' }) + } + if (change === 'recovery') assert.equal(await read('recovered.html'), '
recovered
') + }) +} diff --git a/test-cases/page-additional-outputs/index.test.js b/test-cases/page-additional-outputs/index.test.js index cb5fdcdb..334278a1 100644 --- a/test-cases/page-additional-outputs/index.test.js +++ b/test-cases/page-additional-outputs/index.test.js @@ -39,22 +39,30 @@ test('nested hooks run outer -> inner -> companion with isolated renderer data a return { outputName: 'outer.txt', content: data.outer } }`, 'inner.layout.js': `import assert from 'node:assert/strict' + import { readFile } from 'node:fs/promises' + import { dirname, join } from 'node:path' export const parentLayout = 'root' export const vars = { dataDeps: ['inner'] } export default ({ children, data }) => data.inner + children export async function* additionalOutputs ({ page, data }) { assert.throws(() => data.outer, /undeclared/) + assert.equal(await readFile(join(dirname(page.pageFile.filepath), '../../custom-output/docs/outer.txt'), 'utf8'), 'O') globalThis[page.pageFile.filepath].push('inner') yield { outputName: './inner.txt', content: data.inner } }`, 'docs/page.md': '---\ntitle: page title\n---\n# Body\n', 'docs/page.vars.js': `import assert from 'node:assert/strict' + import { readFile } from 'node:fs/promises' + import { dirname, join } from 'node:path' export default { dataDeps: ['selected'] } export const additionalOutputs = async ({ page, vars, data }) => { assert.throws(() => data.secret, /undeclared/) assert.throws(() => data.inner, /undeclared/) assert.equal(Object.isFrozen(page), true) assert.equal(Object.isFrozen(vars), true) + const outputDir = join(dirname(page.pageFile.filepath), '../../custom-output/docs') + assert.equal(await readFile(join(outputDir, 'outer.txt'), 'utf8'), 'O') + assert.equal(await readFile(join(outputDir, 'inner.txt'), 'utf8'), 'I') const order = globalThis[page.pageFile.filepath] delete globalThis[page.pageFile.filepath] return [ @@ -106,6 +114,36 @@ test('JS page modules support promised async iterables, arrays, and empty result assert.equal(await read('iterator/index.html'), 'empty iterator') }) +test('async generators publish each record before requesting the next at a custom destination', async t => { + const { build, dest, read, mtime } = await setup(t, { + 'page.js': `import assert from 'node:assert/strict' + import { readFile, stat } from 'node:fs/promises' + import { dirname, join } from 'node:path' + export default () => 'main' + export async function* additionalOutputs ({ page }) { + const dest = join(dirname(page.pageFile.filepath), '../custom-output') + yield { outputName: 'replaced.txt', content: 'replacement' } + assert.equal(await readFile(join(dest, 'replaced.txt'), 'utf8'), 'replacement') + yield { outputName: 'nested/new.txt', content: 'new sidecar' } + assert.equal(await readFile(join(dest, 'nested/new.txt'), 'utf8'), 'new sidecar') + const unchangedTime = (await stat(join(dest, 'unchanged.txt'))).mtimeMs + yield { outputName: 'unchanged.txt', content: 'same bytes' } + assert.equal(await readFile(join(dest, 'unchanged.txt'), 'utf8'), 'same bytes') + assert.equal((await stat(join(dest, 'unchanged.txt'))).mtimeMs, unchangedTime) + }`, + }) + await writeFiles(dest, { 'replaced.txt': 'old sidecar', 'unchanged.txt': 'same bytes' }) + const unchangedTime = await mtime('unchanged.txt') + const result = await build() + assert.equal(await read('replaced.txt'), 'replacement') + assert.equal(await read('nested/new.txt'), 'new sidecar') + assert.equal(await mtime('unchanged.txt'), unchangedTime) + assert.equal(await read('index.html'), 'main') + for (const outputRelname of ['replaced.txt', 'nested/new.txt', 'unchanged.txt']) { + assert.ok(result.pageBuildResults?.outputs.some(output => output.outputRelname === outputRelname), `${outputRelname} is reported, including unchanged content`) + } +}) + test('generated pages skip inherited layout hooks', async t => { const { build, read } = await setup(t, { 'root.layout.js': "export default ({ children }) => children; export const additionalOutputs = () => { throw Error('generated hook ran') }", @@ -170,12 +208,12 @@ for (const result of [ }) } -test('iterator failure after a yield leaves the owning page unchanged', async t => { +test('iterator failure retains earlier sidecar writes and the previous HTML', async t => { const { build, dest, read } = await setup(t, { 'a/page.js': "export default () => 'new sibling'; " + hook('sibling.txt', 'new sibling sidecar'), 'z/page.js': `export default () => 'new main'; export async function* additionalOutputs () { yield { outputName: 'old.txt', content: 'replacement' } - yield { outputName: 'partial.txt', content: 'must not publish' } + yield { outputName: 'partial.txt', content: 'published before failure' } throw Error('iterator exploded') }`, }) @@ -185,10 +223,68 @@ test('iterator failure after a yield leaves the owning page unchanged', async t assert.match(errorText(error), /iterator exploded/) return true }) - for (const [name, content] of Object.entries(previous)) assert.equal(await read(name), content) - await assert.rejects(stat(join(dest, 'z/partial.txt')), { code: 'ENOENT' }) + assert.equal(await read('z/index.html'), 'old main') + assert.equal(await read('z/old.txt'), 'replacement') + assert.equal(await read('z/partial.txt'), 'published before failure') + assert.equal(await read('z/stale.txt'), 'retain on failure') }) +for (const provider of ['layout', 'page']) { + test(`a later ${provider} provider failure retains earlier layout files`, async t => { + const failingHook = 'export const additionalOutputs = () => { throw Error(\'later provider exploded\') }' + const { build, dest, read } = await setup(t, { + 'global.vars.js': "export default { layout: 'inner' }", + 'root.layout.js': `export default ({ children }) => children + export async function* additionalOutputs () { + yield { outputName: 'old.txt', content: 'replacement' } + yield { outputName: 'partial.txt', content: 'partial' } + }`, + 'inner.layout.js': `export const parentLayout = 'root'; export default ({ children }) => children; + ${provider === 'layout' ? failingHook : 'export const additionalOutputs = () => []'}`, + 'page.js': `export default () => 'new main'; + ${provider === 'page' ? failingHook : "export const additionalOutputs = () => { throw Error('page provider must not run') }"}`, + }) + await writeFiles(dest, { 'index.html': 'old main', 'old.txt': 'old sidecar' }) + await assert.rejects(build(), error => { + assert.match(errorText(error), /later provider exploded/) + assert.doesNotMatch(errorText(error), /page provider must not run/) + return true + }) + assert.equal(await read('index.html'), 'old main') + assert.equal(await read('old.txt'), 'replacement') + assert.equal(await read('partial.txt'), 'partial') + }) +} + +for (const invalid of [ + { record: "{ outputName: '../escape.txt', content: 'invalid' }", message: /escapes dest/ }, + { record: "{ outputName: 'invalid.txt', content: 42 }", message: /content.*string/i }, +]) { + test(`a later invalid record stops the stream without requesting following yields: ${invalid.record}`, async t => { + const { build, dest, read } = await setup(t, { + 'page.js': `import { writeFile } from 'node:fs/promises' + import { dirname, join } from 'node:path' + export default () => 'new main' + export async function* additionalOutputs ({ page }) { + yield { outputName: 'first.txt', content: 'published' } + yield ${invalid.record} + await writeFile(join(dirname(page.pageFile.filepath), '../following-yield-requested'), 'requested') + yield { outputName: 'following.txt', content: 'must not publish' } + }`, + }) + await writeFiles(dest, { 'index.html': 'old main' }) + await assert.rejects(build(), error => { + assert.match(errorText(error), invalid.message) + return true + }) + assert.equal(await read('first.txt'), 'published') + assert.equal(await read('index.html'), 'old main') + for (const name of ['../escape.txt', 'invalid.txt', '../following-yield-requested', 'following.txt']) { + await assert.rejects(stat(join(dest, name)), { code: 'ENOENT' }) + } + }) +} + test('identical duplicate records from one hook warn rather than reject the build', async t => { const { build, read } = await setup(t, { 'page.js': `export default () => 'main'; export const additionalOutputs = () => [ diff --git a/test-cases/page-additional-outputs/ownership.test.js b/test-cases/page-additional-outputs/ownership.test.js index 14739bf4..36915c6a 100644 --- a/test-cases/page-additional-outputs/ownership.test.js +++ b/test-cases/page-additional-outputs/ownership.test.js @@ -34,7 +34,7 @@ for (const owner of ['page', 'template']) { }) } -test('failed direct builds retain successful ownership for recovery', { timeout: 15_000 }, async t => { +test('repeated failed watch builds union partial paths with successful ownership for recovery', { timeout: 15_000 }, async t => { const { site, src, dest, read, logs } = await setup(t, { 'page.js': "export default () => 'old main'; " + hook('old.txt'), }) @@ -46,13 +46,65 @@ test('failed direct builds retain successful ownership for recovery', { timeout: }`) }, 'ownership failure') assert.equal(await read('old.txt'), 'sidecar', 'failure must not clean up the previous successful output') + assert.equal(await read('partial.txt'), 'partial') + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), `export default () => 'failed again'; export async function* additionalOutputs () { + yield { outputName: 'second-partial.txt', content: 'second partial' } + throw Error('second ownership failure') + }`) + }, 'second ownership failure') + assert.equal(await read('index.html'), 'old main') + assert.equal(await read('old.txt'), 'sidecar') + assert.equal(await read('partial.txt'), 'partial') + assert.equal(await read('second-partial.txt'), 'second partial') await settle(site, logs, async () => { await writeFile(join(src, 'page.js'), "export default () => 'recovered'; " + hook('new.txt')) }) assert.equal(await read('new.txt'), 'sidecar') - await assert.rejects(stat(join(dest, 'old.txt')), { code: 'ENOENT' }) + for (const name of ['old.txt', 'partial.txt', 'second-partial.txt']) { + await assert.rejects(stat(join(dest, name)), { code: 'ENOENT' }) + } }) +for (const change of ['recovery', 'source deletion', 'hook removal']) { + test(`initial failed watch tracks partial ownership for ${change}`, { timeout: 15_000 }, async t => { + const { site, src, dest, read, logs } = await setup(t, { + 'article/page.html': 'Article', + 'article/page.vars.js': `export default {}; export async function* additionalOutputs () { + yield { outputName: 'partial.txt', content: 'partial' } + yield { outputName: '/root-partial.txt', content: 'root partial' } + throw Error('initial ownership failure') + }`, + }) + const result = await site.watch({ serve: false }) + assert.ok(logs.some(line => JSON.parse(line).msg === 'Build Failed!')) + assert.ok(logs.some(line => line.includes('initial ownership failure'))) + assert.equal(await read('article/partial.txt'), 'partial') + assert.equal(await read('root-partial.txt'), 'root partial') + for (const outputRelname of ['article/partial.txt', 'root-partial.txt']) { + const record = result.pageBuildResults?.outputs.find(output => output.outputRelname === outputRelname) + assert.ok(record, `${outputRelname} is included in the failed page build report`) + assert.equal(record.sourceRelname, 'article/page.html') + assert.equal(record.filepath, join(dest, outputRelname)) + } + await assert.rejects(stat(join(dest, 'article/index.html')), { code: 'ENOENT' }) + await settle(site, logs, async () => { + if (change === 'recovery') await writeFile(join(src, 'article/page.vars.js'), 'export default {}; ' + hook('recovered.txt', 'recovered')) + if (change === 'source deletion') await rm(join(src, 'article/page.html')) + if (change === 'hook removal') await writeFile(join(src, 'article/page.vars.js'), 'export default {}') + }) + for (const name of ['article/partial.txt', 'root-partial.txt']) { + await assert.rejects(stat(join(dest, name)), { code: 'ENOENT' }) + } + if (change === 'source deletion') { + await assert.rejects(stat(join(dest, 'article/index.html')), { code: 'ENOENT' }) + } else { + assert.equal(await read('article/index.html'), 'Article') + if (change === 'recovery') assert.equal(await read('article/recovered.txt'), 'recovered') + } + }) +} + test('stale cleanup does not follow symlink ancestors outside dest', { timeout: 15_000 }, async t => { const { site, src, dest, logs } = await setup(t, { 'page.js': "export default () => 'main'; " + hook('nested/owned.txt'), diff --git a/test-cases/page-additional-outputs/watch.test.js b/test-cases/page-additional-outputs/watch.test.js index 62100e87..0f05a4ca 100644 --- a/test-cases/page-additional-outputs/watch.test.js +++ b/test-cases/page-additional-outputs/watch.test.js @@ -102,12 +102,14 @@ test('watch removes sidecars on source rename and draft exclusion', { timeout: 3 assert.equal(await read('renamed.html.txt'), '# Article\n') }) -test('watch hook failure leaves its owning page unchanged and recovery removes stale outputs', { timeout: 30_000 }, async t => { +test('watch hook failure retains partial writes and recovery removes old and partial outputs', { timeout: 30_000 }, async t => { const { site, src, dest, read, mtime, logs } = await setup(t, { - 'page.js': "export default () => 'old main'; " + hook('old.txt', 'old sidecar'), + 'page.js': `export default () => 'old main'; export const additionalOutputs = () => [ + { outputName: 'old.txt', content: 'old sidecar' }, + { outputName: 'stale.txt', content: 'retain until recovery' }, + ]`, }) await site.watch({ serve: false }) - const oldTime = await mtime('old.txt') const oldHtmlTime = await mtime('index.html') await settle(site, logs, async () => { await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* additionalOutputs () { @@ -119,15 +121,17 @@ test('watch hook failure leaves its owning page unchanged and recovery removes s assert.ok(logs.some(line => line.includes('watch iterator exploded')), 'watch reports the hook failure') assert.equal(await read('index.html'), 'old main') assert.equal(await mtime('index.html'), oldHtmlTime) - assert.equal(await read('old.txt'), 'old sidecar') - assert.equal(await mtime('old.txt'), oldTime) - await assert.rejects(stat(join(dest, 'partial.txt')), { code: 'ENOENT' }) + assert.equal(await read('old.txt'), 'failed replacement') + assert.equal(await read('partial.txt'), 'partial') + assert.equal(await read('stale.txt'), 'retain until recovery') await settle(site, logs, async () => { await writeFile(join(src, 'page.js'), "export default () => 'recovered main'; " + hook('new.txt', 'recovered')) }) assert.equal(await read('index.html'), 'recovered main') assert.equal(await read('new.txt'), 'recovered') - await assert.rejects(stat(join(dest, 'old.txt')), { code: 'ENOENT' }) + for (const name of ['old.txt', 'stale.txt', 'partial.txt']) { + await assert.rejects(stat(join(dest, name)), { code: 'ENOENT' }) + } }) for (const change of ['source deletion', 'draft exclusion', 'hook removal', 'companion deletion', 'companion rename']) { diff --git a/test-cases/watch/index.test.js b/test-cases/watch/index.test.js index 3c4438ec..8bb9a897 100644 --- a/test-cases/watch/index.test.js +++ b/test-cases/watch/index.test.js @@ -114,13 +114,13 @@ test('targeted factories reserve untouched owners outputs and recover after a co 'b.pages.js': "export default { outputName: 'b.html', children: 'Owner B' }", }, }) - const retainedTime = (await stat(path.join(dest, 'b.html'))).mtimeMs await writeFile(path.join(src, 'a.pages.js'), "export default { outputName: 'b.html', children: 'Collision' }") await settle(domStack) assert.ok(logs.some(line => line.includes('Output path conflict: b.html is produced by both b.pages.js and a.pages.js#0.')), 'both conflicting producers use source-relative names') assert.match(await readFile(path.join(dest, 'b.html'), 'utf8'), /Owner B/) assert.match(await readFile(path.join(dest, 'a.html'), 'utf8'), /Owner A/) - assert.equal((await stat(path.join(dest, 'b.html'))).mtimeMs, retainedTime) + // A repeated watcher event can trigger a full retry after failure. Streaming + // may rewrite B with its own content before encountering A's collision again. await writeFile(path.join(src, 'a.pages.js'), "export default { outputName: 'c.html', children: 'Recovered' }") await settle(domStack) assert.match(await readFile(path.join(dest, 'c.html'), 'utf8'), /Recovered/) From 67e711f395c65643f42ad05c4717635f6d88dcbb Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Mon, 14 Sep 2026 12:01:16 -0700 Subject: [PATCH 7/8] Address page outputs review: naming, precedence, bookkeeping and hash cache --- docs/generation/README.md | 4 +- docs/layouts/README.md | 35 +- docs/pages/README.md | 124 ++++--- index.js | 14 +- .../additional-outputs-types.test.ts | 49 --- lib/build-pages/index.js | 17 +- .../additional-output-writer.test.js | 138 -------- lib/build-pages/page-builders/js/index.js | 8 +- ...output-writer.js => page-output-writer.js} | 86 ++--- .../page-builders/page-output-writer.test.js | 308 ++++++++++++++++++ lib/build-pages/page-builders/page-writer.js | 59 ++-- ...test.js => page-data-page-outputs.test.js} | 195 ++++++++--- lib/build-pages/page-data.js | 58 ++-- lib/build-pages/page-outputs-types.test.ts | 68 ++++ ...{additional-outputs.js => page-outputs.js} | 42 +-- ...l-outputs.test.js => page-outputs.test.js} | 36 +- lib/domstack-manifest/schema.js | 2 +- lib/domstack-manifest/schema.json | 2 +- test-cases/generated-pages/streaming.test.js | 6 +- test-cases/page-outputs/cache.test.js | 183 +++++++++++ .../helpers.js | 2 +- .../index.test.js | 73 ++--- .../ownership.test.js | 8 +- .../watch.test.js | 10 +- types.ts | 16 +- 25 files changed, 1053 insertions(+), 490 deletions(-) delete mode 100644 lib/build-pages/additional-outputs-types.test.ts delete mode 100644 lib/build-pages/page-builders/additional-output-writer.test.js rename lib/build-pages/page-builders/{additional-output-writer.js => page-output-writer.js} (50%) create mode 100644 lib/build-pages/page-builders/page-output-writer.test.js rename lib/build-pages/{page-data-additional-outputs.test.js => page-data-page-outputs.test.js} (59%) create mode 100644 lib/build-pages/page-outputs-types.test.ts rename lib/build-pages/{additional-outputs.js => page-outputs.js} (62%) rename lib/build-pages/{additional-outputs.test.js => page-outputs.test.js} (63%) create mode 100644 test-cases/page-outputs/cache.test.js rename test-cases/{page-additional-outputs => page-outputs}/helpers.js (96%) rename test-cases/{page-additional-outputs => page-outputs}/index.test.js (85%) rename test-cases/{page-additional-outputs => page-outputs}/ownership.test.js (95%) rename test-cases/{page-additional-outputs => page-outputs}/watch.test.js (94%) diff --git a/docs/generation/README.md b/docs/generation/README.md index 484361eb..4be316bb 100644 --- a/docs/generation/README.md +++ b/docs/generation/README.md @@ -15,6 +15,7 @@ Both can subscribe to shared values prepared by the [data pipeline](../data/). | Archives, tag indexes, or HTML redirects with page variables and layouts | `*.pages.ts` | One or more DOMStack pages | | Feeds, sitemaps, JSON, text, or fully controlled output | `*.template.ts` | One or more files, without layout wrapping | | An ordinary page with its own source directory and browser assets | [Page files](../pages/#page-files) | A source-backed page | +| Markdown downloads, JSON metadata, or other extra files owned by a source-backed page | [`pageOutputs`](../pages/#page-outputs) | Extra files alongside the page's HTML | ## Table of Contents @@ -190,7 +191,8 @@ This keeps factories downstream of source discovery without exposing generation Generated pages use [global bundles](../global-bundles/) and [layout assets](../layouts/#layout-styles). They do not have page-local `style.css`, `client.js`, or worker entries because they do not have their own source-page directory. -Generated pages skip [additional-output hooks](../pages/#additional-outputs), including hooks inherited from layouts; streaming does not enable these hooks. +Support for [page outputs](../pages/#page-outputs) on generated pages is deferred. +Generated pages skip all `pageOutputs` hooks, including hooks inherited from layouts. ### Generated-pages types diff --git a/docs/layouts/README.md b/docs/layouts/README.md index 4889c69f..1d0c7752 100644 --- a/docs/layouts/README.md +++ b/docs/layouts/README.md @@ -63,7 +63,7 @@ DOMStack recognizes these exports from a layout module: | `default` | Yes | A synchronous or asynchronous [layout render function](#layout-render-function). | | `vars` | No | An object, or a sync/async function returning an object, providing [layout defaults](#layout-variables). | | `parentLayout` | No | A non-empty string naming the immediate outer layout; see [Declaring nested layouts](#declaring-nested-layouts). | -| `additionalOutputs` | No | A build-only hook declaring [page-owned additional outputs](../pages/#additional-outputs), such as raw Markdown or JSON sidecars. | +| `pageOutputs` | No | A build-only function returning [extra files for each source page](../pages/#page-outputs), such as Markdown downloads or JSON metadata. | ## Declaring nested layouts @@ -105,17 +105,18 @@ See [Data subscriptions in nested layouts](../cookbook/nested-layouts/#data-subs See [Compose nested layouts](../cookbook/nested-layouts/) for a complete example and asset guidance. -## Additional outputs +## Page outputs -A layout can define shared output policy for its source-backed pages by exporting `additionalOutputs`. -The current source page owns the files, even though the layout declares the hook. -For example, a documentation layout can publish raw Markdown alongside each rendered page: +Export `pageOutputs` from a layout to produce extra files for each source-backed page that uses it. +This is the same export used by page modules and vars companions, and it receives `{ page, vars, data }`. +The function runs for each page, and that page owns the returned files. +For example, a documentation layout can publish a Markdown download alongside each rendered page: ```js // src/docs.layout.js export default ({ children }) => children -export async function additionalOutputs ({ page, vars }) { +export async function pageOutputs ({ page, vars }) { if (page.type !== 'md' || vars.rawExport === false) return [] return { outputName: page.outputName.replace(/\.html$/, '.source.md'), @@ -127,13 +128,21 @@ export async function additionalOutputs ({ page, vars }) { The filename is relative to the current page's output directory, not the layout directory. Using the page's HTML filename helps keep destinations unique when several loose Markdown pages share a directory. Exact duplicate destinations produce best-effort build warnings, not an override contract; avoid sharing output paths between pages or hooks. -Nested hooks run outermost layout → innermost layout → page, and each layout hook shares only that layout renderer's `vars.dataDeps` subscriptions. -Hooks may return a record, an array, or an async iterable, directly or through a promise. -Each record is validated and written (or checked for identical content) before the next record is requested, and a later hook runs only after the preceding hook's files have been processed. -If a later record or provider fails, earlier sidecar writes remain in the destination; outputs are not staged or rolled back. -Watch mode retains these partial outputs in the page's ownership for cleanup after recovery or removal. -Generated pages skip these hooks, including inherited layout hooks. -See [Additional outputs](../pages/#additional-outputs) for the complete API, companion modules, path rules, and watch behavior. +Set `rawExport: false` in a page's frontmatter or vars to opt out of this layout's Markdown download. +Returning `[]` from a page hook does not suppress layout files; the layout itself must check the opt-out variable. + +Nested hooks run outermost layout → innermost layout → selected page-level hook, and their files are additive. +If a JS/TS page module and its vars companion both export `pageOutputs`, the page module's hook wins with a warning; layout hooks still run. +Each layout hook receives the fully resolved page `vars` and only that layout renderer's `vars.dataDeps` subscriptions in `data`. +Declare data needed by the hook in the same subscriptions used by the layout render function. + +Hooks may return a `{ outputName, content }` record, an array of records, or an async iterable of records, directly or through a promise. +DOMStack validates and processes each file before requesting the next record, writing it or retaining an unchanged file during watch rebuilds. +A later hook runs only after the preceding hook's files have been processed. +Files are written directly, without transactions or rollback; if a later record or hook fails, earlier writes remain in the destination. +Watch mode tracks these files for cleanup after a successful rebuild or source removal. +Support for generated `*.pages.*` pages is deferred; they skip these hooks, including inherited layout hooks. +See [Page outputs](../pages/#page-outputs) for the complete arguments, public types, path rules, and watch behavior. ## Layout variables diff --git a/docs/pages/README.md b/docs/pages/README.md index de3315be..a36ccf14 100644 --- a/docs/pages/README.md +++ b/docs/pages/README.md @@ -319,11 +319,14 @@ export default async () => { Page variable files have higher precedence than `global.vars.ts` variables, but lower precedence than frontmatter or `vars` exports from `ts` pages. See [Variables](../../docs/pages/#variables) for the full variable cascade. -## Additional outputs +## Page outputs -Source-backed pages can publish extra files alongside their normal HTML through a named `additionalOutputs` export. -Use a [layout hook](../layouts/#additional-outputs) for shared policy, a JS/TS page-module hook for executable pages, or the page's directly associated vars companion for page-specific Markdown, HTML, or JS/TS behavior. -The hook declares files; it must not write directly to the destination. +Use `pageOutputs` to publish extra files for a source-backed page, such as a Markdown download or a JSON metadata file alongside its HTML. +Export this function from a JS/TS page module, the page's directly associated vars companion, or a [layout](../layouts/#page-outputs). +DOMStack calls the function during the page build and writes the files it returns. +Return output records rather than writing to the destination yourself. + +For example, this page subscribes to `siteMetadata` returned by [`global.data.ts`](../data/#global-data) and writes `article/metadata.json` beneath the destination as well as its normal HTML: ```js // src/article/page.js @@ -334,7 +337,7 @@ export const vars = { export default ({ vars }) => `

${vars.title}

` -export const additionalOutputs = ({ vars, data }) => ({ +export const pageOutputs = ({ vars, data }) => ({ outputName: './metadata.json', content: JSON.stringify({ title: vars.title, site: data.siteMetadata }), }) @@ -342,13 +345,14 @@ export const additionalOutputs = ({ vars, data }) => ({ ### Companion hooks -The existing vars companion can export the same named hook without changing its default vars export: +For Markdown and HTML pages, put `pageOutputs` in the adjacent `page.vars.js` or `page.vars.ts` companion. +Keep page variables in the companion's default export and export the hook separately: ```js // src/article/page.vars.js (alongside page.md) export default { title: 'An article' } -export async function* additionalOutputs ({ page, vars }) { +export async function* pageOutputs ({ page, vars }) { yield { outputName: './source.md', content: await page.readMarkdownContent(), @@ -360,11 +364,11 @@ export async function* additionalOutputs ({ page, vars }) { } ``` -For HTML and JS/TS companions, return suitable text or JSON instead of calling `readMarkdownContent()`. -The hook is a named module export, not a property of resolved vars or executable Markdown frontmatter. +For an HTML page, use the same companion export to return text or JSON rather than calling the Markdown-only `readMarkdownContent()` method. +JS/TS pages can also use a vars companion. Only the directly associated companion provides a page-level hook; global vars and inherited directory vars do not provide hooks. -If both a JS/TS page module and its companion export `additionalOutputs`, the build fails with a provider-conflict error identifying both modules. -Choose one provider rather than relying on precedence. +If both a JS/TS page module and its companion export `pageOutputs`, DOMStack uses the page module's hook and warns with both provider names. +The companion's hook does not run, but applicable layout hooks still run. ### Hook arguments and results @@ -373,27 +377,64 @@ Every hook receives `{ page, vars, data }`: - `page` is a read-only handle to the current source page, including `type`, `path`, `url`, `outputName`, `outputRelname`, `draft`, and read-only `pageFile` metadata. Its `readMarkdownContent()` method reads the Markdown body with YAML frontmatter removed, without rendering Markdown, substituting Handlebars, or rewriting links. The method throws for non-Markdown pages. - The handle does not expose rendering methods, output-writing methods, or another consumer's data. - `vars` contains the fully resolved page variables and is read-only. - `data` contains only the declaring renderer's subscribed global data. Page-module and companion hooks share the page renderer's subscriptions; each layout hook shares that specific layout renderer's subscriptions. - Declare these with the existing static `vars.dataDeps` array convention, or `dataDeps` in the companion's default vars object. - There is no `additionalOutputsDataDeps` export, and undeclared keys are not implicitly available. + Declare required keys through the renderer's existing `vars.dataDeps`, `dataDeps` in a companion's default vars object, or page frontmatter. + Each layout declares its own keys through its `vars.dataDeps`. + Undeclared keys are unavailable, even if another hook subscribes to them. See [Data subscriptions](../data/#data-subscriptions). -A hook returns one explicit `{ outputName: string, content: string }` record, an array of records, or an async iterable of records, directly or through a promise. -Only string content is supported; serialize JSON yourself. -Bare strings are invalid because there is no implicit filename. -An empty array or an async iterator that yields nothing declares no files for that hook. +A hook returns one `{ outputName: string, content: string }` record, an array of records, or an async iterable of records, directly or through a promise. +Use a single record for one file, an array for a fixed set, or an async generator to produce files incrementally. +`outputName` must be a non-empty file path, and `content` must be a string; serialize JSON with `JSON.stringify()`. +Bare strings, `null`, and `undefined` are not valid results. +Return `[]` or yield no records when the hook has no files to produce. + +### Composition and opt-out + +Applicable hooks run in outermost layout → innermost layout → selected page-level hook order. +Layout outputs and page-level outputs are additive; the page hook does not replace layout outputs. +Returning `[]` from the page hook only skips that hook's files. +To let a page opt out of a layout's files, have the layout inspect a resolved variable such as `rawExport: false` and return `[]` itself, as in the [layout example](../layouts/#page-outputs). +Each layout that supports the opt-out must check that variable. + +Hooks run only when building the owning page's output, not when collection or global-data code calls `renderInnerPage()` or `renderFullPage()`. +Support for generated `*.pages.*` pages is deferred; those pages skip all `pageOutputs` hooks, including inherited layout hooks. + +### Streaming results -Applicable hooks execute in outermost layout → innermost layout → page order, and all their outputs are additive. -Records are consumed sequentially, without collecting all hook results first. -DOMStack validates each record and its destination, then writes its content (or checks that the existing bytes are identical) before requesting the next record. +DOMStack consumes records sequentially rather than buffering all hook results. +It validates each record and its destination, then writes the file or retains an unchanged file before requesting the next record. +An async generator resumes after `yield` only once that file has been processed, so it can release resources before preparing the next one. +Arrays use the same per-record processing, but the hook must create the array before returning it. A later layout or page hook starts only after the preceding hook's files have been processed. -Returning `[]` from the page hook does not suppress layout outputs. -For an application-specific opt-out, have the layout inspect a resolved variable such as `rawExport: false` and return `[]` itself. -Hooks run only in the owning page's output-build phase, not when collection or global-data code calls `renderInnerPage()` or `renderFullPage()`. -Generated `*.pages.*` pages skip additional-output hooks entirely, including inherited layout hooks. + +### Page-output types + +Import `PageOutputsFunction` from `@domstack/static/types.js` to type a hook, where `T` is the resolved variables shape and `D` is the declaring renderer's subscribed data shape: + +```ts +// src/article/page.vars.ts +import type { PageOutputsFunction } from '@domstack/static/types.js' + +type ArticleVars = { title: string } +type ArticleData = { siteMetadata: { name: string } } + +export default { + title: 'An article', + dataDeps: ['siteMetadata'], +} + +export const pageOutputs: PageOutputsFunction = ({ vars, data }) => ({ + outputName: './metadata.json', + content: JSON.stringify({ title: vars.title, site: data.siteMetadata.name }), +}) +``` + +`PageOutput` types an individual output record, and `PageOutputsResult` describes the record, array, or async iterable returned by a hook. +`PageOutputsFunctionParams` types the argument object, and `PageOutputsPage` types its read-only `page` handle. +Types describe the values but do not subscribe to global data; keep the runtime `dataDeps` declaration. ### Output paths and failures @@ -408,35 +449,40 @@ Output names resolve beneath the configured destination, including custom destin A leading `/` means destination-root-relative, never filesystem-absolute. Relative paths use the current page's output directory even when a layout declares them. Parent traversal is allowed only while the resolved target remains inside the destination. -Escapes and invalid file targets fail the build. -These rules do not change existing template output-path semantics. +Use portable file names; drive-letter paths, UNC paths, reserved names, and paths ending in a separator are invalid. +Targets that escape the destination, traverse symlinks, or name a directory instead of a file fail the build. Duplicate destinations produce best-effort warnings based on build output reports, including exact duplicates between hooks and conflicts with normal HTML, other pages, templates, copied assets, or bundles. They do not reject the build, even when content differs. -Watch warnings only compare outputs observed in the current page/template phase; this is not a persistent cross-build conflict registry or a case-alias check. +Watch warnings cover outputs observed in the current page/template phase and may miss conflicts with files from earlier builds or names that differ only in case. Choose unique destinations; do not rely on write order or cleanup behavior for conflicting outputs. -Additional files are written directly to the destination as their records arrive, without staging or rollback. -If a later hook, iterator step, output validation, or write fails, earlier sidecar writes remain, including updates to existing files and newly created files. -Processing stops at the failure rather than requesting subsequent records or invoking later hooks. -The page's HTML is currently rendered before sidecar processing and written only after its hooks succeed, so a hook failure leaves the previous HTML in place. -This is not a transactional guarantee: other pages and build phases may already have written their outputs, and filesystem write failures can leave partial updates. +Page outputs are written directly to the destination as their records arrive. +If a later hook, iterator step, output validation, or write fails, earlier writes remain, including updates to existing files and newly created files. +Processing stops at the failure without requesting subsequent records or invoking later hooks. +Async generators can use `try`/`finally` to release resources when iteration stops. +The page's HTML is rendered before its hooks run and written only after they succeed, so a hook failure leaves any previous HTML in place. +Writes are not transactional and are not rolled back: other pages and build phases may already have written their outputs, and filesystem write failures can leave partial updates. ### Watch behavior and ownership -Additional files belong to the source page and appear in page build reports and the build manifest. +Page outputs belong to the source page and appear in page build reports and, when enabled, the build manifest with kind `page-output`. Ownership tracking and cleanup also work when public build-manifest generation is disabled. -A failed build retains prior ownership and adds any sidecar paths already emitted before the failure; it does not clean up the page's old outputs. +A failed build retains previously tracked files and tracks any files successfully written or retained unchanged before the failure; it does not clean up the page's old outputs. After a successful rebuild, DOMStack removes previously owned files no longer returned, including renamed outputs, files from removed hooks, and partial outputs retained from failed attempts. -Partial ownership is tracked even when the initial watch build fails, so recovery, hook removal, or source deletion can clean up those files. +Files written before a failure are tracked even when the initial watch build fails, so recovery, hook removal, or source deletion can clean them up. Source deletion, source rename, or draft exclusion also removes the page's old outputs. Adding, editing, removing, or renaming a companion updates the owning page's hook and output set. Hooks rerun when the owning page rebuilds, including changes to applicable page or layout data subscriptions. A dependency used only by a hook still triggers an HTML rebuild because both outputs rebuild together. -Byte-identical additional files are not rewritten in the live destination, but remain in the complete owned output set. -An article body edit can update that article's HTML and raw export without invalidating sibling raw exports; a shared navigation rebuild can rerun all affected hooks without changing unchanged raw-file mtimes. -Keep collection-wide search indexes and feeds in templates, while using these hooks for per-page artifacts. +During watch rebuilds, DOMStack skips rewriting a previously written page output when its content matches and the destination's filesystem metadata has not changed. +These unchanged files remain owned by the page and keep their modification times. +Missing files or files with changed metadata are written again. +On a fresh build, page outputs are written even if identical files already exist. + +An article body edit can update that article's HTML and Markdown download without rebuilding sibling pages, while a shared data change can rerun affected hooks without rewriting their unchanged files. +Use [templates](../generation/#templates) for collection-wide search indexes and feeds, and page outputs for per-page files. ## Draft pages diff --git a/index.js b/index.js index 0a5c2659..492ae6c1 100644 --- a/index.js +++ b/index.js @@ -11,6 +11,7 @@ * @import { Logger as PinoLogger } from 'pino' * @import { DomstackManifestRecord } from './lib/domstack-manifest/index.js' * @import { WatchDependencyState } from './lib/build-pages/watch-dependencies.js' + * @import { PageOutputCache } from './lib/build-pages/page-builders/page-output-writer.js' * @import { WatchSnapshot, WatchEvent, WatchPlan } from './lib/watch-plan.js' * @typedef {{ dispose: () => Promise }} DisposableBuildContext * @typedef {{ pageFilePath: string, sourcePageFilePath?: string | undefined, pagesFilePath?: string | undefined, layoutNames: string[], outputs?: DomstackManifestRecord[] | undefined }} WatchedPageReport @@ -103,6 +104,8 @@ export class DomStack { #esbuildEntryPoints = new Set() /** @type {Map>} source page or *.pages.* filepath → owned absolute output paths */ #pageOutputMap = new Map() + /** @type {PageOutputCache} Successful writes, including those before an iterator failure. */ + #pageOutputCache = new Map() /** @type {Map>} template filepath → currently claimed absolute output paths */ #templateOutputMap = new Map() /** @type {Map>} *.pages.* filepath → layouts used by its generated pages */ @@ -280,6 +283,8 @@ export class DomStack { ...this.opts, trackWatchDependencies: true, }) + this.#pageOutputCache = pageBuildResults.report.pageOutputCache ?? this.#pageOutputCache + delete pageBuildResults.report.pageOutputCache if (pageBuildResults.errors.length > 0) { this.#rememberPartialPageOutputs(pageBuildResults) throw new DomStackAggregateError(pageBuildResults.errors, 'Page build finished but there were errors.', { @@ -534,8 +539,11 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) ...(templateFilterPaths ? { templateFilterPaths } : {}), ...(pagesFileFilterPaths ? { pagesFileFilterPaths } : {}), previousWatchDependencies: this.#watchDependencies, + previousPageOutputCache: this.#pageOutputCache, trackWatchDependencies: true, }) + this.#pageOutputCache = pageBuildResults.report.pageOutputCache ?? this.#pageOutputCache + delete pageBuildResults.report.pageOutputCache if (pageBuildResults.errors.length > 0) { this.#rememberPartialPageOutputs(pageBuildResults) throw new DomStackAggregateError(pageBuildResults.errors, 'Page build finished but there were errors.', { @@ -595,7 +603,7 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) const templates = isFiltered ? new Map(this.#templateOutputMap) : new Map() // Factories can successfully rebuild to zero pages; regular pages always - // report their HTML output, even when their additional-output hook is gone. + // report their HTML output, even when their page-output hook is gone. for (const owner of results.report.rebuiltPagesFilePaths ?? []) pages.delete(owner) for (const [owner, outputs] of rebuiltPages) pages.set(owner, outputs) for (const report of results.report.templates) { @@ -616,6 +624,10 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) } for (const filepath of stale) await removeStalePageOutput(dest, filepath) + const pageOwnedPaths = new Set([...pages.values()].flatMap(outputs => [...outputs])) + for (const filepath of this.#pageOutputCache.keys()) { + if (!pageOwnedPaths.has(filepath)) this.#pageOutputCache.delete(filepath) + } this.#pageOutputMap = pages this.#templateOutputMap = templates } diff --git a/lib/build-pages/additional-outputs-types.test.ts b/lib/build-pages/additional-outputs-types.test.ts deleted file mode 100644 index 8452dc99..00000000 --- a/lib/build-pages/additional-outputs-types.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { - AdditionalOutput, - AdditionalOutputProvenance, - AdditionalOutputsFunction, - AdditionalOutputsFunctionParams, - AdditionalOutputsPage, - AdditionalOutputsResult, - CollectedAdditionalOutput, - PageData, -} from '../../types.ts' -import { normalizeAdditionalOutputs } from './additional-outputs.js' - -// Compile-only assertions for the public type entry and the narrow hook contract. -export function checkAdditionalOutputsTypes (pageData: PageData<{ title: string }>, page: AdditionalOutputsPage) { - const output: AdditionalOutput = { outputName: 'feed.json', content: '' } - const provenance: AdditionalOutputProvenance = { kind: 'layout', source: 'base.layout.ts', layoutName: 'base' } - const collected: CollectedAdditionalOutput = { ...output, provenance } - const result: AdditionalOutputsResult = [output] - const hook: AdditionalOutputsFunction<{ title: string }, { posts: string[] }> = async ({ page, vars, data }) => ({ - outputName: 'feed.json', content: vars.title + page.url + data.posts.join(','), - }) - const params: AdditionalOutputsFunctionParams<{ title: string }, { posts: string[] }> = { page, vars: { title: 'title' }, data: { posts: [] } } - const outputs: AsyncGenerator = pageData.collectAdditionalOutputs() - const normalized: AsyncGenerator = normalizeAdditionalOutputs(result, provenance) - const iterable: AsyncIterable = outputs - // @ts-expect-error Collection is streamed, not a promise of buffered records. - const buffered: Promise = pageData.collectAdditionalOutputs() - const iterator: AdditionalOutputsFunction = async function * () { yield output } - const promisedIterator: AdditionalOutputsFunction = async () => (async function * () { yield output })() - // @ts-expect-error Bare strings are not hook results. - const badHook: AdditionalOutputsFunction = () => 'html' - // @ts-expect-error Content must be a string. - const badOutput: AdditionalOutput = { outputName: 'feed.json', content: {} } - // @ts-expect-error Page metadata is read-only. - page.url = '/other' - // @ts-expect-error Source metadata is read-only. - page.pageFile.filepath = '/other.md' - // @ts-expect-error Hooks cannot render pages. - page.renderFullPage() - // @ts-expect-error Hooks cannot access a PageData instance. - const bypass = page.pageInfo - // @ts-expect-error Global data is only accessible through the subscribed params.data. - const globalData = page.data - // @ts-expect-error Vars are read-only. - params.vars.title = 'other' - // @ts-expect-error Only declared data is available. - const secret = params.data.secret - return { hook, params, outputs, normalized, iterable, buffered, collected, result, iterator, promisedIterator, badHook, badOutput, bypass, globalData, secret } -} diff --git a/lib/build-pages/index.js b/lib/build-pages/index.js index b7cb17f8..5ce02f0a 100644 --- a/lib/build-pages/index.js +++ b/lib/build-pages/index.js @@ -6,6 +6,7 @@ * @import { ResolvedLayout } from './page-data.js' * @import { DomstackManifestRecord } from '../domstack-manifest/index.js' * @import { WatchDependencyState, WatchConsumer, WatchDependencyTracker } from './watch-dependencies.js' + * @import { PageOutputCache } from './page-builders/page-output-writer.js' */ import { Worker } from 'worker_threads' @@ -44,6 +45,7 @@ const __dirname = import.meta.dirname * @property {PageReport[]} pages * @property {TemplateReport[]} templates * @property {WatchDependencyState | undefined} [watchDependencies] + * @property {PageOutputCache | undefined} [pageOutputCache] * @property {string[] | undefined} [rebuiltPagesFilePaths] */ @@ -90,6 +92,7 @@ const __dirname = import.meta.dirname * @property {boolean | undefined} [buildDrafts] - Include generated page definitions marked as drafts. * @property {WatchDependencyState | null | undefined} [previousWatchDependencies] - Dependency state from the previous successful watch build. * @property {boolean | undefined} [trackWatchDependencies] - Collect subscriptions for incremental watch builds. + * @property {PageOutputCache | undefined} [previousPageOutputCache] - Successful output hashes and metadata retained across watch workers. */ /** @@ -461,6 +464,7 @@ export function buildPages (src, dest, siteData, opts) { buildDrafts: opts?.buildDrafts, previousWatchDependencies: opts?.previousWatchDependencies, trackWatchDependencies: opts?.trackWatchDependencies, + previousPageOutputCache: opts?.previousPageOutputCache, } return new Promise((resolve, reject) => { @@ -519,6 +523,9 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { warnings: [], } + const outputCache = opts?.trackWatchDependencies ? new Map(opts.previousPageOutputCache) : undefined + result.report.pageOutputCache = outputCache + const pageFilterSet = opts?.pageFilterPaths ? new Set(opts.pageFilterPaths) : null const templateFilterSet = opts?.templateFilterPaths ? new Set(opts.templateFilterPaths) : null const pagesFileFilterSet = opts?.pagesFileFilterPaths ? new Set(opts.pagesFileFilterPaths) : null @@ -662,13 +669,11 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { /** @param {PageData} page */ const writePage = async (page) => { - /** @type {DomstackManifestRecord[]} */ - const emittedOutputs = [] try { const buildResult = await pageWriter({ dest, page, - onAdditionalOutput: output => emittedOutputs.push(output), + outputCache, }) result.report.pages.push({ @@ -684,16 +689,16 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { } catch (err) { // Direct writes already emitted by a failed iterator still need ownership // so a later successful watch rebuild can remove them. - if (emittedOutputs.length > 0) { + if (page.outputRecords.length > 0) { result.report.pages.push({ pageFilePath: join(dest, page.pageInfo.outputRelname), sourcePageFilePath: page.pageInfo.generated ? undefined : page.pageInfo.pageFile.filepath, pagesFilePath: page.pageInfo.generated?.pagesFile.pagesFile.filepath, layoutName: page.layout?.name, layoutNames: page.layoutChain.map(layout => layout.name), - outputs: emittedOutputs, + outputs: page.outputRecords, }) - result.outputs.push(...emittedOutputs) + result.outputs.push(...page.outputRecords) } result.errors.push(serializeBuildError(err, { page: pageInfoForWorker(page.pageInfo) }, `Error building page "${page.pageInfo.pageFile.relname}"`)) return false diff --git a/lib/build-pages/page-builders/additional-output-writer.test.js b/lib/build-pages/page-builders/additional-output-writer.test.js deleted file mode 100644 index d713b93c..00000000 --- a/lib/build-pages/page-builders/additional-output-writer.test.js +++ /dev/null @@ -1,138 +0,0 @@ -/** - * @import { PageInfo } from '../../identify-pages.js' - * @import { PageData } from '../page-data.js' - */ -import { test } from 'node:test' -import assert from 'node:assert/strict' -import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join, resolve } from 'node:path' -import { resolveAdditionalOutputPath, writeAdditionalOutputs } from './additional-output-writer.js' -import { pageWriter } from './page-writer.js' -import { createEntry } from '../../domstack-manifest/records.js' - -const pageInfo = /** @type {PageInfo} */ ({ - path: 'posts', - outputName: 'index.html', - outputRelname: 'posts/index.html', - url: '/posts/', - pageFile: { relname: 'posts/page.js' }, -}) - -test('additional output paths use the actual page output directory and destination root', () => { - const dest = resolve('public') - const page = join(dest, 'posts', 'nested', 'index.html') - /** @type {[string, string][]} */ - const cases = [ - ['data.json', 'posts/nested/data.json'], - ['../feed.json', 'posts/feed.json'], - ['../../feed.json', 'feed.json'], - ['/feed.json', 'feed.json'], - ['..\\feed.json', 'posts/feed.json'], - ['./data.json', 'posts/nested/data.json'], - ['..hidden/data.json', 'posts/nested/..hidden/data.json'], - ] - for (const [name, expected] of cases) { - const result = resolveAdditionalOutputPath(dest, page, name) - assert.equal(result.outputRelname, expected) - assert.equal(result.filepath, join(dest, expected)) - } -}) - -test('additional output paths reject escapes, directory names and nonportable file names', () => { - const dest = resolve('public') - const page = join(dest, 'posts/index.html') - for (const name of ['', ' ', '/', '.', '..', 'foo/', 'foo\\', 'foo/.', 'foo/..', '../../escape.json', '/../escape.json', '//server/file', '\\file', '\\\\server\\file', 'C:/file', 'C:file', 'file:stream', 'file\u0000.json', 'file?', 'NUL.json', 'aux', 'COM1.txt', 'dir./file', 'dir /file']) { - assert.throws(() => resolveAdditionalOutputPath(dest, page, name), Error, name) - } -}) - -test('writer writes sidecars with page ownership and non-navigation JSON records', async t => { - const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-')) - t.after(() => rm(dest, { recursive: true, force: true })) - const outputs = await writeAdditionalOutputs({ - dest, - pageFilePath: join(dest, 'posts/index.html'), - pageInfo, - additionalOutputs: [{ outputName: '/feed.json', content: '{"ok":true}' }, { outputName: 'nested/data.txt', content: 'hello' }], - }) - assert.equal(await readFile(join(dest, 'feed.json'), 'utf8'), '{"ok":true}') - assert.equal(await readFile(join(dest, 'posts/nested/data.txt'), 'utf8'), 'hello') - assert.ok(outputs[0]) - assert.equal(outputs[0].kind, 'page-additional') - assert.equal(outputs[0].sourceRelname, 'posts/page.js') - assert.equal(outputs[0].pagePath, 'posts') - assert.equal(outputs[0].pageUrl, '/posts/') - assert.equal(outputs[0].url, '/feed.json') - assert.equal(outputs[0].page, undefined) - const entry = await createEntry({ dest, record: outputs[0] }) - assert.equal(entry?.role, 'subresource') -}) - -test('writer rejects symlink components and existing directories without touching their targets', async t => { - const root = await mkdtemp(join(tmpdir(), 'domstack-additional-links-')) - t.after(() => rm(root, { recursive: true, force: true })) - const dest = join(root, 'dest') - const outside = join(root, 'outside') - await mkdir(dest) - await mkdir(outside) - await writeFile(join(outside, 'data.json'), 'unchanged') - await symlink(outside, join(dest, 'linked'), 'dir') - await symlink(join(outside, 'data.json'), join(dest, 'leaf.json'), 'file') - await mkdir(join(dest, 'directory')) - for (const outputName of ['linked/data.json', 'leaf.json', 'directory']) { - await assert.rejects(writeAdditionalOutputs({ - dest, - pageFilePath: join(dest, 'index.html'), - pageInfo, - additionalOutputs: [{ outputName, content: 'changed' }], - }), /symlink|not a file/) - } - assert.equal(await readFile(join(outside, 'data.json'), 'utf8'), 'unchanged') -}) - -test('writer validates each path before writing it, retaining earlier writes', async t => { - const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-invalid-')) - t.after(() => rm(dest, { recursive: true, force: true })) - await assert.rejects(writeAdditionalOutputs({ - dest, - pageFilePath: join(dest, 'index.html'), - pageInfo, - additionalOutputs: [{ outputName: 'valid.json', content: '{}' }, { outputName: '../escape.json', content: '{}' }], - }), /escapes dest/) - assert.equal(await readFile(join(dest, 'valid.json'), 'utf8'), '{}') -}) - -test('writer preserves duplicate records for duplicate-output warnings', async t => { - const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-claims-')) - t.after(() => rm(dest, { recursive: true, force: true })) - const outputs = await writeAdditionalOutputs({ - dest, - pageFilePath: join(dest, 'index.html'), - pageInfo, - additionalOutputs: [{ outputName: 'data.json', content: 'first' }, { outputName: './data.json', content: 'second' }], - }) - assert.equal(outputs.length, 2) - assert.ok(outputs[0]) - assert.ok(outputs[1]) - assert.equal(outputs[0].outputRelname, outputs[1].outputRelname) - assert.equal(outputs[0].sourceRelname, outputs[1].sourceRelname) -}) - -test('page writer collects once after rendering and reports HTML and sidecars together', async t => { - const dest = await mkdtemp(join(tmpdir(), 'domstack-additional-page-')) - t.after(() => rm(dest, { recursive: true, force: true })) - /** @type {string[]} */ - const events = [] - const page = /** @type {PageData} */ (/** @type {unknown} */ ({ - pageInfo, - vars: {}, - async renderFullPage () { events.push('render'); return '

Page

' }, - async * collectAdditionalOutputs () { events.push('collect'); yield { outputName: 'data.json', content: '{}' } }, - })) - const result = await pageWriter({ dest, page }) - assert.deepEqual(events, ['render', 'collect']) - assert.deepEqual(result.outputs.map(output => output.kind), ['page', 'page-additional']) - assert.equal(await readFile(result.pageFilePath, 'utf8'), '

Page

') - assert.equal(await readFile(join(dest, 'posts/data.json'), 'utf8'), '{}') -}) diff --git a/lib/build-pages/page-builders/js/index.js b/lib/build-pages/page-builders/js/index.js index 94f69bc4..2eb2a02f 100644 --- a/lib/build-pages/page-builders/js/index.js +++ b/lib/build-pages/page-builders/js/index.js @@ -4,7 +4,7 @@ */ import assert from 'node:assert' -import { validateAdditionalOutputsHook } from '../../additional-outputs.js' +import { validatePageOutputsHook } from '../../page-outputs.js' /** * Resolve a JavaScript page module. @@ -27,11 +27,11 @@ export async function jsBuilder ({ pageInfo }) { } } - const { default: pageLayout, vars, additionalOutputs } = await import(pageInfo.pageFile.filepath) + const { default: pageLayout, vars, pageOutputs } = await import(pageInfo.pageFile.filepath) assert(pageLayout, 'js pages must export a page layout default export') assert(typeof pageLayout === 'function', 'js pages pageLayout must be a function') - const hook = validateAdditionalOutputsHook(additionalOutputs, pageInfo.pageFile.filepath) - return { vars, pageLayout, additionalOutputs: hook } + const hook = validatePageOutputsHook(pageOutputs, pageInfo.pageFile.filepath) + return { vars, pageLayout, pageOutputs: hook } } diff --git a/lib/build-pages/page-builders/additional-output-writer.js b/lib/build-pages/page-builders/page-output-writer.js similarity index 50% rename from lib/build-pages/page-builders/additional-output-writer.js rename to lib/build-pages/page-builders/page-output-writer.js index 2a9f263b..d650dfaa 100644 --- a/lib/build-pages/page-builders/additional-output-writer.js +++ b/lib/build-pages/page-builders/page-output-writer.js @@ -1,9 +1,13 @@ /** - * @import { PageInfo } from '../../identify-pages.js' - * @import { DomstackManifestRecord } from '../../domstack-manifest/index.js' + * @import { Stats } from 'node:fs' + * @import { PageData } from '../page-data.js' + * + * @typedef {Map} PageOutputCache */ -import { lstat, mkdir, readFile, writeFile } from 'node:fs/promises' -import { dirname, isAbsolute, relative, resolve, sep } from 'node:path' +import { createHash } from 'node:crypto' +import { lstat, mkdir, writeFile } from 'node:fs/promises' +import { dirname, relative, resolve, sep } from 'node:path' +import { assertInsideDest, toPosix } from '../../helpers/path.js' import { createDomstackManifestRecord } from '../../domstack-manifest/index.js' /** @@ -15,33 +19,33 @@ import { createDomstackManifestRecord } from '../../domstack-manifest/index.js' * @param {string} pageFilePath * @param {string} outputName */ -export function resolveAdditionalOutputPath (dest, pageFilePath, outputName) { +export function resolvePageOutputPath (dest, pageFilePath, outputName) { if (typeof outputName !== 'string' || !outputName.trim()) { - throw new TypeError('Additional outputName must be a non-empty file path') + throw new TypeError('Page outputName must be a non-empty file path') } const name = outputName.replaceAll('\\', '/') if (outputName.startsWith('\\') || name.startsWith('//') || /^[a-z]:/i.test(name)) { - throw new Error(`Additional outputName must not be a drive or UNC path: ${outputName}`) + throw new Error(`Page outputName must not be a drive or UNC path: ${outputName}`) } const parts = name.split('/') if (!parts.at(-1) || ['.', '..'].includes(parts.at(-1) ?? '')) { - throw new Error(`Additional outputName must name a file: ${outputName}`) + throw new Error(`Page outputName must name a file: ${outputName}`) } for (const part of parts) { if (!part || part === '.' || part === '..') continue // Reject Windows aliases and special files even when building on POSIX. if (/[<>:"|?*]/u.test(part) || Array.from(part).some(character => character.charCodeAt(0) < 32) || /[. ]$/.test(part) || /^(con|prn|aux|nul|com[1-9¹²³]|lpt[1-9¹²³])(?:\.|$)/i.test(part)) { - throw new Error(`Additional outputName contains an invalid file path component: ${outputName}`) + throw new Error(`Page outputName contains an invalid file path component: ${outputName}`) } } const filepath = name.startsWith('/') ? resolve(dest, name.slice(1)) : resolve(dirname(pageFilePath), name) const relname = relative(resolve(dest), filepath) - if (!relname || relname === '..' || relname.startsWith(`..${sep}`) || isAbsolute(relname)) { - throw new Error(`Additional outputName escapes dest or names its directory: ${outputName}`) - } - return { filepath, outputRelname: relname.split(sep).join('/') } + const message = `Page outputName escapes dest or names its directory: ${outputName}` + assertInsideDest(dest, filepath, message) + if (!relname) throw new Error(message) + return { filepath, outputRelname: toPosix(relname) } } /** @@ -64,11 +68,12 @@ async function assertWritablePath (dest, filepath) { if (/** @type {NodeJS.ErrnoException} */ (error).code === 'ENOENT') continue throw error } - if (info.isSymbolicLink()) throw new Error(`Additional output path contains a symlink: ${current}`) + if (info.isSymbolicLink()) throw new Error(`Page output path contains a symlink: ${current}`) const leaf = index === components.length - 1 if (leaf ? !info.isFile() : !info.isDirectory()) { - throw new Error(`Additional output path is not a ${leaf ? 'file' : 'directory'}: ${current}`) + throw new Error(`Page output path is not a ${leaf ? 'file' : 'directory'}: ${current}`) } + if (leaf) return info } } @@ -78,42 +83,49 @@ async function assertWritablePath (dest, filepath) { * @param {object} params * @param {string} params.dest * @param {string} params.pageFilePath - * @param {PageInfo} params.pageInfo - * @param {Iterable<{outputName: string, content: string}> | AsyncIterable<{outputName: string, content: string}>} params.additionalOutputs - * @param {((output: DomstackManifestRecord) => void) | undefined} [params.onOutput] - * @returns {Promise} + * @param {Pick, 'pageInfo' | 'outputRecords'>} params.page + * @param {Iterable<{outputName: string, content: string}> | AsyncIterable<{outputName: string, content: string}>} params.pageOutputs + * @param {PageOutputCache | undefined} [params.outputCache] */ -export async function writeAdditionalOutputs ({ dest, pageFilePath, pageInfo, additionalOutputs, onOutput }) { - const records = [] - for await (const output of additionalOutputs) { - if (typeof output.content !== 'string') throw new TypeError('Additional output content must be a string') - const { filepath, outputRelname } = resolveAdditionalOutputPath(dest, pageFilePath, output.outputName) - await assertWritablePath(dest, filepath) - const bytes = Buffer.from(output.content) - let unchanged = false - try { - unchanged = bytes.equals(await readFile(filepath)) - } catch (error) { - if (/** @type {NodeJS.ErrnoException} */ (error).code !== 'ENOENT') throw error - } +export async function writePageOutputs ({ dest, pageFilePath, page, pageOutputs, outputCache }) { + const { pageInfo, outputRecords } = page + for await (const output of pageOutputs) { + if (typeof output.content !== 'string') throw new TypeError('Page output content must be a string') + const { filepath, outputRelname } = resolvePageOutputPath(dest, pageFilePath, output.outputName) + const info = await assertWritablePath(dest, filepath) + const hash = outputCache ? createHash('sha256').update(output.content, 'utf8').digest('hex') : undefined + const cached = outputCache?.get(filepath) + const unchanged = cached && cached.hash === hash && info && cached.metadata === fileMetadata(info) if (!unchanged) { + outputCache?.delete(filepath) await mkdir(dirname(filepath), { recursive: true }) - await writeFile(filepath, bytes) + await writeFile(filepath, output.content) } const record = { ...createDomstackManifestRecord({ dest, filepath, outputRelname, - kind: 'page-additional', + kind: 'page-output', sourceRelname: pageInfo.pageFile.relname, pagePath: pageInfo.path, pageUrl: pageInfo.url, }), pagePath: pageInfo.path, } - records.push(record) - onOutput?.(record) + outputRecords.push(record) + if (!unchanged && outputCache && hash) { + outputCache.set(filepath, { hash, metadata: fileMetadata(await lstat(filepath)) }) + } } - return records + return outputRecords +} + +/** + * Best-effort detection of replacement or external edits, even if mtime is restored. + * Reuses the path validation stat; never rereads destination content. + * @param {Stats} info + */ +function fileMetadata (info) { + return [info.dev, info.ino, info.size, info.mtimeMs, info.ctimeMs].join(':') } diff --git a/lib/build-pages/page-builders/page-output-writer.test.js b/lib/build-pages/page-builders/page-output-writer.test.js new file mode 100644 index 00000000..5a0951b3 --- /dev/null +++ b/lib/build-pages/page-builders/page-output-writer.test.js @@ -0,0 +1,308 @@ +/** + * @import { PageInfo } from '../../identify-pages.js' + * @import { DomstackManifestRecord } from '../../domstack-manifest/index.js' + * @import { PageOutputCache } from './page-output-writer.js' + */ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import fs, { lstat, mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { resolvePageOutputPath, writePageOutputs } from './page-output-writer.js' + +import { createEntry } from '../../domstack-manifest/records.js' + +const pageInfo = /** @type {PageInfo} */ ({ + path: 'posts', + outputName: 'index.html', + outputRelname: 'posts/index.html', + url: '/posts/', + pageFile: { relname: 'posts/page.js' }, +}) + +test('page output paths use the actual page output directory and destination root', () => { + const dest = resolve('public') + const page = join(dest, 'posts', 'nested', 'index.html') + /** @type {[string, string][]} */ + const cases = [ + ['data.json', 'posts/nested/data.json'], + ['../feed.json', 'posts/feed.json'], + ['../../feed.json', 'feed.json'], + ['/feed.json', 'feed.json'], + ['..\\feed.json', 'posts/feed.json'], + ['./data.json', 'posts/nested/data.json'], + ['..hidden/data.json', 'posts/nested/..hidden/data.json'], + ] + for (const [name, expected] of cases) { + const result = resolvePageOutputPath(dest, page, name) + assert.equal(result.outputRelname, expected) + assert.equal(result.filepath, join(dest, expected)) + } +}) + +test('page output paths reject escapes, directory names and nonportable file names', () => { + const dest = resolve('public') + const page = join(dest, 'posts/index.html') + for (const name of ['', ' ', '/', '.', '..', 'foo/', 'foo\\', 'foo/.', 'foo/..', '../../escape.json', '/../escape.json', '//server/file', '\\file', '\\\\server\\file', 'C:/file', 'C:file', 'file:stream', 'file\u0000.json', 'file?', 'NUL.json', 'aux', 'COM1.txt', 'dir./file', 'dir /file']) { + assert.throws(() => resolvePageOutputPath(dest, page, name), Error, name) + } + for (const name of ['../../public', '../../escape.json']) { + assert.throws(() => resolvePageOutputPath(dest, page, name), { + message: `Page outputName escapes dest or names its directory: ${name}`, + }) + } +}) + +test('writer writes sidecars with page ownership and non-navigation JSON records', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-page-output-')) + t.after(() => rm(dest, { recursive: true, force: true })) + const page = { pageInfo, outputRecords: /** @type {DomstackManifestRecord[]} */ ([]) } + const outputs = await writePageOutputs({ + dest, + pageFilePath: join(dest, 'posts/index.html'), + page, + pageOutputs: [{ outputName: '/feed.json', content: '{"ok":true}' }, { outputName: 'nested/data.txt', content: 'hello' }], + }) + assert.equal(outputs, page.outputRecords) + assert.equal(await readFile(join(dest, 'feed.json'), 'utf8'), '{"ok":true}') + assert.equal(await readFile(join(dest, 'posts/nested/data.txt'), 'utf8'), 'hello') + assert.ok(outputs[0]) + assert.equal(outputs[0].kind, 'page-output') + assert.equal(outputs[0].sourceRelname, 'posts/page.js') + assert.equal(outputs[0].pagePath, 'posts') + assert.equal(outputs[0].pageUrl, '/posts/') + assert.equal(outputs[0].url, '/feed.json') + assert.equal(outputs[0].page, undefined) + const entry = await createEntry({ dest, record: outputs[0] }) + assert.equal(entry?.role, 'subresource') +}) + +test('writer rejects symlink components and existing directories without touching their targets', async t => { + const root = await mkdtemp(join(tmpdir(), 'domstack-page-output-links-')) + t.after(() => rm(root, { recursive: true, force: true })) + const dest = join(root, 'dest') + const outside = join(root, 'outside') + await mkdir(dest) + await mkdir(outside) + await writeFile(join(outside, 'data.json'), 'unchanged') + await symlink(outside, join(dest, 'linked'), 'dir') + await symlink(join(outside, 'data.json'), join(dest, 'leaf.json'), 'file') + await mkdir(join(dest, 'directory')) + for (const outputName of ['linked/data.json', 'leaf.json', 'directory']) { + const page = { pageInfo, outputRecords: [] } + /** @type {PageOutputCache} */ + const outputCache = new Map() + await assert.rejects(writePageOutputs({ + dest, + pageFilePath: join(dest, 'index.html'), + page, + pageOutputs: [{ outputName, content: 'changed' }], + outputCache, + }), /symlink|not a file/) + assert.deepEqual(page.outputRecords, []) + assert.equal(outputCache.size, 0) + } + assert.equal(await readFile(join(outside, 'data.json'), 'utf8'), 'unchanged') +}) + +test('writer validates each path before writing it, retaining earlier writes', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-page-output-invalid-')) + t.after(() => rm(dest, { recursive: true, force: true })) + const page = { pageInfo, outputRecords: /** @type {DomstackManifestRecord[]} */ ([]) } + /** @type {PageOutputCache} */ + const outputCache = new Map() + await assert.rejects(writePageOutputs({ + dest, + pageFilePath: join(dest, 'index.html'), + page, + pageOutputs: [{ outputName: 'valid.json', content: '{}' }, { outputName: '../escape.json', content: '{}' }], + outputCache, + }), /escapes dest/) + assert.equal(await readFile(join(dest, 'valid.json'), 'utf8'), '{}') + assert.deepEqual(page.outputRecords.map(output => output.outputRelname), ['valid.json']) + assert.deepEqual([...outputCache.keys()], [join(dest, 'valid.json')]) +}) + +test('writer preserves duplicate records for duplicate-output warnings', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-page-output-claims-')) + t.after(() => rm(dest, { recursive: true, force: true })) + const outputs = await writePageOutputs({ + dest, + pageFilePath: join(dest, 'index.html'), + page: { pageInfo, outputRecords: [] }, + pageOutputs: [{ outputName: 'data.json', content: 'first' }, { outputName: './data.json', content: 'second' }], + }) + assert.equal(outputs.length, 2) + assert.ok(outputs[0]) + assert.ok(outputs[1]) + assert.equal(outputs[0].outputRelname, outputs[1].outputRelname) + assert.equal(outputs[0].sourceRelname, outputs[1].sourceRelname) +}) + +test('writer caches SHA256 of UTF8 bytes and stat metadata, not output content', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-page-output-hash-')) + t.after(() => rm(dest, { recursive: true, force: true })) + const page = { pageInfo, outputRecords: [] } + /** @type {PageOutputCache} */ + const outputCache = new Map() + const content = 'café 🌊\n' + const filepath = join(dest, 'data.txt') + const outputs = await writePageOutputs({ + dest, + pageFilePath: join(dest, 'index.html'), + page, + pageOutputs: [{ outputName: './data.txt', content }], + outputCache, + }) + const info = await lstat(filepath) + assert.equal(info.size, Buffer.byteLength(content, 'utf8')) + assert.deepEqual([...outputCache], [[filepath, { + hash: createHash('sha256').update(Buffer.from(content, 'utf8')).digest('hex'), + metadata: [info.dev, info.ino, info.size, info.mtimeMs, info.ctimeMs].join(':'), + }]]) + assert.equal(outputs, page.outputRecords) + assert.equal(outputs.length, 1) + assert.ok(outputs[0]) + assert.equal('content' in outputs[0], false) + assert.equal(await readFile(filepath, 'utf8'), content) +}) + +for (const cached of [false, true]) { + test(`writer overwrites existing equal bytes without reading with ${cached ? 'a cold' : 'no'} cache`, async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-page-output-cold-')) + t.after(() => rm(dest, { recursive: true, force: true })) + const filepath = join(dest, 'data.txt') + await writeFile(filepath, 'equal bytes') + const writes = t.mock.method(fs, 'writeFile') + const reads = t.mock.method(fs, 'readFile', async () => { throw new Error('must not read destination content') }) + syncBuiltinESMExports() + t.after(() => { t.mock.restoreAll(); syncBuiltinESMExports() }) + const page = { pageInfo, outputRecords: [] } + /** @type {PageOutputCache | undefined} */ + const outputCache = cached ? new Map() : undefined + const outputs = await writePageOutputs({ + dest, + pageFilePath: join(dest, 'index.html'), + page, + pageOutputs: [{ outputName: 'data.txt', content: 'equal bytes' }], + outputCache, + }) + assert.equal(writes.mock.callCount(), 1) + assert.deepEqual(writes.mock.calls[0]?.arguments, [filepath, 'equal bytes']) + assert.equal(reads.mock.callCount(), 0) + assert.equal(outputs, page.outputRecords) + assert.equal(outputs.length, 1) + assert.equal(outputCache?.has(filepath), cached ? true : undefined) + }) +} + +test('writer skips matching cached bytes and metadata but still records every claim', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-page-output-warm-')) + t.after(() => rm(dest, { recursive: true, force: true })) + /** @type {PageOutputCache} */ + const outputCache = new Map() + const page = { pageInfo, outputRecords: [] } + const params = { dest, pageFilePath: join(dest, 'index.html'), page, outputCache } + await writePageOutputs({ ...params, pageOutputs: [{ outputName: 'data.txt', content: 'same' }] }) + const previousCache = outputCache.get(join(dest, 'data.txt')) + const writes = t.mock.method(fs, 'writeFile') + const reads = t.mock.method(fs, 'readFile', async () => { throw new Error('must not read destination content') }) + syncBuiltinESMExports() + t.after(() => { t.mock.restoreAll(); syncBuiltinESMExports() }) + const outputs = await writePageOutputs({ + ...params, + pageOutputs: [{ outputName: './data.txt', content: 'same' }, { outputName: '/data.txt', content: 'same' }], + }) + assert.equal(writes.mock.callCount(), 0) + assert.equal(reads.mock.callCount(), 0) + assert.equal(outputs, page.outputRecords) + assert.equal(outputs.length, 3) + assert.deepEqual(outputs[1], outputs[0]) + assert.deepEqual(outputs[2], outputs[0]) + assert.equal(outputCache.get(join(dest, 'data.txt')), previousCache) +}) + +test('writer rewrites on a hash mismatch or any stat metadata mismatch', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-page-output-mismatch-')) + t.after(() => rm(dest, { recursive: true, force: true })) + /** @type {PageOutputCache} */ + const outputCache = new Map() + const filepath = join(dest, 'data.txt') + const params = { dest, pageFilePath: join(dest, 'index.html'), page: { pageInfo, outputRecords: [] }, outputCache } + const pageOutputs = [{ outputName: 'data.txt', content: 'same' }] + await writePageOutputs({ ...params, pageOutputs }) + const writes = t.mock.method(fs, 'writeFile') + syncBuiltinESMExports() + t.after(() => { t.mock.restoreAll(); syncBuiltinESMExports() }) + for (const field of ['hash', 'dev', 'ino', 'size', 'mtimeMs', 'ctimeMs']) { + const cached = outputCache.get(filepath) + assert.ok(cached) + const metadata = cached.metadata.split(':') + if (field === 'hash') cached.hash = 'stale' + else metadata[['dev', 'ino', 'size', 'mtimeMs', 'ctimeMs'].indexOf(field)] = 'stale' + cached.metadata = metadata.join(':') + writes.mock.resetCalls() + await writePageOutputs({ ...params, pageOutputs }) + assert.equal(writes.mock.callCount(), 1, field) + assert.notEqual(outputCache.get(filepath), cached, field) + } +}) + +test('failed writes leave no successful records or cache entries, including stale cache entries', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-page-output-failure-')) + t.after(() => rm(dest, { recursive: true, force: true })) + const filepath = join(dest, 'data.txt') + const cause = new Error('disk full') + const writes = t.mock.method(fs, 'writeFile', async () => { throw cause }) + syncBuiltinESMExports() + t.after(() => { t.mock.restoreAll(); syncBuiltinESMExports() }) + for (const stale of [false, true]) { + const page = { pageInfo, outputRecords: [] } + /** @type {PageOutputCache} */ + const outputCache = new Map(stale ? [[filepath, { hash: 'stale', metadata: 'stale' }]] : []) + await assert.rejects(writePageOutputs({ + dest, + pageFilePath: join(dest, 'index.html'), + page, + pageOutputs: [{ outputName: 'data.txt', content: 'not written' }], + outputCache, + }), error => error === cause) + assert.deepEqual(page.outputRecords, []) + assert.equal(outputCache.size, 0) + } + assert.equal(writes.mock.callCount(), 2) +}) + +test('writer streams each output to disk and retains successful records and cache after iterator failure', async t => { + const dest = await mkdtemp(join(tmpdir(), 'domstack-page-output-iterator-')) + t.after(() => rm(dest, { recursive: true, force: true })) + /** @type {PageOutputCache} */ + const outputCache = new Map() + const page = { pageInfo, outputRecords: /** @type {DomstackManifestRecord[]} */ ([]) } + const filepath = join(dest, 'first.txt') + const cause = new Error('iterator failed') + let closed = false + async function * pageOutputs () { + try { + yield { outputName: 'first.txt', content: 'first' } + assert.equal(await readFile(filepath, 'utf8'), 'first') + assert.deepEqual(page.outputRecords.map(output => output.outputRelname), ['first.txt']) + assert.equal(outputCache.has(filepath), true) + throw cause + } finally { + closed = true + } + } + await assert.rejects(writePageOutputs({ + dest, + pageFilePath: join(dest, 'index.html'), + page, + pageOutputs: pageOutputs(), + outputCache, + }), error => error === cause) + assert.equal(closed, true) + assert.deepEqual(page.outputRecords.map(output => output.outputRelname), ['first.txt']) + assert.deepEqual([...outputCache.keys()], [filepath]) +}) diff --git a/lib/build-pages/page-builders/page-writer.js b/lib/build-pages/page-builders/page-writer.js index 279a72b6..99c8f1b7 100644 --- a/lib/build-pages/page-builders/page-writer.js +++ b/lib/build-pages/page-builders/page-writer.js @@ -2,13 +2,14 @@ * @import { PageInfo } from '../../identify-pages.js' * @import { PageData as PageDataClass } from '../page-data.js' * @import { DomstackManifestRecord } from '../../domstack-manifest/index.js' - * @import { AdditionalOutputsFunction } from '../additional-outputs.js' + * @import { PageOutputsFunction } from '../page-outputs.js' + * @import { PageOutputCache } from './page-output-writer.js' */ import { join } from 'path' import { writeFile, mkdir } from 'fs/promises' import { createDomstackManifestRecord } from '../../domstack-manifest/index.js' -import { writeAdditionalOutputs } from './additional-output-writer.js' +import { writePageOutputs } from './page-output-writer.js' /** * @typedef {Object} BuilderOptions @@ -76,7 +77,7 @@ import { writeAdditionalOutputs } from './additional-output-writer.js' * @typedef PageBuilderResult * @property {Partial} vars - Any variables resolved by the builder * @property {InternalPageFunction} pageLayout - The function that returns the rendered page - * @property {AdditionalOutputsFunction | undefined} [additionalOutputs] - Optional build-only additional-output hook. + * @property {PageOutputsFunction | undefined} [pageOutputs] - Optional build-only page-output hook. */ /** @@ -99,50 +100,48 @@ import { writeAdditionalOutputs } from './additional-output-writer.js' * @param {object} params * @param {string} params.dest - The dest folder. * @param {PageData} params.page - The PageInfo object of the current page - * @param {((output: DomstackManifestRecord) => void) | undefined} [params.onAdditionalOutput] - Record emitted sidecars even if a later yield fails. + * @param {PageOutputCache | undefined} [params.outputCache] * @returns {Promise<{ pageFilePath: string, outputs: DomstackManifestRecord[] }>} */ export async function pageWriter ({ dest, page, - onAdditionalOutput, + outputCache, }) { if (!page.pageInfo) throw new Error('Uninitialzied page detected') const pageDir = join(dest, page.pageInfo.path) const pageFilePath = join(pageDir, page.pageInfo.outputName) + page.outputRecords.length = 0 const formattedPageOutput = await page.renderFullPage() - const additionalOutputs = await writeAdditionalOutputs({ + await writePageOutputs({ dest, pageFilePath, - pageInfo: page.pageInfo, - additionalOutputs: page.collectAdditionalOutputs(), - onOutput: onAdditionalOutput, + page, + pageOutputs: page.collectPageOutputs(), + outputCache, }) const vars = page.vars const manifestRole = extractManifestRole(vars) await mkdir(pageDir, { recursive: true }) await writeFile(pageFilePath, formattedPageOutput) - /** @type {DomstackManifestRecord[]} */ - const outputs = [ - createDomstackManifestRecord({ - dest, - filepath: pageFilePath, - outputRelname: page.pageInfo.outputRelname, - kind: 'page', + page.outputRecords.unshift(createDomstackManifestRecord({ + dest, + filepath: pageFilePath, + outputRelname: page.pageInfo.outputRelname, + kind: 'page', + url: page.pageInfo.url, + sourceRelname: page.pageInfo.pageFile.relname, + pagePath: page.pageInfo.path, + pageUrl: page.pageInfo.url, + pageVars: copyPageVars(vars), + manifestRole, + page: { + path: page.pageInfo.path, url: page.pageInfo.url, - sourceRelname: page.pageInfo.pageFile.relname, - pagePath: page.pageInfo.path, - pageUrl: page.pageInfo.url, - pageVars: copyPageVars(vars), - manifestRole, - page: { - path: page.pageInfo.path, - url: page.pageInfo.url, - }, - }), - ] + }, + })) // Generate meta.json with worker mappings if page has workers if (page.pageInfo?.workers) { @@ -163,7 +162,7 @@ export async function pageWriter ({ const workersFilePath = join(pageDir, 'workers.json') const workersContent = JSON.stringify(workerMappings, null, 2) await writeFile(workersFilePath, workersContent) - outputs.push(createDomstackManifestRecord({ + page.outputRecords.push(createDomstackManifestRecord({ dest, filepath: workersFilePath, outputRelname: join(page.pageInfo.path, 'workers.json'), @@ -175,9 +174,7 @@ export async function pageWriter ({ } } - outputs.push(...additionalOutputs) - - return { pageFilePath, outputs } + return { pageFilePath, outputs: page.outputRecords } } /** diff --git a/lib/build-pages/page-data-additional-outputs.test.js b/lib/build-pages/page-data-page-outputs.test.js similarity index 59% rename from lib/build-pages/page-data-additional-outputs.test.js rename to lib/build-pages/page-data-page-outputs.test.js index a8a7e26e..d0214fd3 100644 --- a/lib/build-pages/page-data-additional-outputs.test.js +++ b/lib/build-pages/page-data-page-outputs.test.js @@ -2,14 +2,16 @@ * @import { PageInfo } from '../identify-pages.js' * @import { ResolvedLayout } from './page-data.js' * @import { TestContext } from 'node:test' + * @import { PageOutputCache } from './page-builders/page-output-writer.js' */ import { test } from 'node:test' import assert from 'node:assert/strict' -import { mkdtemp, writeFile, rm } from 'node:fs/promises' +import { mkdtemp, readFile, writeFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { PageData, resolveLayout } from './page-data.js' import { identifyPages } from '../identify-pages.js' +import { pageWriter } from './page-builders/page-writer.js' /** * @param {TestContext} t @@ -35,7 +37,7 @@ async function fixture (t, { module, companion, extension = 'mjs' } = {}) { await writeFile(join(dir, 'page.vars.mjs'), companion) pageInfo.pageVars = file('page.vars.mjs') } - const pd = new PageData({ pageInfo, globalVars: { layout: 'inner', additionalOutputs: () => { throw new Error('global vars are not providers') } }, globalStyle: undefined, globalClient: undefined, defaultStyle: null, defaultClient: null, builderOptions: {} }) + const pd = new PageData({ pageInfo, globalVars: { layout: 'inner', pageOutputs: () => { throw new Error('global vars are not providers') } }, globalStyle: undefined, globalClient: undefined, defaultStyle: null, defaultClient: null, builderOptions: {} }) /** @type {Record>} */ const layouts = { outer: { name: 'outer', render: ({ children }) => children, vars: {}, layoutStylePath: null, layoutClientPath: null }, @@ -49,7 +51,7 @@ test('explicit collection runs outer -> inner -> page with renderer subscription module: `export const vars = { dataDeps: ['pageKey'] } export let hookCalls = 0 export default ({ data }) => data.pageKey - export const additionalOutputs = ({ page, vars, data }) => { + export const pageOutputs = ({ page, vars, data }) => { hookCalls++ return { outputName: 'page.txt', content: data.pageKey + data.companionKey + page.url } }`, @@ -60,31 +62,36 @@ test('explicit collection runs outer -> inner -> page with renderer subscription for (const name of ['outer', 'inner']) { const path = join(dir, `${name}.layout.mjs`) await writeFile(path, `export const vars = { dataDeps: ['${name}Key'] }; export default ({ children }) => children; - export const additionalOutputs = ({ data }) => ({ outputName: '${name}.txt', content: data.${name}Key });`) + export const pageOutputs = ({ data }) => ({ outputName: '${name}.txt', content: data.${name}Key });`) const layout = layouts[name] assert.ok(layout) const resolved = await resolveLayout(path) Object.assign(layout, resolved, { parentLayout: name === 'inner' ? 'outer' : undefined }) - const hook = layout.additionalOutputs + const hook = layout.pageOutputs assert.ok(hook) - layout.additionalOutputs = params => { + layout.pageOutputs = params => { calls.push(name) assert.deepEqual(Object.keys(params.data), [`${name}Key`]) assert.throws(() => params.data.pageKey, /undeclared/) return hook(params) } } - await assert.rejects(Array.fromAsync(pd.collectAdditionalOutputs()), /initialized/) + await assert.rejects(Array.fromAsync(pd.collectPageOutputs()), /initialized/) await pd.init({ layouts }) - await assert.rejects(Array.fromAsync(pd.collectAdditionalOutputs()), /outer.*not available/) + await assert.rejects(Array.fromAsync(pd.collectPageOutputs()), /outer.*not available/) assert.deepEqual(pd.dataDeps, ['companionKey', 'innerKey', 'outerKey', 'pageKey']) pd.setGlobalData({ pageKey: 'p', companionKey: 'c', outerKey: 'o', innerKey: 'i', secret: 'hidden' }) + const outputRecords = pd.outputRecords + assert.deepEqual(outputRecords, []) await pd.renderInnerPage() + assert.deepEqual(pd.outputRecords, []) await pd.renderFullPage() + assert.equal(pd.outputRecords, outputRecords) + assert.deepEqual(pd.outputRecords, []) assert.deepEqual(calls, []) const pageModule = await import(pd.pageInfo.pageFile.filepath) assert.equal(pageModule.hookCalls, 0) - const outputs = await Array.fromAsync(pd.collectAdditionalOutputs()) + const outputs = await Array.fromAsync(pd.collectPageOutputs()) assert.equal(pageModule.hookCalls, 1) assert.deepEqual(calls, ['outer', 'inner']) assert.deepEqual(outputs.map(({ outputName, content }) => ({ outputName, content })), [ @@ -92,13 +99,84 @@ test('explicit collection runs outer -> inner -> page with renderer subscription ]) assert.deepEqual(outputs[0]?.provenance, { kind: 'layout', source: join(dir, 'outer.layout.mjs'), layoutName: 'outer' }) assert.deepEqual(outputs[2]?.provenance, { kind: 'page', source: join(dir, 'page.mjs') }) + assert.deepEqual(pd.outputRecords, []) +}) + +test('page writer records HTML and streamed sidecars on the actual PageData handle', async t => { + const { pd, layouts, dir } = await fixture(t, { + module: `export const events = [] + export default () => { events.push('render'); return '

Page

' } + export async function * pageOutputs () { + events.push('collect') + yield { outputName: 'data.json', content: '{"ok":true}' } + }`, + }) + await pd.init({ layouts }) + const pageModule = await import(pd.pageInfo.pageFile.filepath) + const outputRecords = pd.outputRecords + /** @type {PageOutputCache} */ + const outputCache = new Map() + const dest = join(dir, 'public') + const result = await pageWriter({ dest, page: pd, outputCache }) + assert.deepEqual(pageModule.events, ['render', 'collect']) + assert.equal(pd.outputRecords, outputRecords) + assert.equal(result.outputs, outputRecords) + assert.deepEqual(outputRecords.map(output => [output.kind, output.outputRelname]), [ + ['page', 'index.html'], ['page-output', 'data.json'], + ]) + for (const output of outputRecords) { + assert.equal(output.sourceRelname, 'page.mjs') + assert.equal(output.pageUrl, '/') + assert.equal('content' in output, false) + } + assert.equal(await readFile(result.pageFilePath, 'utf8'), '

Page

') + assert.equal(await readFile(join(dest, 'data.json'), 'utf8'), '{"ok":true}') + assert.deepEqual([...outputCache.keys()], [join(dest, 'data.json')]) + const cached = outputCache.get(join(dest, 'data.json')) + const rebuilt = await pageWriter({ dest, page: pd, outputCache }) + assert.equal(rebuilt.outputs, outputRecords) + assert.equal(outputRecords.length, 2) + assert.deepEqual(pageModule.events, ['render', 'collect', 'render', 'collect']) + assert.equal(outputCache.get(join(dest, 'data.json')), cached) +}) + +test('page writer retains emitted sidecars on PageData when a provider fails mid-stream', async t => { + const { pd, layouts, dir } = await fixture(t, { + module: "export default () => '

Page

'; export const pageOutputs = () => { throw new Error('page must not run') }", + }) + const dest = join(dir, 'public') + const filepath = join(dest, 'first.txt') + /** @type {PageOutputCache} */ + const outputCache = new Map() + const { outer } = layouts + assert.ok(outer) + const cause = new Error('iterator failed') + outer.pageOutputs = async function * () { + yield { outputName: 'first.txt', content: 'first' } + assert.equal(await readFile(filepath, 'utf8'), 'first') + assert.deepEqual(pd.outputRecords.map(output => output.outputRelname), ['first.txt']) + assert.equal(outputCache.has(filepath), true) + throw cause + } + await pd.init({ layouts }) + const outputRecords = pd.outputRecords + await assert.rejects(pageWriter({ dest, page: pd, outputCache }), error => { + assert.ok(error instanceof Error) + assert.match(error.message, /pageOutputs for page "page.mjs" from layout "outer" failed: Invalid pageOutputs.*iterator failed/) + assert.ok(error.cause instanceof Error) + assert.equal(error.cause.cause, cause) + return true + }) + assert.equal(pd.outputRecords, outputRecords) + assert.deepEqual(outputRecords.map(output => [output.kind, output.outputRelname]), [['page-output', 'first.txt']]) + assert.deepEqual([...outputCache.keys()], [filepath]) }) test('collection invokes providers lazily in outer -> inner -> page order', async t => { const { pd, layouts } = await fixture(t, { module: `export default () => '' export let hookCalls = 0 - export function additionalOutputs () { + export function pageOutputs () { hookCalls++ return [{ outputName: 'page.txt', content: 'page' }] }`, @@ -108,7 +186,7 @@ test('collection invokes providers lazily in outer -> inner -> page order', asyn for (const name of ['outer', 'inner']) { const layout = layouts[name] assert.ok(layout) - layout.additionalOutputs = () => { + layout.pageOutputs = () => { events.push(`${name}:called`) return (async function * () { try { @@ -124,7 +202,7 @@ test('collection invokes providers lazily in outer -> inner -> page order', asyn } await pd.init({ layouts }) const pageModule = await import(pd.pageInfo.pageFile.filepath) - const outputs = pd.collectAdditionalOutputs() + const outputs = pd.collectPageOutputs() assert.equal(outputs[Symbol.asyncIterator](), outputs) assert.deepEqual(events, []) assert.equal(pageModule.hookCalls, 0) @@ -148,14 +226,14 @@ test('closing collection runs the active provider finally and never invokes late const { pd, layouts } = await fixture(t, { module: `export default () => '' export let hookCalls = 0 - export function additionalOutputs () { hookCalls++; return [] }`, + export function pageOutputs () { hookCalls++; return [] }`, }) /** @type {string[]} */ const events = [] const { outer, inner } = layouts assert.ok(outer) assert.ok(inner) - outer.additionalOutputs = () => { + outer.pageOutputs = () => { events.push('outer:called') return (async function * () { try { @@ -169,13 +247,13 @@ test('closing collection runs the active provider finally and never invokes late })() } inner.vars = { dataDeps: ['unready'] } - inner.additionalOutputs = () => { events.push('inner:called'); return [] } + inner.pageOutputs = () => { events.push('inner:called'); return [] } await pd.init({ layouts }) const pageModule = await import(pd.pageInfo.pageFile.filepath) - const unopened = pd.collectAdditionalOutputs() + const unopened = pd.collectPageOutputs() await unopened.return() assert.deepEqual(events, []) - const outputs = pd.collectAdditionalOutputs() + const outputs = pd.collectPageOutputs() if (close === 'return') { assert.equal((await outputs.next()).value?.outputName, 'first.txt') assert.deepEqual(events, ['outer:called']) @@ -196,13 +274,13 @@ test('closing collection runs the active provider finally and never invokes late test('streamed provider errors retain page context and causes after earlier records', async t => { const { pd, layouts } = await fixture(t, { - module: "export default () => ''; export const additionalOutputs = () => { throw new Error('page must not run') }", + module: "export default () => ''; export const pageOutputs = () => { throw new Error('page must not run') }", }) const cause = new Error('iterator failed') let closed = false const { outer } = layouts assert.ok(outer) - outer.additionalOutputs = async function * () { + outer.pageOutputs = async function * () { try { yield { outputName: 'first.txt', content: 'first' } throw cause @@ -211,12 +289,12 @@ test('streamed provider errors retain page context and causes after earlier reco } } await pd.init({ layouts }) - const outputs = pd.collectAdditionalOutputs() + const outputs = pd.collectPageOutputs() assert.equal((await outputs.next()).value?.outputName, 'first.txt') assert.equal(closed, false) await assert.rejects(outputs.next(), error => { assert.ok(error instanceof Error) - assert.match(error.message, /additionalOutputs for page "page.mjs" from layout "outer" failed: Invalid additionalOutputs.*iterator failed/) + assert.match(error.message, /pageOutputs for page "page.mjs" from layout "outer" failed: Invalid pageOutputs.*iterator failed/) assert.ok(error.cause instanceof Error) assert.equal(error.cause.cause, cause) return true @@ -230,11 +308,11 @@ test('markdown companion gets a frozen narrow source handle and subscribed data' import assert from 'node:assert/strict' export default { dataDeps: ['selected'] } export const dataDeps = ['secret'] // A named export is not a subscription declaration. - export async function additionalOutputs ({ page, vars, data }) { + export async function pageOutputs ({ page, vars, data }) { assert.equal(Object.isFrozen(page), true) assert.equal(Object.isFrozen(page.pageFile), true) assert.equal(Object.isFrozen(vars), true) - for (const key of ['data', 'pageInfo', 'renderInnerPage', 'renderFullPage', 'setGlobalData', 'collectAdditionalOutputs']) assert.equal(key in page, false) + for (const key of ['data', 'pageInfo', 'renderInnerPage', 'renderFullPage', 'setGlobalData', 'collectPageOutputs', 'outputRecords']) assert.equal(key in page, false) assert.throws(() => { page.pageFile.filepath = '/other.md' }, TypeError) assert.throws(() => data.secret, /undeclared/) const read = page.readMarkdownContent @@ -242,9 +320,9 @@ test('markdown companion gets a frozen narrow source handle and subscribed data' }` }) await pd.init({ layouts }) - await assert.rejects(Array.fromAsync(pd.collectAdditionalOutputs()), /companion.*not available/) + await assert.rejects(Array.fromAsync(pd.collectPageOutputs()), /companion.*not available/) pd.setGlobalData({ selected: 'selected:', secret: 'hidden' }) - assert.deepEqual(await Array.fromAsync(pd.collectAdditionalOutputs()), [{ outputName: 'body.md', content: 'selected:\n# Body\n', provenance: { kind: 'companion', source: join(dir, 'page.vars.mjs') } }]) + assert.deepEqual(await Array.fromAsync(pd.collectPageOutputs()), [{ outputName: 'body.md', content: 'selected:\n# Body\n', provenance: { kind: 'companion', source: join(dir, 'page.vars.mjs') } }]) }) test('JS and TS page providers work and cannot read markdown or undeclared data', async t => { @@ -253,7 +331,7 @@ test('JS and TS page providers work and cannot read markdown or undeclared data' extension, module: ` import assert from 'node:assert/strict' export default () => 'page' - export async function additionalOutputs ({ page, data }) { + export async function pageOutputs ({ page, data }) { await assert.rejects(page.readMarkdownContent(), /only.*markdown/) assert.throws(() => data.secret, /undeclared/) return [] @@ -261,7 +339,7 @@ test('JS and TS page providers work and cannot read markdown or undeclared data' }) await pd.init({ layouts }) pd.setGlobalData({ secret: 'hidden' }) - assert.deepEqual(await Array.fromAsync(pd.collectAdditionalOutputs()), []) + assert.deepEqual(await Array.fromAsync(pd.collectPageOutputs()), []) } }) @@ -270,7 +348,7 @@ test('companions also provide outputs for JS and HTML pages', async t => { const { pd, layouts } = await fixture(t, { ...(type === 'js' ? { module: "export const vars = { dataDeps: ['selected'] }; export default () => ''" } : {}), companion: `export default { dataDeps: ['selected'] }; - export const additionalOutputs = async ({ data }) => [{ outputName: 'extra.txt', content: data.selected }]`, + export const pageOutputs = async ({ data }) => [{ outputName: 'extra.txt', content: data.selected }]`, }) if (type === 'html') { pd.pageInfo.type = 'html' @@ -278,39 +356,68 @@ test('companions also provide outputs for JS and HTML pages', async t => { } await pd.init({ layouts }) pd.setGlobalData({ selected: type }) - const outputs = await Array.fromAsync(pd.collectAdditionalOutputs()) + const outputs = await Array.fromAsync(pd.collectPageOutputs()) assert.equal(outputs[0]?.content, type) assert.equal(outputs[0]?.provenance.kind, 'companion') } }) -test('conflicting page providers and invalid exports identify their sources', async t => { - const { pd, layouts } = await fixture(t, { module: "export default () => ''; export const additionalOutputs = () => []", companion: 'export const additionalOutputs = () => []' }) - await assert.rejects(pd.init({ layouts }), /page.mjs.*page.vars.mjs.*both export additionalOutputs/) - for (const options of [{ module: "export default () => ''; export const additionalOutputs = 1" }, { companion: 'export const additionalOutputs = null' }]) { +test('page module outputs take precedence over companion outputs with a warning and additive layouts', async t => { + const warn = t.mock.method(console, 'warn', () => {}) + for (const extension of ['mjs', 'ts']) { + const { pd, layouts, dir } = await fixture(t, { + extension, + module: "export default () => ''; export const pageOutputs = () => ({ outputName: 'page.txt', content: 'page' })", + companion: "export const pageOutputs = () => { throw new Error('ignored companion must not run') }", + }) + for (const name of ['outer', 'inner']) { + const layout = layouts[name] + assert.ok(layout) + layout.pageOutputs = () => ({ outputName: `${name}.txt`, content: name }) + } + const warningCount = warn.mock.callCount() + await pd.init({ layouts }) + await pd.init({ layouts }) + assert.equal(warn.mock.callCount(), warningCount + 1) + const message = warn.mock.calls.at(-1)?.arguments[0] + assert.ok(message.includes(join(dir, `page.${extension}`))) + assert.ok(message.includes(join(dir, 'page.vars.mjs'))) + assert.match(message, /both export pageOutputs; using the page module export and ignoring the companion export/) + const outputs = await Array.fromAsync(pd.collectPageOutputs()) + assert.deepEqual(outputs.map(({ outputName, content }) => ({ outputName, content })), [ + { outputName: 'outer.txt', content: 'outer' }, + { outputName: 'inner.txt', content: 'inner' }, + { outputName: 'page.txt', content: 'page' }, + ]) + assert.deepEqual(outputs[2]?.provenance, { kind: 'page', source: join(dir, `page.${extension}`) }) + } +}) + +test('invalid output exports identify their sources', async t => { + for (const options of [{ module: "export default () => ''; export const pageOutputs = 1" }, { companion: 'export const pageOutputs = null' }]) { const { pd, layouts } = await fixture(t, options) - await assert.rejects(pd.init({ layouts }), /additionalOutputs.*page.*must be a function/) + await assert.rejects(pd.init({ layouts }), /pageOutputs.*page.*must be a function/) } const { dir } = await fixture(t) const path = join(dir, 'bad.layout.mjs') - await writeFile(path, "export default () => ''; export const additionalOutputs = {}") - await assert.rejects(resolveLayout(path), /additionalOutputs.*bad.layout.mjs.*function/) + await writeFile(path, "export default () => ''; export const pageOutputs = {}") + await assert.rejects(resolveLayout(path), /pageOutputs.*bad.layout.mjs.*function/) }) test('generated pages skip page, companion and all layout hooks', async t => { - const { pd, layouts } = await fixture(t, { module: "throw new Error('must not import source module')", companion: 'export const additionalOutputs = 123' }) + const { pd, layouts } = await fixture(t, { module: "throw new Error('must not import source module')", companion: 'export const pageOutputs = 123' }) pd.pageInfo.generated = { pagesFile: { pagesFile: pd.pageInfo.pageFile, path: '', name: 'generated' }, children: 'generated' } for (const layout of Object.values(layouts)) { layout.vars = { dataDeps: ['unready'] } - layout.additionalOutputs = () => { throw new Error('generated hook ran') } + layout.pageOutputs = () => { throw new Error('generated hook ran') } } await pd.init({ layouts }) - assert.deepEqual(await pd.collectAdditionalOutputs().next(), { value: undefined, done: true }) + assert.deepEqual(await pd.collectPageOutputs().next(), { value: undefined, done: true }) }) test('discovery excludes ignored providers and disabled drafts but enabled drafts collect outputs', async t => { const { pd, layouts, dir } = await fixture(t, { - companion: "export const additionalOutputs = () => ({ outputName: 'draft.txt', content: 'draft output' })", + companion: "export const pageOutputs = () => ({ outputName: 'draft.txt', content: 'draft output' })", }) await rm(pd.pageInfo.pageFile.filepath) await writeFile(join(dir, 'page.draft.md'), '# Draft') @@ -325,16 +432,16 @@ test('discovery excludes ignored providers and disabled drafts but enabled draft pd.pageInfo = pageInfo await pd.init({ layouts }) pd.setGlobalData({}) - assert.equal((await Array.fromAsync(pd.collectAdditionalOutputs()))[0]?.content, 'draft output') + assert.equal((await Array.fromAsync(pd.collectPageOutputs()))[0]?.content, 'draft output') }) test('page hook failures include page and provider context, and no providers is valid', async t => { for (const body of ["throw new Error('hook failed')", "return 'bare string'", "return (async function * () { throw new Error('iterator failed') })()"]) { - const { pd, layouts } = await fixture(t, { companion: `export function additionalOutputs () { ${body} }` }) + const { pd, layouts } = await fixture(t, { companion: `export function pageOutputs () { ${body} }` }) await pd.init({ layouts }) - await assert.rejects(Array.fromAsync(pd.collectAdditionalOutputs()), /additionalOutputs for page "page.md" from companion.*page.vars.mjs.*(failed|Record)/) + await assert.rejects(Array.fromAsync(pd.collectPageOutputs()), /pageOutputs for page "page.md" from companion.*page.vars.mjs.*(failed|Record)/) } const { pd, layouts } = await fixture(t) await pd.init({ layouts }) - assert.deepEqual(await Array.fromAsync(pd.collectAdditionalOutputs()), []) + assert.deepEqual(await Array.fromAsync(pd.collectPageOutputs()), []) }) diff --git a/lib/build-pages/page-data.js b/lib/build-pages/page-data.js index cf98d73c..5caa2525 100644 --- a/lib/build-pages/page-data.js +++ b/lib/build-pages/page-data.js @@ -1,7 +1,8 @@ /** * @import { PageInfo } from '../identify-pages.js' + * @import { DomstackManifestRecord } from '../domstack-manifest/index.js' * @import { BuilderOptions, InternalPageFunction } from './page-builders/page-writer.js' - * @import { AdditionalOutputsFunction, AdditionalOutputProvenance, CollectedAdditionalOutput } from './additional-outputs.js' + * @import { PageOutputsFunction, PageOutputProvenance, CollectedPageOutput } from './page-outputs.js' */ import { readFile } from 'node:fs/promises' @@ -12,7 +13,7 @@ import { createSubscribedData, extractDataDeps } from './data-deps.js' import { DomStackDataError } from '../helpers/domstack-error.js' import pretty from 'pretty' import { resolveLayoutChain } from './resolve-layout-chain.js' -import { createAdditionalOutputsPage, normalizeAdditionalOutputs, validateAdditionalOutputsHook } from './additional-outputs.js' +import { createPageOutputsPage, normalizePageOutputs, validatePageOutputsHook } from './page-outputs.js' /** * @typedef {Object} WorkerFiles @@ -27,10 +28,10 @@ import { createAdditionalOutputsPage, normalizeAdditionalOutputs, validateAdditi * @template [V=string] V - The return type of the layout function (defaults to string) * @template {object} [D=Record] - Declared global data. * @param {string} layoutPath - The string path to the layout ESM module. - * @returns {Promise<{ render: InternalLayoutFunction, vars: Partial, parentLayout: string | undefined, additionalOutputs: AdditionalOutputsFunction | undefined, source: string }>} The resolved layout module exports. + * @returns {Promise<{ render: InternalLayoutFunction, vars: Partial, parentLayout: string | undefined, pageOutputs: PageOutputsFunction | undefined, source: string }>} The resolved layout module exports. */ export async function resolveLayout (layoutPath) { - const { default: layout, vars, parentLayout, additionalOutputs } = await import(layoutPath) + const { default: layout, vars, parentLayout, pageOutputs } = await import(layoutPath) if (typeof layout !== 'function') throw new TypeError(`Layout "${layoutPath}" must export a default render function`) if (parentLayout !== undefined && (typeof parentLayout !== 'string' || !parentLayout.trim())) { throw new TypeError(`Layout "${layoutPath}" parentLayout must be a non-empty string`) @@ -40,7 +41,7 @@ export async function resolveLayout (layoutPath) { render: layout, parentLayout, source: layoutPath, - additionalOutputs: validateAdditionalOutputsHook(additionalOutputs, layoutPath), + pageOutputs: validatePageOutputsHook(pageOutputs, layoutPath), vars: /** @type {Partial} */ (await resolveVarsExport(vars, 'Layout vars')), } } @@ -132,7 +133,7 @@ export async function resolveLayout (layoutPath) { * @typedef ResolvedLayout * @property {InternalLayoutFunction} render - The layout function * @property {Partial} [vars] - Variables exported by the layout module. - * @property {AdditionalOutputsFunction | undefined} [additionalOutputs] - Explicit output-phase hook. + * @property {PageOutputsFunction | undefined} [pageOutputs] - Explicit output-phase hook. * @property {string} [source] - Layout module path for diagnostics. * @property {string} name - The name of the layout * @property {string | undefined} [parentLayout] - Name of the optional outer layout. @@ -149,6 +150,7 @@ export async function resolveLayout (layoutPath) { */ export class PageData { /** @type {PageInfo} */ pageInfo + /** @type {DomstackManifestRecord[]} Successfully emitted files, including partial builds; never populated by rendering alone. */ outputRecords = [] /** @type {ResolvedLayout | null | undefined} */ layout /** @type {ResolvedLayout[]} Each parent has its own render and data contracts. */ layoutChain = [] /** @type {Partial} */ globalVars @@ -157,7 +159,7 @@ export class PageData { /** @type {Partial | null} */ builderVars = null /** @type {string[]} Union of the page and entire layout chain, for output invalidation. */ dataDeps = [] /** @type {string[]} */ #pageDataDeps = [] - /** @type {{ hook: AdditionalOutputsFunction, provenance: AdditionalOutputProvenance } | undefined} */ #pageAdditionalOutputs + /** @type {{ hook: PageOutputsFunction, provenance: PageOutputProvenance } | undefined} */ #pageOutputs /** @type {Map }>} */ #layoutSubscriptions = new Map() /** @type {string[]} */ styles = [] /** @type {string[]} */ scripts = [] @@ -320,20 +322,20 @@ export class PageData { const built = await builder({ pageInfo, options: this.builderOptions }) const { vars: builderVars } = built if (!pageInfo.generated) { - const moduleHook = type === 'js' ? built.additionalOutputs : undefined - const companionHook = pageVars?.filepath - ? validateAdditionalOutputsHook((await import(pageVars.filepath)).additionalOutputs, pageVars.filepath) + const pageModuleOutputs = type === 'js' ? built.pageOutputs : undefined + const varsCompanionOutputs = pageVars?.filepath + ? validatePageOutputsHook((await import(pageVars.filepath)).pageOutputs, pageVars.filepath) : undefined - if (moduleHook && companionHook) { - throw new Error(`Page "${pageInfo.pageFile.filepath}" and companion "${pageVars?.filepath}" both export additionalOutputs; only one page-level provider is allowed`) + if (pageModuleOutputs && varsCompanionOutputs) { + console.warn(`Page "${pageInfo.pageFile.filepath}" and companion "${pageVars?.filepath}" both export pageOutputs; using the page module export and ignoring the companion export`) } - const hook = moduleHook ?? companionHook + const hook = pageModuleOutputs ?? varsCompanionOutputs if (hook) { - this.#pageAdditionalOutputs = { + this.#pageOutputs = { hook, provenance: { - kind: moduleHook ? 'page' : 'companion', - source: moduleHook ? pageInfo.pageFile.filepath : /** @type {string} */ (pageVars?.filepath), + kind: pageModuleOutputs ? 'page' : 'companion', + source: pageModuleOutputs ? pageInfo.pageFile.filepath : /** @type {string} */ (pageVars?.filepath), }, } } @@ -396,34 +398,34 @@ export class PageData { /** * Run hooks lazily as the output phase consumes each record. * Layouts run outermost first, followed by the single page-level provider. - * @returns {AsyncGenerator} + * @returns {AsyncGenerator} */ - async * collectAdditionalOutputs () { - if (!this.#initialized) throw new Error('Must be initialized before collecting additionalOutputs') + async * collectPageOutputs () { + if (!this.#initialized) throw new Error('Must be initialized before collecting pageOutputs') if (this.pageInfo.generated) return // Capture source metadata so rebinding the reader cannot change its source. const sourceInfo = { ...this.pageInfo, pageFile: { ...this.pageInfo.pageFile } } - const page = createAdditionalOutputsPage(sourceInfo, async () => { + const page = createPageOutputsPage(sourceInfo, async () => { if (sourceInfo.type !== 'md') throw new Error('Markdown content can only be read from markdown pages') return parseMdFileContents(await readFile(sourceInfo.pageFile.filepath, 'utf8')).markdownContent }) const pageData = this /** - * @param {AdditionalOutputsFunction} hook - * @param {AdditionalOutputProvenance} provenance + * @param {PageOutputsFunction} hook + * @param {PageOutputProvenance} provenance * @param {() => object} getData - * @returns {AsyncGenerator} + * @returns {AsyncGenerator} */ const collect = async function * (hook, provenance, getData) { try { - yield * normalizeAdditionalOutputs(hook({ page, vars: pageData.vars, data: getData() }), provenance) + yield * normalizePageOutputs(hook({ page, vars: pageData.vars, data: getData() }), provenance) } catch (cause) { - throw new Error(`additionalOutputs for page "${pageData.pageInfo.pageFile.relname}" from ${provenance.kind} "${provenance.source}" failed: ${cause instanceof Error ? cause.message : String(cause)}`, { cause }) + throw new Error(`pageOutputs for page "${pageData.pageInfo.pageFile.relname}" from ${provenance.kind} "${provenance.source}" failed: ${cause instanceof Error ? cause.message : String(cause)}`, { cause }) } } for (const layout of this.layoutChain) { const source = layout.source ?? layout.name - const hook = validateAdditionalOutputsHook(layout.additionalOutputs, source) + const hook = validatePageOutputsHook(layout.pageOutputs, source) if (!hook) continue yield * collect(hook, { kind: 'layout', source, layoutName: layout.name }, () => { const subscription = this.#layoutSubscriptions.get(layout.name) @@ -431,8 +433,8 @@ export class PageData { return subscription?.data ?? Object.freeze({}) }) } - if (this.#pageAdditionalOutputs) { - const { hook, provenance } = this.#pageAdditionalOutputs + if (this.#pageOutputs) { + const { hook, provenance } = this.#pageOutputs yield * collect(hook, provenance, () => this.data) } } diff --git a/lib/build-pages/page-outputs-types.test.ts b/lib/build-pages/page-outputs-types.test.ts new file mode 100644 index 00000000..4b82d6b3 --- /dev/null +++ b/lib/build-pages/page-outputs-types.test.ts @@ -0,0 +1,68 @@ +import type { + PageOutput, + PageOutputProvenance, + PageOutputsFunction, + PageOutputsFunctionParams, + PageOutputsPage, + PageOutputsResult, + CollectedPageOutput, + PageData, + DomstackManifestRecord, +} from '../../types.ts' +import { normalizePageOutputs } from './page-outputs.js' +import { writePageOutputs } from './page-builders/page-output-writer.js' +import type { PageOutputCache } from './page-builders/page-output-writer.js' + +// Compile-only assertions for the public type entry and the narrow hook contract. +export function checkPageOutputsTypes (pageData: PageData<{ title: string }>, page: PageOutputsPage) { + const output: PageOutput = { outputName: 'feed.json', content: '' } + const provenance: PageOutputProvenance = { kind: 'layout', source: 'base.layout.ts', layoutName: 'base' } + const collected: CollectedPageOutput = { ...output, provenance } + const result: PageOutputsResult = [output] + const hook: PageOutputsFunction<{ title: string }, { posts: string[] }> = async ({ page, vars, data }) => ({ + outputName: 'feed.json', content: vars.title + page.url + data.posts.join(','), + }) + const params: PageOutputsFunctionParams<{ title: string }, { posts: string[] }> = { page, vars: { title: 'title' }, data: { posts: [] } } + const outputs: AsyncGenerator = pageData.collectPageOutputs() + const normalized: AsyncGenerator = normalizePageOutputs(result, provenance) + const iterable: AsyncIterable = outputs + const outputRecords: DomstackManifestRecord[] = pageData.outputRecords + const outputCache: PageOutputCache = new Map() + const written: Promise = writePageOutputs({ + dest: 'public', + pageFilePath: 'public/index.html', + page: { pageInfo: pageData.pageInfo, outputRecords }, + pageOutputs: outputs, + outputCache, + }) + const uncached: Promise = writePageOutputs({ + dest: 'public', pageFilePath: 'public/index.html', page: pageData, pageOutputs: [output], + }) + // @ts-expect-error Emitted records describe files, not buffered output content. + const bufferedContent = outputRecords[0]?.content + // @ts-expect-error Hooks cannot access the internal emitted-file records. + const hookRecords = page.outputRecords + // @ts-expect-error Collection is streamed, not a promise of buffered records. + const buffered: Promise = pageData.collectPageOutputs() + const iterator: PageOutputsFunction = async function * () { yield output } + const promisedIterator: PageOutputsFunction = async () => (async function * () { yield output })() + // @ts-expect-error Bare strings are not hook results. + const badHook: PageOutputsFunction = () => 'html' + // @ts-expect-error Content must be a string. + const badOutput: PageOutput = { outputName: 'feed.json', content: {} } + // @ts-expect-error Page metadata is read-only. + page.url = '/other' + // @ts-expect-error Source metadata is read-only. + page.pageFile.filepath = '/other.md' + // @ts-expect-error Hooks cannot render pages. + page.renderFullPage() + // @ts-expect-error Hooks cannot access a PageData instance. + const bypass = page.pageInfo + // @ts-expect-error Global data is only accessible through the subscribed params.data. + const globalData = page.data + // @ts-expect-error Vars are read-only. + params.vars.title = 'other' + // @ts-expect-error Only declared data is available. + const secret = params.data.secret + return { hook, params, outputs, normalized, iterable, outputRecords, outputCache, written, uncached, bufferedContent, hookRecords, buffered, collected, result, iterator, promisedIterator, badHook, badOutput, bypass, globalData, secret } +} diff --git a/lib/build-pages/additional-outputs.js b/lib/build-pages/page-outputs.js similarity index 62% rename from lib/build-pages/additional-outputs.js rename to lib/build-pages/page-outputs.js index 4bb89873..f8f8668f 100644 --- a/lib/build-pages/additional-outputs.js +++ b/lib/build-pages/page-outputs.js @@ -1,25 +1,25 @@ /** * @import { PageInfo } from '../identify-pages.js' * - * @typedef {object} AdditionalOutput + * @typedef {object} PageOutput * @property {string} outputName * @property {string} content * - * @typedef {object} AdditionalOutputProvenance + * @typedef {object} PageOutputProvenance * @property {'page' | 'companion' | 'layout'} kind * @property {string} source - Provider module path (layout name when no path is available). * @property {string} [layoutName] * - * @typedef {AdditionalOutput & { provenance: AdditionalOutputProvenance }} CollectedAdditionalOutput - * @typedef {AdditionalOutput | AdditionalOutput[] | AsyncIterable} AdditionalOutputsResult - * @typedef {Readonly> & { readonly pageFile: Readonly, readonly readMarkdownContent: () => Promise }} AdditionalOutputsPage + * @typedef {PageOutput & { provenance: PageOutputProvenance }} CollectedPageOutput + * @typedef {PageOutput | PageOutput[] | AsyncIterable} PageOutputsResult + * @typedef {Readonly> & { readonly pageFile: Readonly, readonly readMarkdownContent: () => Promise }} PageOutputsPage */ /** * @template {Record} [T=Record] * @template {object} [D=Record] - * @typedef {object} AdditionalOutputsFunctionParams - * @property {AdditionalOutputsPage} page - Read-only source metadata, without rendering or global-data access. + * @typedef {object} PageOutputsFunctionParams + * @property {PageOutputsPage} page - Read-only source metadata, without rendering or global-data access. * @property {Readonly} vars * @property {D} data - The same subscriptions as this provider's renderer. */ @@ -27,34 +27,34 @@ /** * @template {Record} [T=Record] * @template {object} [D=Record] - * @callback AdditionalOutputsFunction - * @param {AdditionalOutputsFunctionParams} params - * @returns {AdditionalOutputsResult | Promise} + * @callback PageOutputsFunction + * @param {PageOutputsFunctionParams} params + * @returns {PageOutputsResult | Promise} */ /** * @param {unknown} hook * @param {string} source - * @returns {AdditionalOutputsFunction | undefined} + * @returns {PageOutputsFunction | undefined} */ -export function validateAdditionalOutputsHook (hook, source) { +export function validatePageOutputsHook (hook, source) { if (hook === undefined) return undefined - if (typeof hook !== 'function') throw new TypeError(`additionalOutputs in "${source}" must be a function`) - return /** @type {AdditionalOutputsFunction} */ (hook) + if (typeof hook !== 'function') throw new TypeError(`pageOutputs in "${source}" must be a function`) + return /** @type {PageOutputsFunction} */ (hook) } /** * Normalize one provider's result, preserving order and diagnostic context. * The writer checks each destination and writes it before requesting another record. * @param {unknown} result - * @param {AdditionalOutputProvenance} provenance - * @returns {AsyncGenerator} + * @param {PageOutputProvenance} provenance + * @returns {AsyncGenerator} */ -export async function * normalizeAdditionalOutputs (result, provenance) { +export async function * normalizePageOutputs (result, provenance) { let recordNumber = 0 /** * @param {unknown} record - * @returns {CollectedAdditionalOutput} + * @returns {CollectedPageOutput} */ const validate = (record) => { recordNumber++ @@ -75,16 +75,16 @@ export async function * normalizeAdditionalOutputs (result, provenance) { yield validate(resolved) } } catch (cause) { - throw new Error(`Invalid additionalOutputs from ${provenance.kind} "${provenance.source}": ${cause instanceof Error ? cause.message : String(cause)}`, { cause }) + throw new Error(`Invalid pageOutputs from ${provenance.kind} "${provenance.source}": ${cause instanceof Error ? cause.message : String(cause)}`, { cause }) } } /** * @param {PageInfo} info * @param {() => Promise} readMarkdownContent - Bound to the source, never to caller-supplied metadata. - * @returns {AdditionalOutputsPage} + * @returns {PageOutputsPage} */ -export function createAdditionalOutputsPage (info, readMarkdownContent) { +export function createPageOutputsPage (info, readMarkdownContent) { const { type, path, url, outputName, outputRelname, draft, pageFile } = info return Object.freeze({ type, diff --git a/lib/build-pages/additional-outputs.test.js b/lib/build-pages/page-outputs.test.js similarity index 63% rename from lib/build-pages/additional-outputs.test.js rename to lib/build-pages/page-outputs.test.js index d3f83f23..78a3dd43 100644 --- a/lib/build-pages/additional-outputs.test.js +++ b/lib/build-pages/page-outputs.test.js @@ -1,33 +1,33 @@ -/** @import { AdditionalOutputProvenance } from './additional-outputs.js' */ +/** @import { PageOutputProvenance } from './page-outputs.js' */ import { test } from 'node:test' import assert from 'node:assert/strict' -import { normalizeAdditionalOutputs, validateAdditionalOutputsHook } from './additional-outputs.js' +import { normalizePageOutputs, validatePageOutputsHook } from './page-outputs.js' -/** @type {AdditionalOutputProvenance} */ +/** @type {PageOutputProvenance} */ const provenance = { kind: 'page', source: '/src/page.ts' } const record = { outputName: 'feed.json', content: '' } -test('additionalOutputs normalizes records, arrays, promises and async iterables', async () => { +test('pageOutputs normalizes records, arrays, promises and async iterables', async () => { const expected = [{ ...record, provenance }] - assert.deepEqual(await Array.fromAsync(normalizeAdditionalOutputs(record, provenance)), expected) - assert.deepEqual(await Array.fromAsync(normalizeAdditionalOutputs(Promise.resolve([record]), provenance)), expected) + assert.deepEqual(await Array.fromAsync(normalizePageOutputs(record, provenance)), expected) + assert.deepEqual(await Array.fromAsync(normalizePageOutputs(Promise.resolve([record]), provenance)), expected) async function * records () { yield record; yield { ...record, outputName: 'second.json' } } - assert.deepEqual(await Array.fromAsync(normalizeAdditionalOutputs(Promise.resolve(records()), provenance)), [...expected, { ...record, outputName: 'second.json', provenance }]) - assert.deepEqual(await Array.fromAsync(normalizeAdditionalOutputs([], provenance)), []) + assert.deepEqual(await Array.fromAsync(normalizePageOutputs(Promise.resolve(records()), provenance)), [...expected, { ...record, outputName: 'second.json', provenance }]) + assert.deepEqual(await Array.fromAsync(normalizePageOutputs([], provenance)), []) async function * empty () {} - assert.deepEqual(await Array.fromAsync(normalizeAdditionalOutputs(empty(), provenance)), []) + assert.deepEqual(await Array.fromAsync(normalizePageOutputs(empty(), provenance)), []) }) -test('additionalOutputs rejects invalid hooks and records with source context', async () => { +test('pageOutputs rejects invalid hooks and records with source context', async () => { for (const hook of [null, true, {}, 'content']) { - assert.throws(() => validateAdditionalOutputsHook(hook, provenance.source), /additionalOutputs.*\/src\/page.ts.*function/) + assert.throws(() => validatePageOutputsHook(hook, provenance.source), /pageOutputs.*\/src\/page.ts.*function/) } for (const result of ['bare string', undefined, null, {}, { outputName: '', content: '' }, { outputName: 'x', content: 1 }, [record, 'bad'], new Set([record])]) { - await assert.rejects(Array.fromAsync(normalizeAdditionalOutputs(result, provenance)), /Invalid additionalOutputs.*\/src\/page.ts.*Record/) + await assert.rejects(Array.fromAsync(normalizePageOutputs(result, provenance)), /Invalid pageOutputs.*\/src\/page.ts.*Record/) } async function * broken () { yield record; throw new Error('iterator failed') } - await assert.rejects(Array.fromAsync(normalizeAdditionalOutputs(broken(), provenance)), /\/src\/page.ts.*iterator failed/) - await assert.rejects(Array.fromAsync(normalizeAdditionalOutputs(Promise.reject(new Error('promise failed')), provenance)), /\/src\/page.ts.*promise failed/) + await assert.rejects(Array.fromAsync(normalizePageOutputs(broken(), provenance)), /\/src\/page.ts.*iterator failed/) + await assert.rejects(Array.fromAsync(normalizePageOutputs(Promise.reject(new Error('promise failed')), provenance)), /\/src\/page.ts.*promise failed/) }) test('normalization pulls one record at a time and closes the provider on return or break', async () => { @@ -44,7 +44,7 @@ test('normalization pulls one record at a time and closes the provider on return events.push('closed') } } - const outputs = normalizeAdditionalOutputs(records(), provenance) + const outputs = normalizePageOutputs(records(), provenance) assert.equal(outputs[Symbol.asyncIterator](), outputs) assert.deepEqual(events, []) if (close === 'return') { @@ -77,11 +77,11 @@ test('normalization validates each record only when requested with its record nu } } for (const result of [[record, { outputName: 'invalid.json', content: 1 }], records()]) { - const outputs = normalizeAdditionalOutputs(result, provenance) + const outputs = normalizePageOutputs(result, provenance) assert.deepEqual(await outputs.next(), { value: { ...record, provenance }, done: false }) await assert.rejects(outputs.next(), error => { assert.ok(error instanceof Error) - assert.match(error.message, /Invalid additionalOutputs from page "\/src\/page.ts": Record 2/) + assert.match(error.message, /Invalid pageOutputs from page "\/src\/page.ts": Record 2/) assert.ok(error.cause instanceof TypeError) return true }) @@ -95,7 +95,7 @@ test('normalization preserves iterator failure causes after yielding valid recor yield record throw cause } - const outputs = normalizeAdditionalOutputs(records(), provenance) + const outputs = normalizePageOutputs(records(), provenance) assert.deepEqual(await outputs.next(), { value: { ...record, provenance }, done: false }) await assert.rejects(outputs.next(), error => { assert.ok(error instanceof Error) diff --git a/lib/domstack-manifest/schema.js b/lib/domstack-manifest/schema.js index d75cc038..fb8bd34c 100644 --- a/lib/domstack-manifest/schema.js +++ b/lib/domstack-manifest/schema.js @@ -18,7 +18,7 @@ export const domstackManifestKindSchema = /** @satisfies {JSONSchema} */ (/** @t description: 'Classifies the build pipeline step or artifact type that produced this output.', enum: [ 'page', - 'page-additional', + 'page-output', 'template', 'script', 'style', diff --git a/lib/domstack-manifest/schema.json b/lib/domstack-manifest/schema.json index 6d7e7ad3..2f618bc8 100644 --- a/lib/domstack-manifest/schema.json +++ b/lib/domstack-manifest/schema.json @@ -33,7 +33,7 @@ "description": "Classifies the build pipeline step or artifact type that produced this output.", "enum": [ "page", - "page-additional", + "page-output", "template", "script", "style", diff --git a/test-cases/generated-pages/streaming.test.js b/test-cases/generated-pages/streaming.test.js index 5a99a7c7..69e7c611 100644 --- a/test-cases/generated-pages/streaming.test.js +++ b/test-cases/generated-pages/streaming.test.js @@ -10,7 +10,7 @@ import pino from 'pino' import { DomStack } from '../../index.js' import { builder } from '../../lib/builder.js' import { DomStackAggregateError } from '../../lib/helpers/domstack-aggregate-error.js' -import { errorText, settle, writeFiles } from '../page-additional-outputs/helpers.js' +import { errorText, settle, writeFiles } from '../page-outputs/helpers.js' /** @param {TestContext} t @param {Record} files @param {boolean} [buildDrafts] */ async function setup (t, files, buildDrafts = false) { @@ -88,13 +88,13 @@ for (const [form, factoryExport] of Object.entries({ assert.throws(() => data.pageValue, /undeclared/) return '
' + vars.title + ':' + vars.layoutOnly + ':' + data.layoutValue + ':' + children + '
' } - export const additionalOutputs = () => { throw Error('generated layout output hook must be skipped') }`, + export const pageOutputs = () => { throw Error('generated layout output hook must be skipped') }`, 'stream.pages.js': `import assert from 'node:assert/strict' import { readFile } from 'node:fs/promises' import { join } from 'node:path' import globals from './global.vars.js' export const dataDeps = ['collection'] - export const additionalOutputs = () => { throw Error('pages-file output hook must be skipped') } + export const pageOutputs = () => { throw Error('pages-file output hook must be skipped') } async function* pages (context) { if (context) { assert.deepEqual(context.data.collection, ['Initialized concrete']) diff --git a/test-cases/page-outputs/cache.test.js b/test-cases/page-outputs/cache.test.js new file mode 100644 index 00000000..bd77de57 --- /dev/null +++ b/test-cases/page-outputs/cache.test.js @@ -0,0 +1,183 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { rm, stat, utimes, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { hook, setup, settle } from './helpers.js' + +// Install the guard inside each build worker; parent-side reads remain available +// for assertions, and syncBuiltinESMExports also guards already-imported bindings. +const noOutputReadsLayout = `import fs from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +const readFile = fs.readFile +fs.readFile = async (...args) => { + if (String(args[0]).endsWith('/cached.txt')) throw Error('page-output writer reread destination bytes') + return readFile(...args) +} +syncBuiltinESMExports() +export default ({ children }) => children` + +test('watch caches identical hook bytes across workers without rereads and recreates deleted or cleaned outputs', { timeout: 30_000 }, async t => { + const companionSource = 'export default {}; ' + hook('cached.txt', 'cached bytes') + const { site, src, dest, read, mtime, logs } = await setup(t, { + 'root.layout.js': noOutputReadsLayout, + 'page.html': 'initial main', + 'page.vars.js': companionSource, + }) + await site.watch({ serve: false }) + assert.equal(await read('cached.txt'), 'cached bytes') + const originalTime = await mtime('cached.txt') + for (const content of ['first rebuild', 'second rebuild']) { + await settle(site, logs, async () => { + await writeFile(join(src, 'page.html'), content) + }) + assert.equal(await read('index.html'), content, 'the hook owner actually rebuilt') + assert.equal(await read('cached.txt'), 'cached bytes') + assert.equal(await mtime('cached.txt'), originalTime, 'identical bytes retain their mtime across workers') + } + + await rm(join(dest, 'cached.txt')) + await assert.rejects(read('cached.txt'), { code: 'ENOENT' }) + await settle(site, logs, async () => { + await writeFile(join(src, 'page.html'), 'rebuild after destination deletion') + }) + assert.equal(await read('cached.txt'), 'cached bytes', 'a cache hit must still validate destination existence') + assert.notEqual(await mtime('cached.txt'), originalTime) + + await settle(site, logs, async () => { + await writeFile(join(src, 'page.vars.js'), 'export default {}') + }) + await assert.rejects(read('cached.txt'), { code: 'ENOENT' }) + await settle(site, logs, async () => { + await writeFile(join(src, 'page.vars.js'), companionSource) + }) + assert.equal(await read('cached.txt'), 'cached bytes', 're-adding the same hook recreates the same destination and content') + const restoredTime = await mtime('cached.txt') + await settle(site, logs, async () => { + await writeFile(join(src, 'page.html'), 'rebuild after hook re-addition') + }) + assert.equal(await mtime('cached.txt'), restoredTime, 're-added outputs participate in caching again') +}) + +test('watch repairs same-size external edits with exactly restored mtime using changed ctime', { timeout: 15_000 }, async t => { + // Normalize writes before the writer records metadata, so utimes can restore + // mtime exactly even on filesystems whose native timestamps have nanoseconds. + const timestamp = 1_600_000_000 + const { site, src, dest, read, logs } = await setup(t, { + 'root.layout.js': `import fs from 'node:fs/promises' + import { syncBuiltinESMExports } from 'node:module' + const writeFile = fs.writeFile + fs.writeFile = async (...args) => { + await writeFile(...args) + if (String(args[0]).endsWith('/metadata.txt')) await fs.utimes(args[0], ${timestamp}, ${timestamp}) + } + syncBuiltinESMExports() + export default ({ children }) => children`, + 'page.html': 'initial main', + 'page.vars.js': 'export default {}; ' + hook('metadata.txt', 'original'), + }) + await site.watch({ serve: false }) + const output = join(dest, 'metadata.txt') + const original = await stat(output, { bigint: true }) + assert.equal(original.mtimeNs, BigInt(timestamp) * 1_000_000_000n) + await settle(site, logs, async () => { + await writeFile(join(src, 'page.html'), 'unchanged sidecar rebuild') + }) + assert.equal((await stat(output, { bigint: true })).ctimeNs, original.ctimeNs, 'the normalized output was cached, not rewritten') + + await writeFile(output, 'tampered') + await utimes(output, timestamp, timestamp) + const modified = await stat(output, { bigint: true }) + assert.equal(modified.size, original.size) + assert.equal(modified.mtimeNs, original.mtimeNs, 'mtime is restored exactly, not merely within a tolerance') + assert.equal(modified.ino, original.ino) + assert.equal(modified.dev, original.dev) + assert.notEqual(modified.ctimeNs, original.ctimeNs, 'ctime is the changed cache-validation metadata') + assert.equal(await read('metadata.txt'), 'tampered') + await settle(site, logs, async () => { + await writeFile(join(src, 'page.html'), 'rebuild after external edit') + }) + assert.equal(await read('metadata.txt'), 'original', 'matching hash, size, inode and mtime cannot hide an external edit') +}) + +test('watch retains cached writes before iterator failure through another failed worker and recovery', { timeout: 30_000 }, async t => { + const outputs = `yield { outputName: 'existing.txt', content: 'updated before failure' } + yield { outputName: 'partial.txt', content: 'new before failure' }` + const { site, src, dest, read, mtime, logs } = await setup(t, { + 'page.js': `export default () => 'initial main'; export const pageOutputs = () => [ + { outputName: 'existing.txt', content: 'initial sidecar' }, + { outputName: 'stale.txt', content: 'keep until recovery' }, + ]`, + }) + await site.watch({ serve: false }) + const mainTime = await mtime('index.html') + const existingTime = await mtime('existing.txt') + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* pageOutputs () { + ${outputs} + throw Error('first cache iterator failure') + }`) + }, 'first cache iterator failure') + assert.equal(await read('existing.txt'), 'updated before failure') + assert.equal(await read('partial.txt'), 'new before failure') + assert.notEqual(await mtime('existing.txt'), existingTime) + const cachedTimes = [await mtime('existing.txt'), await mtime('partial.txt')] + + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), `export default () => 'failed again'; export async function* pageOutputs () { + ${outputs} + throw Error('second cache iterator failure') + }`) + }, 'second cache iterator failure') + assert.deepEqual([await mtime('existing.txt'), await mtime('partial.txt')], cachedTimes, 'failed workers retain both updated and newly created cache entries') + assert.equal(await read('index.html'), 'initial main') + assert.equal(await mtime('index.html'), mainTime) + assert.equal(await read('stale.txt'), 'keep until recovery') + + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), `export default () => 'recovered main'; export async function* pageOutputs () { + ${outputs} + }`) + }) + assert.equal(await read('index.html'), 'recovered main') + assert.equal(await read('existing.txt'), 'updated before failure') + assert.equal(await read('partial.txt'), 'new before failure') + assert.deepEqual([await mtime('existing.txt'), await mtime('partial.txt')], cachedTimes, 'recovery also skips unchanged outputs from failed workers') + await assert.rejects(stat(join(dest, 'stale.txt')), { code: 'ENOENT' }) +}) + +for (const writer of ['template', 'page']) { + test(`watch repairs a cached hook destination overwritten by another ${writer}`, { timeout: 20_000 }, async t => { + const otherSource = writer === 'template' ? 'shared.template.js' : 'shared.md' + const otherContent = writer === 'template' + ? "export default () => ({ outputName: 'shared.html', content: 'other writer' })" + : 'other writer' + const { site, src, read, mtime, logs } = await setup(t, { + 'page.js': "export default () => 'initial owner'", + [otherSource]: writer === 'template' + ? "export default () => ({ outputName: 'shared.html', content: 'initial other writer' })" + : 'initial other writer', + }) + await site.watch({ serve: false }) + // Add the hook only after the competing output exists, avoiding concurrent + // writes to the same destination during the initial build. + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), "export default () => 'seeded owner'; " + hook('shared.html', 'hook content')) + }) + assert.equal(await read('shared.html'), 'hook content') + const ownerTime = await mtime('index.html') + await settle(site, logs, async () => { + await writeFile(join(src, otherSource), otherContent) + }) + if (writer === 'template') { + assert.equal(await read('shared.html'), 'other writer') + } else { + assert.match(await read('shared.html'), /

other writer<\/p>/) + } + assert.equal(await mtime('index.html'), ownerTime, 'the competing writer rebuild leaves the hook owner untouched') + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), "export default () => 'repaired owner'; " + hook('shared.html', 'hook content')) + }) + assert.equal(await read('index.html'), 'repaired owner') + assert.equal(await read('shared.html'), 'hook content', 'the next hook owner rebuild detects another writer despite unchanged hook bytes') + }) +} diff --git a/test-cases/page-additional-outputs/helpers.js b/test-cases/page-outputs/helpers.js similarity index 96% rename from test-cases/page-additional-outputs/helpers.js rename to test-cases/page-outputs/helpers.js index 75bb200a..58d9e524 100644 --- a/test-cases/page-additional-outputs/helpers.js +++ b/test-cases/page-outputs/helpers.js @@ -87,4 +87,4 @@ export function errorText (error) { } /** @param {string} outputName @param {string} [content] */ -export const hook = (outputName, content = 'sidecar') => `export const additionalOutputs = () => ({ outputName: ${JSON.stringify(outputName)}, content: ${JSON.stringify(content)} })` +export const hook = (outputName, content = 'sidecar') => `export const pageOutputs = () => ({ outputName: ${JSON.stringify(outputName)}, content: ${JSON.stringify(content)} })` diff --git a/test-cases/page-additional-outputs/index.test.js b/test-cases/page-outputs/index.test.js similarity index 85% rename from test-cases/page-additional-outputs/index.test.js rename to test-cases/page-outputs/index.test.js index 334278a1..f7012011 100644 --- a/test-cases/page-additional-outputs/index.test.js +++ b/test-cases/page-outputs/index.test.js @@ -1,11 +1,11 @@ import { test } from 'node:test' import assert from 'node:assert/strict' -import { stat } from 'node:fs/promises' +import { stat, utimes } from 'node:fs/promises' import { join } from 'node:path' import { errorText, hook, setup, writeFiles } from './helpers.js' const rawLayout = `export default ({ children }) => '

' + children + '
' -export const additionalOutputs = async ({ page }) => ({ outputName: './source.txt', content: await page.readMarkdownContent() })` +export const pageOutputs = async ({ page }) => ({ outputName: './source.txt', content: await page.readMarkdownContent() })` test('builder renders Markdown and exports the unrendered body from its layout at a custom destination', async t => { const body = '# Article\n\nKeep **Markdown**, {{ vars.title }}, and [links](./other.md).\n' @@ -18,19 +18,19 @@ test('builder renders Markdown and exports the unrendered body from its layout a assert.match(await read('docs/index.html'), /Markdown<\/strong>/) assert.equal(await read('docs/source.txt'), '\n' + body) const record = result.pageBuildResults?.outputs.find(output => output.outputRelname === 'docs/source.txt') - assert.ok(record, 'additional output is included in the page build report') + assert.ok(record, 'page output is included in the page build report') assert.equal(record.filepath, join(dest, 'docs/source.txt')) assert.equal(record.sourceRelname, 'docs/page.md') }) test('nested hooks run outer -> inner -> companion with isolated renderer data and resolved vars', async t => { const { build, read } = await setup(t, { - 'global.vars.js': "export default { layout: 'inner', title: 'global' }; export const additionalOutputs = () => { throw Error('global provider ran') }", + 'global.vars.js': "export default { layout: 'inner', title: 'global' }; export const pageOutputs = () => { throw Error('global provider ran') }", 'global.data.js': "export default { outer: 'O', inner: 'I', selected: 'P', secret: 'hidden' }", 'root.layout.js': `import assert from 'node:assert/strict' export const vars = { dataDeps: ['outer'] } export default ({ children, data }) => data.outer + children - export const additionalOutputs = ({ page, vars, data }) => { + export const pageOutputs = ({ page, vars, data }) => { assert.equal(vars.title, 'page title') assert.throws(() => data.selected, /undeclared/) assert.equal('renderFullPage' in page, false) @@ -44,7 +44,7 @@ test('nested hooks run outer -> inner -> companion with isolated renderer data a export const parentLayout = 'root' export const vars = { dataDeps: ['inner'] } export default ({ children, data }) => data.inner + children - export async function* additionalOutputs ({ page, data }) { + export async function* pageOutputs ({ page, data }) { assert.throws(() => data.outer, /undeclared/) assert.equal(await readFile(join(dirname(page.pageFile.filepath), '../../custom-output/docs/outer.txt'), 'utf8'), 'O') globalThis[page.pageFile.filepath].push('inner') @@ -55,7 +55,7 @@ test('nested hooks run outer -> inner -> companion with isolated renderer data a import { readFile } from 'node:fs/promises' import { dirname, join } from 'node:path' export default { dataDeps: ['selected'] } - export const additionalOutputs = async ({ page, vars, data }) => { + export const pageOutputs = async ({ page, vars, data }) => { assert.throws(() => data.secret, /undeclared/) assert.throws(() => data.inner, /undeclared/) assert.equal(Object.isFrozen(page), true) @@ -86,7 +86,7 @@ for (const extension of ['html', 'js', 'ts']) { [`article/page.${extension}`]: extension === 'html' ? '

{{ vars.title }}

' : 'export default ({ vars, data }) => vars.title + data.selected', 'article/page.vars.js': `import assert from 'node:assert/strict' export default { title: 'Companion', dataDeps: ['selected'] } - export async function additionalOutputs ({ page, vars, data }) { + export async function pageOutputs ({ page, vars, data }) { await assert.rejects(page.readMarkdownContent()) assert.throws(() => data.secret, /undeclared/) return { outputName: 'metadata.json', content: JSON.stringify({ title: vars.title, value: data.selected }) } @@ -100,12 +100,12 @@ for (const extension of ['html', 'js', 'ts']) { test('JS page modules support promised async iterables, arrays, and empty results', async t => { const { build, read } = await setup(t, { - 'page.js': `export default () => 'main'; export const additionalOutputs = async () => (async function* () { + 'page.js': `export default () => 'main'; export const pageOutputs = async () => (async function* () { yield { outputName: 'one.txt', content: 'one' }; yield { outputName: './two.txt', content: 'two' } })()`, - 'array/page.js': "export default () => 'array'; export const additionalOutputs = () => [{ outputName: 'array.txt', content: 'array' }]", - 'empty/page.js': "export default () => 'empty'; export const additionalOutputs = () => []", - 'iterator/page.js': "export default () => 'empty iterator'; export async function* additionalOutputs () {}", + 'array/page.js': "export default () => 'array'; export const pageOutputs = () => [{ outputName: 'array.txt', content: 'array' }]", + 'empty/page.js': "export default () => 'empty'; export const pageOutputs = () => []", + 'iterator/page.js': "export default () => 'empty iterator'; export async function* pageOutputs () {}", }) await build() for (const name of ['one', 'two']) assert.equal(await read(`${name}.txt`), name) @@ -120,7 +120,7 @@ test('async generators publish each record before requesting the next at a custo import { readFile, stat } from 'node:fs/promises' import { dirname, join } from 'node:path' export default () => 'main' - export async function* additionalOutputs ({ page }) { + export async function* pageOutputs ({ page }) { const dest = join(dirname(page.pageFile.filepath), '../custom-output') yield { outputName: 'replaced.txt', content: 'replacement' } assert.equal(await readFile(join(dest, 'replaced.txt'), 'utf8'), 'replacement') @@ -129,15 +129,16 @@ test('async generators publish each record before requesting the next at a custo const unchangedTime = (await stat(join(dest, 'unchanged.txt'))).mtimeMs yield { outputName: 'unchanged.txt', content: 'same bytes' } assert.equal(await readFile(join(dest, 'unchanged.txt'), 'utf8'), 'same bytes') - assert.equal((await stat(join(dest, 'unchanged.txt'))).mtimeMs, unchangedTime) + assert.notEqual((await stat(join(dest, 'unchanged.txt'))).mtimeMs, unchangedTime, 'a cold build writes even identical bytes') }`, }) await writeFiles(dest, { 'replaced.txt': 'old sidecar', 'unchanged.txt': 'same bytes' }) + await utimes(join(dest, 'unchanged.txt'), 1, 1) const unchangedTime = await mtime('unchanged.txt') const result = await build() assert.equal(await read('replaced.txt'), 'replacement') assert.equal(await read('nested/new.txt'), 'new sidecar') - assert.equal(await mtime('unchanged.txt'), unchangedTime) + assert.notEqual(await mtime('unchanged.txt'), unchangedTime) assert.equal(await read('index.html'), 'main') for (const outputRelname of ['replaced.txt', 'nested/new.txt', 'unchanged.txt']) { assert.ok(result.pageBuildResults?.outputs.some(output => output.outputRelname === outputRelname), `${outputRelname} is reported, including unchanged content`) @@ -146,26 +147,24 @@ test('async generators publish each record before requesting the next at a custo test('generated pages skip inherited layout hooks', async t => { const { build, read } = await setup(t, { - 'root.layout.js': "export default ({ children }) => children; export const additionalOutputs = () => { throw Error('generated hook ran') }", + 'root.layout.js': "export default ({ children }) => children; export const pageOutputs = () => { throw Error('generated hook ran') }", 'items.pages.js': "export default { outputName: 'generated.html', children: 'Generated' }", }) await build() assert.equal(await read('generated.html'), 'Generated') }) -test('duplicate JS and companion providers fail with both module names', async t => { - const { build } = await setup(t, { - 'page.js': "export default () => 'main'; " + hook('page.txt'), - 'page.vars.js': 'export default {}; ' + hook('companion.txt'), - }) - await assert.rejects(build(), error => { - const message = errorText(error) - assert.match(message, /page\.js/) - assert.match(message, /page\.vars\.js/) - assert.match(message, /additionalOutputs/) - assert.match(message, /both|conflict/i) - return true +test('JS page outputs take precedence over companion outputs while layouts remain additive', async t => { + const { build, read } = await setup(t, { + 'root.layout.js': 'export default ({ children }) => children; ' + hook('layout.txt', 'layout'), + 'page.js': "export default () => 'main'; " + hook('page.txt', 'page'), + 'page.vars.js': "export default {}; export const pageOutputs = () => { throw Error('ignored companion must not run') }", }) + await build() + assert.equal(await read('index.html'), 'main') + assert.equal(await read('layout.txt'), 'layout') + assert.equal(await read('page.txt'), 'page') + await assert.rejects(read('companion.txt'), { code: 'ENOENT' }) }) for (const scenario of [ @@ -197,9 +196,9 @@ for (const result of [ "{ outputName: '../escape.txt', content: 'bad' }", "{ outputName: '/', content: 'bad' }", ]) { - test(`builder rejects invalid additional output: ${result}`, async t => { + test(`builder rejects invalid page output: ${result}`, async t => { const { build, dest, read } = await setup(t, { - 'page.js': `export default () => 'new'; export const additionalOutputs = () => (${result})`, + 'page.js': `export default () => 'new'; export const pageOutputs = () => (${result})`, }) await writeFiles(dest, { 'index.html': 'old' }) await assert.rejects(build()) @@ -211,7 +210,7 @@ for (const result of [ test('iterator failure retains earlier sidecar writes and the previous HTML', async t => { const { build, dest, read } = await setup(t, { 'a/page.js': "export default () => 'new sibling'; " + hook('sibling.txt', 'new sibling sidecar'), - 'z/page.js': `export default () => 'new main'; export async function* additionalOutputs () { + 'z/page.js': `export default () => 'new main'; export async function* pageOutputs () { yield { outputName: 'old.txt', content: 'replacement' } yield { outputName: 'partial.txt', content: 'published before failure' } throw Error('iterator exploded') @@ -231,18 +230,18 @@ test('iterator failure retains earlier sidecar writes and the previous HTML', as for (const provider of ['layout', 'page']) { test(`a later ${provider} provider failure retains earlier layout files`, async t => { - const failingHook = 'export const additionalOutputs = () => { throw Error(\'later provider exploded\') }' + const failingHook = 'export const pageOutputs = () => { throw Error(\'later provider exploded\') }' const { build, dest, read } = await setup(t, { 'global.vars.js': "export default { layout: 'inner' }", 'root.layout.js': `export default ({ children }) => children - export async function* additionalOutputs () { + export async function* pageOutputs () { yield { outputName: 'old.txt', content: 'replacement' } yield { outputName: 'partial.txt', content: 'partial' } }`, 'inner.layout.js': `export const parentLayout = 'root'; export default ({ children }) => children; - ${provider === 'layout' ? failingHook : 'export const additionalOutputs = () => []'}`, + ${provider === 'layout' ? failingHook : 'export const pageOutputs = () => []'}`, 'page.js': `export default () => 'new main'; - ${provider === 'page' ? failingHook : "export const additionalOutputs = () => { throw Error('page provider must not run') }"}`, + ${provider === 'page' ? failingHook : "export const pageOutputs = () => { throw Error('page provider must not run') }"}`, }) await writeFiles(dest, { 'index.html': 'old main', 'old.txt': 'old sidecar' }) await assert.rejects(build(), error => { @@ -265,7 +264,7 @@ for (const invalid of [ 'page.js': `import { writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' export default () => 'new main' - export async function* additionalOutputs ({ page }) { + export async function* pageOutputs ({ page }) { yield { outputName: 'first.txt', content: 'published' } yield ${invalid.record} await writeFile(join(dirname(page.pageFile.filepath), '../following-yield-requested'), 'requested') @@ -287,7 +286,7 @@ for (const invalid of [ test('identical duplicate records from one hook warn rather than reject the build', async t => { const { build, read } = await setup(t, { - 'page.js': `export default () => 'main'; export const additionalOutputs = () => [ + 'page.js': `export default () => 'main'; export const pageOutputs = () => [ { outputName: 'same.txt', content: 'same' }, { outputName: './same.txt', content: 'same' }, ]`, diff --git a/test-cases/page-additional-outputs/ownership.test.js b/test-cases/page-outputs/ownership.test.js similarity index 95% rename from test-cases/page-additional-outputs/ownership.test.js rename to test-cases/page-outputs/ownership.test.js index 36915c6a..02dcd7a3 100644 --- a/test-cases/page-additional-outputs/ownership.test.js +++ b/test-cases/page-outputs/ownership.test.js @@ -7,7 +7,7 @@ import { hook, setup, settle } from './helpers.js' test('data-invalidated pages replace ownership using actual reports', { timeout: 15_000 }, async t => { const { site, src, dest, read, logs } = await setup(t, { 'global.data.js': "export default { name: 'old.txt' }", - 'page.js': "export const vars = { dataDeps: ['name'] }; export default () => 'main'; export const additionalOutputs = ({ data }) => ({ outputName: data.name, content: 'sidecar' })", + 'page.js': "export const vars = { dataDeps: ['name'] }; export default () => 'main'; export const pageOutputs = ({ data }) => ({ outputName: data.name, content: 'sidecar' })", }) await site.watch({ serve: false }) await settle(site, logs, async () => { @@ -40,7 +40,7 @@ test('repeated failed watch builds union partial paths with successful ownership }) await site.watch({ serve: false }) await settle(site, logs, async () => { - await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* additionalOutputs () { + await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* pageOutputs () { yield { outputName: 'partial.txt', content: 'partial' } throw Error('ownership failure') }`) @@ -48,7 +48,7 @@ test('repeated failed watch builds union partial paths with successful ownership assert.equal(await read('old.txt'), 'sidecar', 'failure must not clean up the previous successful output') assert.equal(await read('partial.txt'), 'partial') await settle(site, logs, async () => { - await writeFile(join(src, 'page.js'), `export default () => 'failed again'; export async function* additionalOutputs () { + await writeFile(join(src, 'page.js'), `export default () => 'failed again'; export async function* pageOutputs () { yield { outputName: 'second-partial.txt', content: 'second partial' } throw Error('second ownership failure') }`) @@ -70,7 +70,7 @@ for (const change of ['recovery', 'source deletion', 'hook removal']) { test(`initial failed watch tracks partial ownership for ${change}`, { timeout: 15_000 }, async t => { const { site, src, dest, read, logs } = await setup(t, { 'article/page.html': 'Article', - 'article/page.vars.js': `export default {}; export async function* additionalOutputs () { + 'article/page.vars.js': `export default {}; export async function* pageOutputs () { yield { outputName: 'partial.txt', content: 'partial' } yield { outputName: '/root-partial.txt', content: 'root partial' } throw Error('initial ownership failure') diff --git a/test-cases/page-additional-outputs/watch.test.js b/test-cases/page-outputs/watch.test.js similarity index 94% rename from test-cases/page-additional-outputs/watch.test.js rename to test-cases/page-outputs/watch.test.js index 0f05a4ca..1fe021f2 100644 --- a/test-cases/page-additional-outputs/watch.test.js +++ b/test-cases/page-outputs/watch.test.js @@ -6,7 +6,7 @@ import { hook, setup, settle, writeFiles } from './helpers.js' const rawLayout = `export const vars = { dataDeps: ['navigation'] } export default ({ children, data }) => data.navigation + children -export const additionalOutputs = async ({ page }) => ({ outputName: 'source.txt', content: await page.readMarkdownContent() })` +export const pageOutputs = async ({ page }) => ({ outputName: 'source.txt', content: await page.readMarkdownContent() })` test('watch updates only changed raw content and retains unchanged ownership without a public manifest', { timeout: 30_000 }, async t => { const { site, src, dest, read, mtime, logs } = await setup(t, { @@ -82,7 +82,7 @@ test('watch reconciles companion addition, output rename, hook removal, companio test('watch removes sidecars on source rename and draft exclusion', { timeout: 30_000 }, async t => { const { site, src, dest, read, logs } = await setup(t, { 'root.layout.js': `export default ({ children }) => children - export const additionalOutputs = async ({ page }) => ({ outputName: page.outputName + '.txt', content: await page.readMarkdownContent() })`, + export const pageOutputs = async ({ page }) => ({ outputName: page.outputName + '.txt', content: await page.readMarkdownContent() })`, 'article.md': '# Article\n', }) await site.watch({ serve: false }) @@ -104,7 +104,7 @@ test('watch removes sidecars on source rename and draft exclusion', { timeout: 3 test('watch hook failure retains partial writes and recovery removes old and partial outputs', { timeout: 30_000 }, async t => { const { site, src, dest, read, mtime, logs } = await setup(t, { - 'page.js': `export default () => 'old main'; export const additionalOutputs = () => [ + 'page.js': `export default () => 'old main'; export const pageOutputs = () => [ { outputName: 'old.txt', content: 'old sidecar' }, { outputName: 'stale.txt', content: 'retain until recovery' }, ]`, @@ -112,7 +112,7 @@ test('watch hook failure retains partial writes and recovery removes old and par await site.watch({ serve: false }) const oldHtmlTime = await mtime('index.html') await settle(site, logs, async () => { - await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* additionalOutputs () { + await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* pageOutputs () { yield { outputName: 'old.txt', content: 'failed replacement' } yield { outputName: 'partial.txt', content: 'partial' } throw Error('watch iterator exploded') @@ -164,7 +164,7 @@ test('hook-only data subscriptions invalidate their owner but not an unrelated s const { site, src, read, mtime, logs } = await setup(t, { 'global.data.js': "export default { selected: 'first', unrelated: 'unchanged' }", 'a/page.html': 'A', - 'a/page.vars.js': "export default { dataDeps: ['selected'] }; export const additionalOutputs = ({ data }) => ({ outputName: 'data.txt', content: data.selected })", + 'a/page.vars.js': "export default { dataDeps: ['selected'] }; export const pageOutputs = ({ data }) => ({ outputName: 'data.txt', content: data.selected })", 'b/page.html': 'B', }) await site.watch({ serve: false }) diff --git a/types.ts b/types.ts index 32d0d66d..32f40804 100644 --- a/types.ts +++ b/types.ts @@ -7,14 +7,14 @@ import type { Results } from './lib/builder.js' export type { DataDeps } from './lib/build-pages/data-deps.js' export type { - AdditionalOutput, - AdditionalOutputProvenance, - AdditionalOutputsFunction, - AdditionalOutputsFunctionParams, - AdditionalOutputsPage, - AdditionalOutputsResult, - CollectedAdditionalOutput, -} from './lib/build-pages/additional-outputs.js' + PageOutput, + PageOutputProvenance, + PageOutputsFunction, + PageOutputsFunctionParams, + PageOutputsPage, + PageOutputsResult, + CollectedPageOutput, +} from './lib/build-pages/page-outputs.js' export type { BuildOptions } from 'esbuild' export type { DomStackOpts, Results, SiteData } from './lib/builder.js' From 9eadf6538c8fc60050fdd9cb960152422d09cde4 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Mon, 14 Sep 2026 14:29:04 -0700 Subject: [PATCH 8/8] Report page output provider conflicts through structured warnings --- lib/build-pages/index.js | 1 + .../page-data-page-outputs.test.js | 9 +++++--- lib/build-pages/page-data.js | 7 +++++- lib/helpers/domstack-warning.js | 1 + test-cases/page-outputs/index.test.js | 11 +++++++-- test-cases/page-outputs/watch.test.js | 23 +++++++++++++++++++ 6 files changed, 46 insertions(+), 6 deletions(-) diff --git a/lib/build-pages/index.js b/lib/build-pages/index.js index 5ce02f0a..d8ba54fb 100644 --- a/lib/build-pages/index.js +++ b/lib/build-pages/index.js @@ -604,6 +604,7 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { } catch (err) { result.errors.push(serializeBuildError(err, { page: pageInfoForWorker(pageInfo) }, 'Error resolving page vars')) } + result.warnings.push(...pageData.warnings) return pageData } diff --git a/lib/build-pages/page-data-page-outputs.test.js b/lib/build-pages/page-data-page-outputs.test.js index d0214fd3..2a701a7b 100644 --- a/lib/build-pages/page-data-page-outputs.test.js +++ b/lib/build-pages/page-data-page-outputs.test.js @@ -375,11 +375,14 @@ test('page module outputs take precedence over companion outputs with a warning assert.ok(layout) layout.pageOutputs = () => ({ outputName: `${name}.txt`, content: name }) } - const warningCount = warn.mock.callCount() await pd.init({ layouts }) await pd.init({ layouts }) - assert.equal(warn.mock.callCount(), warningCount + 1) - const message = warn.mock.calls.at(-1)?.arguments[0] + assert.equal(warn.mock.callCount(), 0) + assert.equal(pd.warnings.length, 1) + const warning = pd.warnings[0] + assert.ok(warning) + assert.equal(warning.code, 'DOM_STACK_WARNING_DUPLICATE_PAGE_OUTPUTS_PROVIDER') + const { message } = warning assert.ok(message.includes(join(dir, `page.${extension}`))) assert.ok(message.includes(join(dir, 'page.vars.mjs'))) assert.match(message, /both export pageOutputs; using the page module export and ignoring the companion export/) diff --git a/lib/build-pages/page-data.js b/lib/build-pages/page-data.js index 5caa2525..d326cd3d 100644 --- a/lib/build-pages/page-data.js +++ b/lib/build-pages/page-data.js @@ -1,6 +1,7 @@ /** * @import { PageInfo } from '../identify-pages.js' * @import { DomstackManifestRecord } from '../domstack-manifest/index.js' + * @import { DomStackWarning } from '../helpers/domstack-warning.js' * @import { BuilderOptions, InternalPageFunction } from './page-builders/page-writer.js' * @import { PageOutputsFunction, PageOutputProvenance, CollectedPageOutput } from './page-outputs.js' */ @@ -150,6 +151,7 @@ export async function resolveLayout (layoutPath) { */ export class PageData { /** @type {PageInfo} */ pageInfo + /** @type {DomStackWarning[]} */ warnings = [] /** @type {DomstackManifestRecord[]} Successfully emitted files, including partial builds; never populated by rendering alone. */ outputRecords = [] /** @type {ResolvedLayout | null | undefined} */ layout /** @type {ResolvedLayout[]} Each parent has its own render and data contracts. */ layoutChain = [] @@ -327,7 +329,10 @@ export class PageData { ? validatePageOutputsHook((await import(pageVars.filepath)).pageOutputs, pageVars.filepath) : undefined if (pageModuleOutputs && varsCompanionOutputs) { - console.warn(`Page "${pageInfo.pageFile.filepath}" and companion "${pageVars?.filepath}" both export pageOutputs; using the page module export and ignoring the companion export`) + this.warnings.push({ + code: 'DOM_STACK_WARNING_DUPLICATE_PAGE_OUTPUTS_PROVIDER', + message: `Page "${pageInfo.pageFile.filepath}" and companion "${pageVars?.filepath}" both export pageOutputs; using the page module export and ignoring the companion export`, + }) } const hook = pageModuleOutputs ?? varsCompanionOutputs if (hook) { diff --git a/lib/helpers/domstack-warning.js b/lib/helpers/domstack-warning.js index e3641de8..d741d0db 100644 --- a/lib/helpers/domstack-warning.js +++ b/lib/helpers/domstack-warning.js @@ -13,6 +13,7 @@ * 'DOM_STACK_WARNING_DUPLICATE_DOMSTACK_MANIFEST_SETTINGS' | * 'DOM_STACK_WARNING_CONFLICTING_MANIFEST_OUTPUT' | * 'DOM_STACK_WARNING_DUPLICATE_OUTPUT' | + * 'DOM_STACK_WARNING_DUPLICATE_PAGE_OUTPUTS_PROVIDER' | * 'DOM_STACK_WARNING_DUPLICATE_GLOBAL_VARS' | * 'DOM_STACK_WARNING_DUPLICATE_GLOBAL_DATA' | * 'DOM_STACK_WARNING_PAGE_MD_SHADOWS_README' diff --git a/test-cases/page-outputs/index.test.js b/test-cases/page-outputs/index.test.js index f7012011..00d802bf 100644 --- a/test-cases/page-outputs/index.test.js +++ b/test-cases/page-outputs/index.test.js @@ -155,12 +155,19 @@ test('generated pages skip inherited layout hooks', async t => { }) test('JS page outputs take precedence over companion outputs while layouts remain additive', async t => { - const { build, read } = await setup(t, { + const { build, read, src } = await setup(t, { 'root.layout.js': 'export default ({ children }) => children; ' + hook('layout.txt', 'layout'), 'page.js': "export default () => 'main'; " + hook('page.txt', 'page'), 'page.vars.js': "export default {}; export const pageOutputs = () => { throw Error('ignored companion must not run') }", }) - await build() + const result = await build() + const warnings = result.pageBuildResults?.warnings.filter(warning => 'code' in warning && warning.code === 'DOM_STACK_WARNING_DUPLICATE_PAGE_OUTPUTS_PROVIDER') + assert.equal(warnings?.length, 1) + const warning = warnings?.[0] + assert.ok(warning && 'message' in warning) + assert.ok(warning.message.includes(join(src, 'page.js'))) + assert.ok(warning.message.includes(join(src, 'page.vars.js'))) + assert.ok(result.warnings.includes(warning), 'worker warnings propagate to the aggregate build result') assert.equal(await read('index.html'), 'main') assert.equal(await read('layout.txt'), 'layout') assert.equal(await read('page.txt'), 'page') diff --git a/test-cases/page-outputs/watch.test.js b/test-cases/page-outputs/watch.test.js index 1fe021f2..aa68ca96 100644 --- a/test-cases/page-outputs/watch.test.js +++ b/test-cases/page-outputs/watch.test.js @@ -160,6 +160,29 @@ for (const change of ['source deletion', 'draft exclusion', 'hook removal', 'com }) } +test('provider precedence warnings reach the configured logger on initial and incremental watch builds', { timeout: 15_000 }, async t => { + const { site, src, read, logs } = await setup(t, { + 'page.js': "export default () => 'initial'; " + hook('selected.txt', 'page module'), + 'page.vars.js': "export default {}; export const pageOutputs = () => { throw Error('ignored companion ran') }", + }) + const result = await site.watch({ serve: false }) + assert.equal(result.pageBuildResults?.warnings.filter(warning => 'code' in warning && warning.code === 'DOM_STACK_WARNING_DUPLICATE_PAGE_OUTPUTS_PROVIDER').length, 1) + const cursor = logs.length + await settle(site, logs, async () => { + await writeFile(join(src, 'page.js'), "export default () => 'rebuilt'; " + hook('selected.txt', 'page module')) + }) + for (const messages of [logs.slice(0, cursor), logs.slice(cursor)]) { + const warnings = messages.map(line => JSON.parse(line)).filter(entry => entry.level === 40 && entry.msg.includes('both export pageOutputs')) + assert.ok(warnings.length > 0, 'the configured logger receives provider warnings for this watch phase') + for (const warning of warnings) { + assert.ok(warning.msg.includes(join(src, 'page.js'))) + assert.ok(warning.msg.includes(join(src, 'page.vars.js'))) + } + } + assert.equal(await read('index.html'), 'rebuilt') + assert.equal(await read('selected.txt'), 'page module') +}) + test('hook-only data subscriptions invalidate their owner but not an unrelated sibling', { timeout: 30_000 }, async t => { const { site, src, read, mtime, logs } = await setup(t, { 'global.data.js': "export default { selected: 'first', unrelated: 'unchanged' }",