From 7e298eac2a49e9995a0aa11c2f9f872bcef6bb53 Mon Sep 17 00:00:00 2001 From: Matthias Osswald Date: Wed, 7 Oct 2026 10:53:51 +0200 Subject: [PATCH 1/8] feat(fs): Add the step-runner write buffer and forward useGitignore The step-based build tasks in @ui5/project and the standalone step runner in @ui5/builder both flush their per-step writes in key order and must reject two concurrent keys that write the same path. @ui5/builder cannot import @ui5/project, so without a shared module the two runners would drift. Add internal/stepWriteBuffer.js as the single write-buffer contract: assertDistinctWrite guards against a same-path collision across keys, flushWriteBuffer writes the buffered resources in key order. Export it through the package exports map so both runners load the same code. createReader now accepts useGitignore (default false) and forwards it to the adapter, matching createAdapter. Add createMonitor(readerWriter) for the input-recording reader wrapper that MonitoredTaskUtil uses. Co-authored-by: Merlin Beutlberger --- packages/fs/lib/internal/stepWriteBuffer.js | 61 +++++++++++++++++++ packages/fs/lib/resourceFactory.js | 7 ++- packages/fs/package.json | 3 +- .../fs/test/lib/internal/stepWriteBuffer.js | 51 ++++++++++++++++ packages/fs/test/lib/package-exports.js | 6 +- packages/fs/test/lib/resourceFactory.js | 17 ++++++ 6 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 packages/fs/lib/internal/stepWriteBuffer.js create mode 100644 packages/fs/test/lib/internal/stepWriteBuffer.js diff --git a/packages/fs/lib/internal/stepWriteBuffer.js b/packages/fs/lib/internal/stepWriteBuffer.js new file mode 100644 index 00000000000..c8a4d4a7437 --- /dev/null +++ b/packages/fs/lib/internal/stepWriteBuffer.js @@ -0,0 +1,61 @@ +/** + * @module @ui5/fs/internal/stepWriteBuffer + * @description Shared write-buffer contract for the step-based build task runners. + * + * A map step fans out over its keys and runs the per-key units concurrently. Concurrent units are + * required to be independent, so their writes are buffered instead of hitting the workspace directly and + * are flushed in key order once every unit has finished. Two concurrent keys writing the same resource + * path is a contract violation rather than a last-wins race. + * + * Two runners implement this: the cache-aware runner in @ui5/project + * (build/helpers/StepRunner) and the cache-free runner in @ui5/builder + * (tasks/runSteps). @ui5/builder cannot import @ui5/project + * (the dependency direction is @ui5/cli -> @ui5/project -> @ui5/builder), so the two runners + * stay separate, but the same-path guard, its user-visible error message and the key-order flush live here + * so both runners share one definition. + * + * The buffer is a Map keyed by resource path (or, for a test fake without + * getPath, by identity). Each entry has the shape + * {resource, args, index}: resource is the resource to write, args + * are the trailing arguments to replay to workspace.write(resource, ...args), and + * index is the unit's position in the key order. + */ + +/** + * Rejects a second concurrent key writing a path another key already buffered. + * + * A unit writing a path it buffered itself (same index) is allowed, so a unit may overwrite + * its own earlier write within one run. A different unit (different index) writing the same + * path throws, because concurrent keys must be independent. + * + * @param {Map} buffer Shared write + * buffer, keyed by resource path + * @param {string} path Resource path being written, used as the buffer key + * @param {number} index Position of the writing unit in the key order + */ +export function assertDistinctWrite(buffer, path, index) { + const existing = buffer.get(path); + if (existing && existing.index !== index) { + throw new Error( + `Concurrent map-step keys must not write the same resource path ${path}. ` + + `Pass {sequential: true} if a later key must build on an earlier key's writes.`); + } +} + +/** + * Flushes a map step's buffered writes to the workspace in key order. + * + * The buffered entries are replayed sorted by index, so the observable write sequence matches + * the key order regardless of the order the concurrent units finished in. + * + * @param {Map} buffer Shared write + * buffer, keyed by resource path + * @param {@ui5/fs/AbstractReaderWriter} workspace Workspace the buffered writes are flushed to + * @returns {Promise} Resolves once every buffered write has been flushed + */ +export async function flushWriteBuffer(buffer, workspace) { + const ordered = [...buffer.values()].sort((a, b) => a.index - b.index); + for (const {resource, args} of ordered) { + await workspace.write(resource, ...args); + } +} diff --git a/packages/fs/lib/resourceFactory.js b/packages/fs/lib/resourceFactory.js index cfa27fd7bc5..a0339e1bbe6 100644 --- a/packages/fs/lib/resourceFactory.js +++ b/packages/fs/lib/resourceFactory.js @@ -61,9 +61,11 @@ export function createAdapter({name, fsBasePath, virBasePath, project, excludes, * @param {object} [parameters.project] Experimental, internal parameter. Do not use * @param {string[]} [parameters.excludes] List of glob patterns to exclude * @param {string} [parameters.name] Name for the reader collection + * @param {boolean} [parameters.useGitignore=false] + * Whether to apply any excludes defined in an optional .gitignore in the fsBasePath directory * @returns {@ui5/fs/ReaderCollection} Reader collection wrapping an adapter */ -export function createReader({fsBasePath, virBasePath, project, excludes = [], name}) { +export function createReader({fsBasePath, virBasePath, project, excludes = [], name, useGitignore = false}) { if (!fsBasePath) { // Creating a reader with a memory adapter seems pointless right now // since there would be no way to fill the adapter with resources @@ -96,7 +98,8 @@ export function createReader({fsBasePath, virBasePath, project, excludes = [], n fsBasePath, virBasePath, project, - excludes: normalizedExcludes + excludes: normalizedExcludes, + useGitignore })] }); } diff --git a/packages/fs/package.json b/packages/fs/package.json index 14b649e72fa..dd82695ee3d 100644 --- a/packages/fs/package.json +++ b/packages/fs/package.json @@ -30,7 +30,8 @@ "./resourceFactory": "./lib/resourceFactory.js", "./package.json": "./package.json", "./internal/ResourceTagCollection": "./lib/ResourceTagCollection.js", - "./internal/MonitoredResourceTagCollection": "./lib/MonitoredResourceTagCollection.js" + "./internal/MonitoredResourceTagCollection": "./lib/MonitoredResourceTagCollection.js", + "./internal/stepWriteBuffer": "./lib/internal/stepWriteBuffer.js" }, "engines": { "node": "^22.22.2 || ^24.15.0 || >=26.0.0", diff --git a/packages/fs/test/lib/internal/stepWriteBuffer.js b/packages/fs/test/lib/internal/stepWriteBuffer.js new file mode 100644 index 00000000000..5081dc9ab15 --- /dev/null +++ b/packages/fs/test/lib/internal/stepWriteBuffer.js @@ -0,0 +1,51 @@ +import test from "ava"; +import {assertDistinctWrite, flushWriteBuffer} from "../../../lib/internal/stepWriteBuffer.js"; + +// The exact user-visible message. Both step runners surface this verbatim, so pin it here as the single +// definition: a change to the shared message must update this assertion deliberately. +const SAME_PATH_MESSAGE = + "Concurrent map-step keys must not write the same resource path /a.out. " + + "Pass {sequential: true} if a later key must build on an earlier key's writes."; + +test("assertDistinctWrite passes for an empty buffer", (t) => { + t.notThrows(() => assertDistinctWrite(new Map(), "/a.out", 0)); +}); + +test("assertDistinctWrite passes when the same unit overwrites its own buffered path", (t) => { + const buffer = new Map([["/a.out", {resource: {}, args: [], index: 2}]]); + t.notThrows(() => assertDistinctWrite(buffer, "/a.out", 2), + "A unit may overwrite a path it buffered itself"); +}); + +test("assertDistinctWrite throws the exact message when a different unit writes the same path", (t) => { + const buffer = new Map([["/a.out", {resource: {}, args: [], index: 0}]]); + const err = t.throws(() => assertDistinctWrite(buffer, "/a.out", 1)); + t.is(err.message, SAME_PATH_MESSAGE, "The user-visible message has one definition"); +}); + +test("flushWriteBuffer replays writes in key order regardless of insertion order", async (t) => { + const written = []; + const workspace = {write: async (resource, ...args) => written.push({path: resource.getPath(), args})}; + const resource = (path) => ({getPath: () => path}); + // Insert out of key order: index 2, then 0, then 1. + const buffer = new Map([ + ["/c", {resource: resource("/c"), args: [{drain: true}], index: 2}], + ["/a", {resource: resource("/a"), args: [], index: 0}], + ["/b", {resource: resource("/b"), args: [{readOnly: true}], index: 1}], + ]); + + await flushWriteBuffer(buffer, workspace); + + t.deepEqual(written.map(({path}) => path), ["/a", "/b", "/c"], + "Flushed sorted by index, not by insertion order"); + t.deepEqual(written.map(({args}) => args), [[], [{readOnly: true}], [{drain: true}]], + "Each entry's args are replayed verbatim via write(resource, ...args)"); +}); + +test("flushWriteBuffer on an empty buffer is a no-op", async (t) => { + let called = false; + await flushWriteBuffer(new Map(), {write: async () => { + called = true; + }}); + t.false(called, "Nothing written for an empty buffer"); +}); diff --git a/packages/fs/test/lib/package-exports.js b/packages/fs/test/lib/package-exports.js index 9000ba782cb..64805c46b52 100644 --- a/packages/fs/test/lib/package-exports.js +++ b/packages/fs/test/lib/package-exports.js @@ -12,7 +12,7 @@ test("export of package.json", (t) => { // Check number of definied exports test("check number of exports", (t) => { const packageJson = require("@ui5/fs/package.json"); - t.is(Object.keys(packageJson.exports).length, 13); + t.is(Object.keys(packageJson.exports).length, 14); }); // Public API contract (exported modules) @@ -78,6 +78,10 @@ test("check number of exports", (t) => { exportedSpecifier: "@ui5/fs/internal/MonitoredResourceTagCollection", mappedModule: "../../lib/MonitoredResourceTagCollection.js" }, + { + exportedSpecifier: "@ui5/fs/internal/stepWriteBuffer", + mappedModule: "../../lib/internal/stepWriteBuffer.js" + }, ].forEach(({exportedSpecifier, mappedModule}) => { test(`${exportedSpecifier}`, async (t) => { const actual = await import(exportedSpecifier); diff --git a/packages/fs/test/lib/resourceFactory.js b/packages/fs/test/lib/resourceFactory.js index 9cee521a892..a140224d8bb 100644 --- a/packages/fs/test/lib/resourceFactory.js +++ b/packages/fs/test/lib/resourceFactory.js @@ -189,6 +189,23 @@ test("createReader: No project", async (t) => { ], "Excludes do not get prefixed."); }); +test("createReader: forwards useGitignore to the adapter", (t) => { + const withGitignore = createReader({ + fsBasePath: "./test/fixtures/application.a/webapp", + virBasePath: "/", + name: "reader name", + useGitignore: true + }); + t.true(withGitignore._readers[0]._useGitignore, "useGitignore reaches the underlying adapter"); + + const withoutGitignore = createReader({ + fsBasePath: "./test/fixtures/application.a/webapp", + virBasePath: "/", + name: "reader name" + }); + t.false(withoutGitignore._readers[0]._useGitignore, "useGitignore defaults to false"); +}); + test("createReader: Throw error missing 'fsBasePath'", (t) => { const error = t.throws(() => createReader({ virBasePath: "/resources/app/", From d04f98b1d888d8ee602d87a2e6a44b0b01ddcded Mon Sep 17 00:00:00 2001 From: Matthias Osswald Date: Wed, 7 Oct 2026 10:54:14 +0200 Subject: [PATCH 2/8] feat(project): Add the step-based build task API with per-step stage caching A custom or built-in task opts into the step-based API by exporting a static stepBased flag and a default build(options) factory that returns an array of step descriptors ({name, keys, each}). Task.getStepBased() reads the flag, replacing the old getSupportsDifferentialBuildsCallback(). The built-in definitions flip their cacheable tasks from supportsDifferentialBuilds to stepBased. StepRunner drives each map step: it resolves the step keys, derives a per-key identity to select the changed keys for a delta build, buffers writes, and records per-key invocation data. TaskRunner calls the factory once at plan time to discover step names for setTasks, then drives StepRunner through per-stage hooks. Each step maps to its own pipeline stage, so BuildStageCache replaces the per-task BuildTaskCache and stageSignature defines the four-component stage signature (project resources, dependency resources, non-resource inputs, root resources). The build cache also tracks non-resource task inputs. MonitoredTaskUtil wraps the TaskUtil handed to a task and records getEnv, getTime, and tracked getProject reads, and routes reader results through a monitored reader so resource reads become cache inputs. TaskInputSet holds the tracked inputs and their signature. quantizeTime buckets a timestamp so getTime stays stable within a granularity, while getBuildTime stays an untracked per-run passthrough. ProjectResources routes tag operations through monitored tag collections so a step's tags are recorded and replayed. Co-authored-by: Merlin Beutlberger --- packages/project/lib/build/ProjectBuilder.js | 4 + packages/project/lib/build/TaskRunner.js | 533 ++++++- .../lib/build/cache/BuildCacheStorage.js | 34 +- .../lib/build/cache/BuildStageCache.js | 520 ++++++ .../project/lib/build/cache/BuildTaskCache.js | 294 ---- .../project/lib/build/cache/CacheManager.js | 22 +- .../lib/build/cache/ProjectBuildCache.js | 1142 +++++++++---- .../lib/build/cache/ResourceRequestManager.js | 99 +- .../project/lib/build/cache/StageCache.js | 8 +- .../lib/build/cache/index/TaskInputSet.js | 231 +++ .../project/lib/build/cache/stageSignature.js | 53 + .../lib/build/definitions/application.js | 9 +- .../lib/build/definitions/component.js | 9 +- .../project/lib/build/definitions/library.js | 13 +- .../lib/build/definitions/themeLibrary.js | 6 +- .../project/lib/build/helpers/BuildContext.js | 36 + .../lib/build/helpers/MonitoredTaskUtil.js | 363 +++++ .../lib/build/helpers/ProjectBuildContext.js | 63 +- .../project/lib/build/helpers/StepRunner.js | 1173 ++++++++++++++ .../project/lib/build/helpers/TaskUtil.js | 94 ++ .../project/lib/build/helpers/quantizeTime.js | 58 + .../project/lib/resources/ProjectResources.js | 75 +- .../lib/specifications/extensions/Task.js | 12 +- .../application.a/task.root-conditional.js | 19 + .../application.a/task.root-config.js | 13 + .../fixtures/application.a/task.root-glob.js | 14 + .../fixtures/application.a/task.step-based.js | 40 + .../ui5-customTask-root-conditional.yaml | 17 + .../ui5-customTask-root-config.yaml | 17 + .../ui5-customTask-root-multi.yaml | 27 + .../ui5-customTask-stepBased.yaml | 17 + .../main/src/library/framework/.library | 11 + .../main/src/library/framework/some.js | 4 + .../fixtures/library.framework/package.json | 9 + .../test/fixtures/library.framework/ui5.yaml | 11 + .../BuildServer.abortRetry.integration.js | 8 +- .../BuildServer.buildSignature.integration.js | 2 +- .../ProjectBuilder.bundling.integration.js | 103 +- .../ProjectBuilder.caching.integration.js | 358 ++++- .../ProjectBuilder.customTasks.integration.js | 133 ++ ...ProjectBuilder.dependencies.integration.js | 42 +- packages/project/test/lib/build/TaskRunner.js | 954 ++++++++++- .../__helper__/ProjectBuilderFixtureTester.js | 13 +- .../test/lib/build/cache/BuildStageCache.js | 961 +++++++++++ .../test/lib/build/cache/BuildTaskCache.js | 526 ------ .../test/lib/build/cache/ProjectBuildCache.js | 670 ++++++-- .../lib/build/cache/ResourceRequestManager.js | 76 +- .../test/lib/build/cache/StageCache.js | 26 + .../lib/build/cache/index/TaskInputSet.js | 180 +++ .../test/lib/build/cache/stageSignature.js | 51 + .../test/lib/build/definitions/application.js | 31 +- .../test/lib/build/definitions/component.js | 20 +- .../test/lib/build/definitions/library.js | 55 +- .../lib/build/definitions/themeLibrary.js | 12 +- .../test/lib/build/helpers/BuildContext.js | 28 + .../lib/build/helpers/MonitoredTaskUtil.js | 324 ++++ .../lib/build/helpers/ProjectBuildContext.js | 99 ++ .../test/lib/build/helpers/StepRunner.js | 1406 +++++++++++++++++ .../test/lib/build/helpers/TaskUtil.js | 100 ++ .../test/lib/build/helpers/quantizeTime.js | 60 + .../test/lib/resources/ProjectResources.js | 49 + .../lib/specifications/extensions/Task.js | 12 - 62 files changed, 9807 insertions(+), 1542 deletions(-) create mode 100644 packages/project/lib/build/cache/BuildStageCache.js delete mode 100644 packages/project/lib/build/cache/BuildTaskCache.js create mode 100644 packages/project/lib/build/cache/index/TaskInputSet.js create mode 100644 packages/project/lib/build/cache/stageSignature.js create mode 100644 packages/project/lib/build/helpers/MonitoredTaskUtil.js create mode 100644 packages/project/lib/build/helpers/StepRunner.js create mode 100644 packages/project/lib/build/helpers/quantizeTime.js create mode 100644 packages/project/test/fixtures/application.a/task.root-conditional.js create mode 100644 packages/project/test/fixtures/application.a/task.root-config.js create mode 100644 packages/project/test/fixtures/application.a/task.root-glob.js create mode 100644 packages/project/test/fixtures/application.a/task.step-based.js create mode 100644 packages/project/test/fixtures/application.a/ui5-customTask-root-conditional.yaml create mode 100644 packages/project/test/fixtures/application.a/ui5-customTask-root-config.yaml create mode 100644 packages/project/test/fixtures/application.a/ui5-customTask-root-multi.yaml create mode 100644 packages/project/test/fixtures/application.a/ui5-customTask-stepBased.yaml create mode 100644 packages/project/test/fixtures/library.framework/main/src/library/framework/.library create mode 100644 packages/project/test/fixtures/library.framework/main/src/library/framework/some.js create mode 100644 packages/project/test/fixtures/library.framework/package.json create mode 100644 packages/project/test/fixtures/library.framework/ui5.yaml create mode 100644 packages/project/test/lib/build/cache/BuildStageCache.js delete mode 100644 packages/project/test/lib/build/cache/BuildTaskCache.js create mode 100644 packages/project/test/lib/build/cache/index/TaskInputSet.js create mode 100644 packages/project/test/lib/build/cache/stageSignature.js create mode 100644 packages/project/test/lib/build/helpers/MonitoredTaskUtil.js create mode 100644 packages/project/test/lib/build/helpers/StepRunner.js create mode 100644 packages/project/test/lib/build/helpers/quantizeTime.js diff --git a/packages/project/lib/build/ProjectBuilder.js b/packages/project/lib/build/ProjectBuilder.js index 4d28d9ad0e6..15584776f9b 100644 --- a/packages/project/lib/build/ProjectBuilder.js +++ b/packages/project/lib/build/ProjectBuilder.js @@ -412,6 +412,8 @@ class ProjectBuilder { throw new Error("A build is already running"); } this.#buildIsRunning = true; + // Fix a single timestamp for this run's time quantization (see BuildContext#refreshBuildTime) + this._buildContext.refreshBuildTime(); let cleanupSigHooks; const pCacheWrites = []; try { @@ -533,6 +535,8 @@ class ProjectBuilder { throw new Error("A build is already running"); } this.#buildIsRunning = true; + // Fix a single timestamp for this run's time quantization (see BuildContext#refreshBuildTime) + this._buildContext.refreshBuildTime(); try { // Initialize (or reuse) the build contexts for the requested projects and their // transitive build-time dependencies, mirroring what #build does. Validation is diff --git a/packages/project/lib/build/TaskRunner.js b/packages/project/lib/build/TaskRunner.js index 763743bf993..e61900e3eca 100644 --- a/packages/project/lib/build/TaskRunner.js +++ b/packages/project/lib/build/TaskRunner.js @@ -1,7 +1,106 @@ import {getLogger} from "@ui5/logger"; import composeTaskList from "./helpers/composeTaskList.js"; +import MonitoredTaskUtil from "./helpers/MonitoredTaskUtil.js"; +import StepRunner, {validateSteps} from "./helpers/StepRunner.js"; import {createReaderCollection, createMonitor} from "@ui5/fs/resourceFactory"; +const EMPTY_RESOURCE_REQUESTS = {paths: [], patterns: []}; + +/** + * Concatenates two resource-request recordings into one. + * + * base is the recording from a reader the TaskRunner monitors directly (the workspace or + * dependencies reader); extra is the corresponding bucket recorded by the + * MonitoredTaskUtil for reads a task made through getProject(name).getReader(). + * + * When base is undefined (no reader was provided to the task) the result stays undefined + * unless the task read resources through the taskUtil, preserving the "intentionally requested no + * dependencies" signal that recordStageResult distinguishes from an empty request set. + * + * @param {{paths: string[], patterns: string[]}|undefined} base Requests from a monitored reader + * @param {{paths: string[], patterns: string[]}} [extra] Requests recorded via the taskUtil + * @returns {{paths: string[], patterns: string[]}|undefined} Merged requests, or undefined + */ +function mergeResourceRequests(base, extra = EMPTY_RESOURCE_REQUESTS) { + if (!base) { + if (!extra.paths.length && !extra.patterns.length) { + return undefined; + } + return {paths: [...extra.paths], patterns: [...extra.patterns]}; + } + return { + paths: [...base.paths, ...extra.paths], + patterns: [...base.patterns, ...extra.patterns], + }; +} + +/** + * Merges two recorded non-resource input sets, deduping by input type and name (the second argument's + * entries win on overlap; equal inputs agree either way). + * + * @param {Array<{type: string, name: string, value: string|undefined}>} base First input recording + * @param {Array<{type: string, name: string, value: string|undefined}>} extra Second input recording + * @returns {Array<{type: string, name: string, value: string|undefined}>} Merged, deduped input recording + */ +function mergeInputRecordings(base, extra) { + const merged = new Map(); + for (const entry of base) { + merged.set(`${entry.type}\0${entry.name}`, entry); + } + for (const entry of extra) { + merged.set(`${entry.type}\0${entry.name}`, entry); + } + return [...merged.values()]; +} + +/** + * Folds a stage's per-key reads (the {@link StepRunner} fold) into the stage-level monitored requests, + * deduping against what the base recording already requests. + * + * The recorder stores resolved paths and the glob patterns a unit issued, and a path or pattern is commonly + * read by more than one key (a shared marker probe, a dependency each key resolves, a shared glob) and also + * recorded at the stage level, so a plain concatenation carried duplicates that only collapse later in the + * request graph (which keys on a Set). Deduping here keeps the recording the request graph rebuilds minimal; + * it does not move the stage signature, since the dropped entries are already present. The patterns matter: a + * cached map-step key does not re-issue its globs, so its patterns reach the request set only through this + * fold, which is how a newly matching file keeps moving the stage signature. + * + * @param {{paths: string[], patterns: string[]}|undefined} base Stage-level monitored requests + * @param {{paths: string[], patterns: string[]}} fold The stage's folded per-key reads and patterns + * @returns {{paths: string[], patterns: string[]}|undefined} The base requests with the not-yet-present + * fold paths and patterns added, or undefined when there was nothing to record (preserving the + * "requested nothing" signal {@link #recordStageResult} distinguishes from an empty request set) + */ +function foldReadsInto(base, fold) { + const basePaths = base ? base.paths : []; + const covered = new Set(basePaths); + const newPaths = []; + for (const path of fold.paths) { + if (covered.has(path)) { + continue; // already requested (by the stage monitor or an earlier fold entry) + } + covered.add(path); + newPaths.push(path); + } + const basePatterns = base ? base.patterns : []; + const coveredPatterns = new Set(basePatterns); + const newPatterns = []; + for (const pattern of fold.patterns) { + if (coveredPatterns.has(pattern)) { + continue; + } + coveredPatterns.add(pattern); + newPatterns.push(pattern); + } + if (!base) { + if (!newPaths.length && !newPatterns.length) { + return undefined; + } + return {paths: newPaths, patterns: newPatterns}; + } + return {paths: [...basePaths, ...newPaths], patterns: [...basePatterns, ...newPatterns]}; +} + /** * TaskRunner * @@ -87,6 +186,8 @@ class TaskRunner { */ async runTasks(signal) { await this._initTasks(); + // Kept for the per-task step runner to check between steps. + this._signal = signal; // Ensure cached dependencies reader is initialized and up-to-date (TODO: improve this lifecycle) await this.getDependenciesReader(this._directDependencies); @@ -108,15 +209,35 @@ class TaskRunner { }); this._log.setTasks(allTasks); - this._buildCache.setTasks(allTasks); + + // Expand step-based tasks into their per-step stages: each step is its own stage, so the + // stage list must enumerate a step-based task's step names in step order. The factory is pure over + // options (it may not read readers/taskUtil) and options are fixed for this build, so one factory + // call produces the steps that both create the stages here and run at execution. Keep the step array + // on the task so the execution path reuses it instead of calling the factory a second time. Freeze it + // so a custom task cannot mutate the shared value between the two uses; discovery re-derives it on the + // next build, so a surviving TaskRunner never serves a stale array. + const stageTasks = await Promise.all(allTasks.map(async (taskName) => { + const taskDef = this._tasks[taskName]; + if (!taskDef.stepBased) { + return {taskName}; + } + const factory = await taskDef.stepFactory(); + const steps = await factory(taskDef.options); + // Validate the whole step list before it creates any stage: setTasks below derives a stage id per + // step name, so a duplicate name or a malformed declaration must be rejected here, not later when + // the step runs (by then two steps sharing a name have already created two stages with one id). + validateSteps(steps, {taskName}); + taskDef.steps = Object.freeze(steps); + return {taskName, stepNames: steps.map((step) => step.name)}; + })); + this._buildCache.setTasks(stageTasks); + for (let i = 0; i < allTasks.length; i++) { signal?.throwIfAborted(); const taskName = allTasks[i]; const taskFunction = this._tasks[taskName].task; - if (i + 1 < allTasks.length) { - this._buildCache.prefetchStageCache(allTasks[i + 1]); - } if (typeof taskFunction === "function") { await this._executeTask(taskName, taskFunction); } @@ -176,15 +297,16 @@ class TaskRunner { * @param {object} [parameters] Task parameters * @param {boolean} [parameters.requiresDependencies=false] * Whether the task requires access to project dependencies - * @param {boolean} [parameters.supportsDifferentialBuilds=false] - * Whether the task supports differential updates using cache + * @param {boolean} [parameters.stepBased=false] + * Whether the task's default export is a step factory build(options) => Step[] driven + * by the step runner, rather than a legacy task body * @param {object} [parameters.options={}] Options to pass to the task * @param {Function|null} [parameters.taskFunction] * Task function to execute, or null to explicitly skip the task * @returns {void} */ _addTask(taskName, { - requiresDependencies = false, supportsDifferentialBuilds = false, options = {}, taskFunction + requiresDependencies = false, stepBased = false, options = {}, taskFunction } = {}) { if (this._tasks[taskName]) { throw new Error(`Failed to add duplicate task ${taskName} for project ${this._project.getName()}`); @@ -194,60 +316,121 @@ class TaskRunner { `It has already been scheduled for execution`); } + // Complete the options before the task is registered, not when it runs. runTasks calls a step-based + // task's factory at plan time to discover its step names, and the factory may branch on + // projectNamespace (generateThemeDesignerResources emits its libraryTheming step only for a + // namespace). Assigning these at execution time left the plan-time call reading an incomplete + // options object, so the discovered step set missed a stage the step runner then asked for. + options.projectName = this._project.getName(); + options.projectNamespace = this._project.getNamespace(); + let task; if (taskFunction === null) { this._log.verbose(`Task ${taskName} is set to be explicitly skipped in definitions.`); task = null; } else { task = async (log) => { - options.projectName = this._project.getName(); - options.projectNamespace = this._project.getNamespace(); + if (!taskFunction) { + const {task} = await this._taskRepository.getTask(taskName); + taskFunction = task; + } + + if (stepBased) { + // Step-based task: the default export is a factory build(options) => Step[]. runTasks + // already called the factory at discovery and kept the returned step array on the task, so + // reuse it here instead of calling the factory a second time. The factory is pure over + // options, so one call per build is the single source of truth for both the stage list and + // execution. Fall back to a direct call for a task invoked outside runTasks, where no + // discovery ran. Each step is its own pipeline stage; the step runner drives one stage per + // step via the per-stage hooks below. Every input a step reads arrives through its + // arguments, so no task body closes over the readers or taskUtil. + const steps = this._tasks[taskName].steps ?? await taskFunction(options); + this._taskStart = performance.now(); + const taskReport = this.#createTaskExecutionReport(taskName); + const stepDriver = new StepRunner({ + steps, + options, + ...this.#createStepStageHooks(taskName, requiresDependencies), + returnValueStore: this._buildCache.getStepReturnValueStore(), + resolveInputValue: this._buildCache.getResolveInputValue(), + applyTagOperations: (tagOperations) => + this._project.getProjectResources().replayTagOperations(tagOperations), + notifyStepExecution: taskReport.started, + signal: this._signal, + }); + const {anyStepExecuted, writtenResourcePaths} = await stepDriver.runSteps(); + if (this._log.isLevelEnabled("perf")) { + this._log.perf( + `Task ${taskName} finished in ${Math.round((performance.now() - this._taskStart))} ms`); + } + // Report the task as skipped when every step was served from cache, else as finished, + // preserving the task-level reporting contract now that caching is per step. The + // matching task-start was emitted by the step runner's notification, before the work. + if (anyStepExecuted) { + taskReport.finished(writtenResourcePaths); + } else { + this._log.skipTask(taskName); + } + return; + } - const cacheInfo = await this._buildCache.prepareTaskExecutionAndValidateCache(taskName); + // Legacy task: the default export is a task body, not a step factory. It has a single stage. + const cacheInfo = await this._buildCache.prepareStageExecutionAndValidateCache(taskName); if (cacheInfo === true) { this._log.skipTask(taskName); return; } - const usingCache = !!(supportsDifferentialBuilds && cacheInfo); const workspace = createMonitor(this._project.getWorkspace()); + let dependencies; + if (requiresDependencies) { + dependencies = createMonitor(this._cachedDependenciesReader); + } + const monitoredTaskUtil = new MonitoredTaskUtil(this._taskUtil); + const params = { workspace, - taskUtil: this._taskUtil, + taskUtil: monitoredTaskUtil, options, }; - - let dependencies; - if (requiresDependencies) { - dependencies = createMonitor(this._cachedDependenciesReader); + if (dependencies) { params.dependencies = dependencies; } - if (usingCache) { - params.changedProjectResourcePaths = cacheInfo.changedProjectResourcePaths; - if (requiresDependencies) { - params.changedDependencyResourcePaths = cacheInfo.changedDependencyResourcePaths; - } - } - if (!taskFunction) { - const {task} = await this._taskRepository.getTask(taskName); - taskFunction = task; - } - this._log.startTask(taskName, usingCache); + this._log.startTask(taskName, !!cacheInfo); this._taskStart = performance.now(); await taskFunction(params); if (this._log.isLevelEnabled("perf")) { this._log.perf( `Task ${taskName} finished in ${Math.round((performance.now() - this._taskStart))} ms`); } - const writtenResourcePaths = await this._buildCache.recordTaskResult(taskName, - workspace.getResourceRequests(), - dependencies?.getResourceRequests(), - usingCache ? cacheInfo : undefined, - supportsDifferentialBuilds); - this._log.endTask(taskName, usingCache, writtenResourcePaths); + const taskUtilRequests = monitoredTaskUtil.getResourceRequests(); + const projectRequests = + mergeResourceRequests(workspace.getResourceRequests(), taskUtilRequests.project); + const dependencyRequests = + mergeResourceRequests(dependencies?.getResourceRequests(), taskUtilRequests.dependencies); + const inputRecording = monitoredTaskUtil.getInputRecording(); + + const writtenResourcePaths = await this._buildCache.recordStageResult({ + taskName, + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + inputRecording, + rootResourceRequests: taskUtilRequests.root, + }); + this._log.endTask(taskName, !!cacheInfo, writtenResourcePaths); }; } this._tasks[taskName] = { task, + stepBased, + // Lazily resolves the task factory so runTasks can enumerate a step-based task's step names for + // setTasks (called only for tasks that actually run). A legacy task keeps this undefined. + stepFactory: stepBased ? (async () => { + if (!taskFunction) { + taskFunction = (await this._taskRepository.getTask(taskName)).task; + } + return taskFunction; + }) : undefined, + options, requiredDependencies: requiresDependencies ? this._directDependencies : new Set() }; this._taskExecutionOrder.push(taskName); @@ -282,7 +465,6 @@ class TaskRunner { const requiredDependenciesCallback = await task.getRequiredDependenciesCallback(); // const buildSignatureCallback = await task.getBuildSignatureCallback(); // const expectedOutputCallback = await task.getExpectedOutputCallback(); - const supportsDifferentialBuildsCallback = await task.getSupportsDifferentialBuildsCallback(); const specVersion = task.getSpecVersion(); let requiredDependencies; @@ -345,11 +527,18 @@ class TaskRunner { } }); } - let supportsDifferentialBuilds = false; - if (specVersion.gte("5.0") && supportsDifferentialBuildsCallback && supportsDifferentialBuildsCallback()) { - supportsDifferentialBuilds = true; - } - + // A custom task opts into the step-factory API with a static `stepBased` export, honored from + // Specification Version 5.0. Below 5.0 the export is ignored and the task runs as a legacy body. + const stepBased = specVersion.gte("5.0") && (await task.getStepBased()) === true; + // Options the factory is called with at step-name discovery (runTasks). The factory is pure over + // options, so discovering step names by calling it early is safe, and the step array it returns is + // kept on the task and reused for execution rather than calling the factory again. + const stepOptions = stepBased ? { + projectName: project.getName(), + projectNamespace: project.getNamespace(), + configuration: taskDef.configuration, + ...(specVersion.gte("3.0") ? {taskName} : {}), + } : undefined; this._tasks[taskName] = { task: this._createCustomTaskWrapper({ task, @@ -358,12 +547,16 @@ class TaskRunner { taskName, taskConfiguration: taskDef.configuration, provideDependenciesReader, - supportsDifferentialBuilds, + stepBased, + stepOptions, getDependenciesReaderCb: () => { // Create the dependencies reader on-demand return this.getDependenciesReader(requiredDependencies); }, }), + stepBased, + stepFactory: stepBased ? (async () => task.getTask()) : undefined, + options: stepOptions, requiredDependencies }; @@ -410,25 +603,21 @@ class TaskRunner { * Callback to get dependencies reader on-demand * @param {boolean} parameters.provideDependenciesReader * Whether to provide dependencies reader to the task - * @param {boolean} parameters.supportsDifferentialBuilds - * Whether the task supports differential updates + * @param {boolean} parameters.stepBased + * Whether the task's default export is a step factory (honored from Specification Version 5.0) + * @param {object} [parameters.stepOptions] + * The options object a step factory is called with at discovery in {@link #runTasks}. Kept so the + * fallback path (a task invoked outside runTasks) calls the factory with the same options * @param {@ui5/project/specifications/Extension} parameters.task Task extension instance * @param {string} parameters.taskName Runtime name of the task (may include suffix) * @param {object} [parameters.taskConfiguration] Task configuration from ui5.yaml * @returns {Function} Async wrapper function for the custom task */ _createCustomTaskWrapper({ - project, taskUtil, getDependenciesReaderCb, provideDependenciesReader, supportsDifferentialBuilds, + project, taskUtil, getDependenciesReaderCb, provideDependenciesReader, stepBased, stepOptions, task, taskName, taskConfiguration }) { return async () => { - const cacheInfo = await this._buildCache.prepareTaskExecutionAndValidateCache(taskName); - if (cacheInfo === true) { - this._log.skipTask(taskName); - return; - } - const usingCache = !!(supportsDifferentialBuilds && cacheInfo); - /* Custom Task Interface Parameters: {Object} parameters Parameters @@ -451,47 +640,233 @@ class TaskRunner { Returns: {Promise} Promise resolving with undefined once data has been written */ - const workspace = createMonitor(this._project.getWorkspace()); - const params = { - workspace, - options: { - projectName: project.getName(), - projectNamespace: project.getNamespace(), - configuration: taskConfiguration, - } - }; - if (usingCache) { - params.changedProjectResourcePaths = cacheInfo.changedProjectResourcePaths; - if (provideDependenciesReader) { - params.changedDependencyResourcePaths = cacheInfo.changedDependencyResourcePaths; - } - } const specVersion = task.getSpecVersion(); const taskUtilInterface = taskUtil.getInterface(specVersion); - // Interface is undefined if specVersion does not support taskUtil - if (taskUtilInterface) { - params.taskUtil = taskUtilInterface; - } const taskFunction = await task.getTask(); + const isSpec3 = specVersion.gte("3.0"); - if (specVersion.gte("3.0")) { - params.options.taskName = taskName; - params.log = getLogger(`builder:custom-task:${taskName}`); + const options = { + projectName: project.getName(), + projectNamespace: project.getNamespace(), + configuration: taskConfiguration, + }; + if (isSpec3) { + options.taskName = taskName; } + if (stepBased) { + // Step-based custom task: gated at Specification Version 5.0 in _addCustomTask, which always + // provides a taskUtil interface. The default export is a factory build(options) => Step[]; + // each step is its own pipeline stage, driven by the step runner via per-stage hooks. The + // factory receives options only. runTasks already called it at discovery and kept the step + // array on the task, so reuse it instead of calling the factory a second time (see the + // standard-task path). Fall back to a direct call for a task invoked outside runTasks. + const factoryOptions = stepOptions ?? options; + const steps = this._tasks[taskName].steps ?? await taskFunction(factoryOptions); + const taskReport = this.#createTaskExecutionReport(taskName); + const stepDriver = new StepRunner({ + steps, + options: factoryOptions, + ...this.#createStepStageHooks(taskName, provideDependenciesReader, taskUtilInterface), + returnValueStore: this._buildCache.getStepReturnValueStore(), + resolveInputValue: this._buildCache.getResolveInputValue(), + applyTagOperations: (tagOperations) => + this._project.getProjectResources().replayTagOperations(tagOperations), + notifyStepExecution: taskReport.started, + signal: this._signal, + }); + const {anyStepExecuted, writtenResourcePaths} = await stepDriver.runSteps(); + // Report the task as skipped when every step was served from cache, else as finished. The + // matching task-start was emitted by the step runner's notification, before the work. + if (anyStepExecuted) { + taskReport.finished(writtenResourcePaths); + } else { + this._log.skipTask(taskName); + } + return; + } + + // Legacy custom task: the default export is a task body, not a step factory. It has a single stage. + const cacheInfo = await this._buildCache.prepareStageExecutionAndValidateCache(taskName); + if (cacheInfo === true) { + this._log.skipTask(taskName); + return; + } + + const workspace = createMonitor(this._project.getWorkspace()); + const params = {workspace, options}; + let dependencies; if (provideDependenciesReader) { dependencies = createMonitor(await getDependenciesReaderCb()); params.dependencies = dependencies; } - this._log.startTask(taskName, usingCache); + + // The interface is undefined for a task that does not support taskUtil (spec version <= 2.1). + let monitoredTaskUtil; + if (taskUtilInterface) { + monitoredTaskUtil = new MonitoredTaskUtil(taskUtilInterface); + params.taskUtil = monitoredTaskUtil; + } + if (isSpec3) { + params.log = getLogger(`builder:custom-task:${taskName}`); + } + + this._log.startTask(taskName, !!cacheInfo); await taskFunction(params); - const writtenResourcePaths = await this._buildCache.recordTaskResult(taskName, - workspace.getResourceRequests(), - dependencies?.getResourceRequests(), - usingCache ? cacheInfo : undefined, - supportsDifferentialBuilds); - this._log.endTask(taskName, usingCache, writtenResourcePaths); + + const taskUtilRequests = monitoredTaskUtil?.getResourceRequests(); + const projectRequests = mergeResourceRequests(workspace.getResourceRequests(), taskUtilRequests?.project); + const dependencyRequests = + mergeResourceRequests(dependencies?.getResourceRequests(), taskUtilRequests?.dependencies); + const inputRecording = monitoredTaskUtil ? monitoredTaskUtil.getInputRecording() : []; + + const writtenResourcePaths = await this._buildCache.recordStageResult({ + taskName, + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + inputRecording, + rootResourceRequests: taskUtilRequests?.root, + }); + this._log.endTask(taskName, !!cacheInfo, writtenResourcePaths); + }; + } + + /** + * Builds the start/end reporting pair for one execution of a step-based task. + * + * A step-based task's skip verdict is only known once every stage has been driven, so the task cannot + * be announced up front like a legacy task. The [StepRunner]{@link StepRunner} instead calls + * started from the first stage that stops being a pure cache hit, before that stage runs + * anything, so task-start ("Running task ...") precedes the work it announces and a + * project-build-status consumer sees the task as running while it runs. A task whose + * every stage was served from cache never calls it and is reported skipped instead. + * + * isDifferentialBuild is taken from the first executing stage's cache verdict, matching + * the legacy path's !!cacheInfo, and is latched for the endTask report so + * both ends of one execution agree. + * + * @param {string} taskName Task name + * @returns {{started: function(boolean): void, finished: function(string[]): void}} Reporting pair + */ + #createTaskExecutionReport(taskName) { + let hasStarted = false; + let isDifferentialBuild = false; + const started = (differential) => { + if (hasStarted) { + return; + } + hasStarted = true; + isDifferentialBuild = !!differential; + this._log.startTask(taskName, isDifferentialBuild); + }; + return { + started, + finished: (writtenResourcePaths) => { + // A task reported as executed always announced itself first; the guard keeps the logger's + // start/end pairing intact even if a future caller reports a finish without a start. + started(isDifferentialBuild); + this._log.endTask(taskName, isDifferentialBuild, writtenResourcePaths); + }, + }; + } + + /** + * Builds the per-stage hooks the {@link StepRunner} uses to drive one pipeline stage per step of a + * step-based task. Shared by the standard-task and custom-task paths. + * + *
    + *
  • prepareStage(stepName) switches the project to the step's own stage and returns + * its cache verdict (true = fully cached, an object = map-step internal key-delta, false = run).
  • + *
  • getPreviousInvocationData(stepName) returns that stage's previous per-key data.
  • + *
  • createStageContext() builds fresh monitored workspace/dependencies readers and a + * MonitoredTaskUtil bound to the stage prepareStage just switched to.
  • + *
  • recordStage(stepName, outcome) records the step's stage from its own monitored + * requests, inputs and stale outputs — no cross-step fold.
  • + *
+ * + * @param {string} taskName Task name + * @param {boolean} requiresDependencies Whether the task's steps read dependencies + * @param {object} [taskUtilInterface] TaskUtil interface for a custom task; defaults to the standard + * task util + * @returns {object} The step-stage hooks + */ + #createStepStageHooks(taskName, requiresDependencies, taskUtilInterface = this._taskUtil) { + return { + prepareStage: async (stepName) => { + const cacheInfo = await this._buildCache.prepareStageExecutionAndValidateCache(taskName, stepName); + if (cacheInfo === true) { + this._log.verbose(`Step ${taskName}/${stepName} served from cache`); + } + return cacheInfo; + }, + getPreviousInvocationData: (stepName) => + this._buildCache.getStepInvocationData(this._buildCache.getStageId(taskName, stepName)), + reopenStage: async (stepName) => { + // A full stage-cache hit whose consumed needs return changed must re-run. Reopen the stage + // with a fresh live writer (the full hit had installed the cached read-only stage) and run + // it as a full execution, so its output reflects the changed producer return. + this._buildCache.reopenStageForRerun(taskName, stepName); + return false; + }, + createStageContext: () => { + // Built after prepareStage switched the stage, so the monitored readers reflect the + // cumulative output of all earlier stages (the reader stack) with this stage's writer on top. + const workspace = createMonitor(this._project.getWorkspace()); + const dependencies = requiresDependencies ? + createMonitor(this._cachedDependenciesReader) : undefined; + const monitoredTaskUtil = new MonitoredTaskUtil(taskUtilInterface); + return {workspace, dependencies, taskUtil: monitoredTaskUtil, monitoredTaskUtil}; + }, + recordStage: async (stepName, outcome) => { + const {ctx, cacheInfo, invocationData, staleOutputs, foldedReads, foldedInputs} = outcome; + const {workspace, dependencies, monitoredTaskUtil} = ctx; + const taskUtilRequests = monitoredTaskUtil.getResourceRequests(); + let projectRequests = + mergeResourceRequests(workspace.getResourceRequests(), taskUtilRequests.project); + let dependencyRequests = + mergeResourceRequests(dependencies?.getResourceRequests(), taskUtilRequests.dependencies); + let inputRecording = monitoredTaskUtil.getInputRecording(); + + // Fold the stage's complete per-key reads and inputs (from its invocation data) into the + // stage-level monitored requests. This covers keys served from cache on a delta build, whose + // reads and inputs the stage-level monitor never observed, so the stage re-keys on its full + // input set. On a full build the monitor already saw every path (through the enumerator's own + // glob or the keys' individual reads), so the fold adds nothing new: foldReadsInto dedups the + // fold against the monitored paths rather than concatenating duplicates that only collapse + // later in the request graph. + if (foldedReads) { + projectRequests = foldReadsInto(projectRequests, foldedReads.project); + dependencyRequests = dependencies ? + foldReadsInto(dependencyRequests, foldedReads.dependencies) : dependencyRequests; + } + if (foldedInputs) { + inputRecording = mergeInputRecordings(inputRecording, foldedInputs); + } + + this._buildCache.setStepInvocationData( + this._buildCache.getStageId(taskName, stepName), invocationData); + + // A map step's stage served a partial (key-delta) run: pass its stale outputs alongside the + // delta's changed paths so recordStageResult drops the outputs its not-re-run keys no longer + // produce from the carried-forward stage. The verdict object is left unmutated here: the + // StepRunner still holds it and #selectStepsToRun already read its changed paths before this + // point, so the extended list is handed over as an explicit field instead. + const changedProjectResourcePaths = cacheInfo ? + [...(cacheInfo.changedProjectResourcePaths ?? []), ...staleOutputs] : undefined; + + return this._buildCache.recordStageResult({ + taskName, + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo: cacheInfo || undefined, + inputRecording, + rootResourceRequests: taskUtilRequests.root, + stepBased: true, + stepName, + changedProjectResourcePaths, + }); + }, }; } diff --git a/packages/project/lib/build/cache/BuildCacheStorage.js b/packages/project/lib/build/cache/BuildCacheStorage.js index 13fb50c491a..35ecf32d89a 100644 --- a/packages/project/lib/build/cache/BuildCacheStorage.js +++ b/packages/project/lib/build/cache/BuildCacheStorage.js @@ -73,10 +73,10 @@ export default class BuildCacheStorage { CREATE TABLE IF NOT EXISTS task_metadata ( project_id TEXT NOT NULL, build_signature TEXT NOT NULL, - task_name TEXT NOT NULL, + stage_id TEXT NOT NULL, type TEXT NOT NULL, data BLOB NOT NULL, - PRIMARY KEY (project_id, build_signature, task_name, type) + PRIMARY KEY (project_id, build_signature, stage_id, type) ) WITHOUT ROWID; CREATE TABLE IF NOT EXISTS result_metadata ( @@ -126,14 +126,14 @@ export default class BuildCacheStorage { (project_id, build_signature, stage_id, stage_signature, data) VALUES (?, ?, ?, ?, ?)` ), - // Task metadata + // Stage request metadata readTaskMetadata: this.#db.prepare( `SELECT data FROM task_metadata - WHERE project_id = ? AND build_signature = ? AND task_name = ? AND type = ?` + WHERE project_id = ? AND build_signature = ? AND stage_id = ? AND type = ?` ), writeTaskMetadata: this.#db.prepare( `INSERT OR REPLACE INTO task_metadata - (project_id, build_signature, task_name, type, data) VALUES (?, ?, ?, ?, ?)` + (project_id, build_signature, stage_id, type, data) VALUES (?, ?, ?, ?, ?)` ), // Result metadata @@ -327,41 +327,41 @@ export default class BuildCacheStorage { } /** - * Reads task metadata from cache + * Reads stage request metadata from cache * * @param {string} projectId Project identifier * @param {string} buildSignature Build signature hash - * @param {string} taskName Task name + * @param {string} stageId Stage id * @param {string} type "project" or "dependency" - * @returns {object|null} Parsed task metadata or null if not found + * @returns {object|null} Parsed stage metadata or null if not found */ - readTaskMetadata(projectId, buildSignature, taskName, type) { + readTaskMetadata(projectId, buildSignature, stageId, type) { try { const row = this.#stmts.readTaskMetadata.get( - projectId, buildSignature, taskName, type + projectId, buildSignature, stageId, type ); return row ? this.#deserializeMetadata(row.data) : null; } catch (err) { throw new Error( - `Failed to read task metadata from cache for ` + - `${projectId} / ${buildSignature} / ${taskName} / ${type}: ${err.message}`, + `Failed to read stage metadata from cache for ` + + `${projectId} / ${buildSignature} / ${stageId} / ${type}: ${err.message}`, {cause: err} ); } } /** - * Writes task metadata to cache + * Writes stage request metadata to cache * * @param {string} projectId Project identifier * @param {string} buildSignature Build signature hash - * @param {string} taskName Task name + * @param {string} stageId Stage id * @param {string} type "project" or "dependency" - * @param {object} metadata Task metadata object to serialize + * @param {object} metadata Stage metadata object to serialize */ - writeTaskMetadata(projectId, buildSignature, taskName, type, metadata) { + writeTaskMetadata(projectId, buildSignature, stageId, type, metadata) { this.#stmts.writeTaskMetadata.run( - projectId, buildSignature, taskName, type, this.#serializeMetadata(metadata) + projectId, buildSignature, stageId, type, this.#serializeMetadata(metadata) ); } diff --git a/packages/project/lib/build/cache/BuildStageCache.js b/packages/project/lib/build/cache/BuildStageCache.js new file mode 100644 index 00000000000..2d49a729818 --- /dev/null +++ b/packages/project/lib/build/cache/BuildStageCache.js @@ -0,0 +1,520 @@ +import {getLogger} from "@ui5/logger"; +import crypto from "node:crypto"; +import ResourceRequestManager from "./ResourceRequestManager.js"; +import TaskInputSet from "./index/TaskInputSet.js"; +import {createStageSignature} from "./stageSignature.js"; +const log = getLogger("build:cache:BuildStageCache"); + +// Root signature of a stage with no recorded root requests: the sha256 digest of an empty signature +// list. Both root request sets are empty for every stage of a standard build (no shipped builder task +// reads through getRootReader), so getRootSignature returns this constant instead of re-hashing. +const EMPTY_ROOT_SIGNATURE = crypto.createHash("sha256").update("").digest("hex"); + +// Serialized form of an empty, unmodified request manager. Restoring a root manager from this (rather +// than constructing a fresh one) marks it clean, so a stage that recorded no root reads is not +// re-persisted on every build. +function emptyRequestManagerCache() { + return {requestSetGraph: {nodes: [], nextId: 1}, rootIndices: [], deltaIndices: [], unusedAtLeastOnce: false}; +} + +/** + * @typedef {object} @ui5/project/build/cache/BuildStageCache~ResourceRequests + * @property {Set} paths Specific resource paths that were accessed + * @property {Set} patterns Glob patterns used to access resources + */ + +/** + * Manages the build cache for a single stage + * + * This class tracks all resources accessed by a stage (both project and dependency resources) + * and maintains a graph of resource request sets. Each request set represents a unique + * combination of resource accesses, enabling efficient cache invalidation and reuse. + * + * Key features: + * - Tracks resource reads using paths and glob patterns + * - Maintains resource indices for different request combinations + * - Supports incremental updates when resources change + * - Provides cache invalidation based on changed resources + * - Serializes/deserializes cache metadata for persistence + * + * The request graph allows derived request sets (when a stage reads additional resources) + * to reuse existing resource indices, optimizing both memory and computation. + * + * @class + */ +export default class BuildStageCache { + #projectName; + #stageId; + #stepBased; + + #projectRequestManager; + #dependencyRequestManager; + + // Resources read through the project's root reader (files outside the UI5 resource model: a + // tsconfig.json in the project root, third-party packages under node_modules). Kept in two + // managers because getRootReader's useGitignore flag changes which resources a glob matches, so a + // request recorded with the flag on must re-materialize against a root reader with the flag on. + // Both are resolved against a dedicated root reader (not the stage-pipeline project reader) and + // their signatures fold into the task's stage signature. + #rootRequestManagers; + + // Tracks non-resource inputs (environment variables and TaskUtil interface reads) recorded during + // the last execution of this task. Its signature is folded into the task's stage signature so that + // a changed input invalidates the cached result. Only entry names are persisted; values are + // re-read on lookup (see #inputSet.getSignatureWithCurrentValues). + #inputSet; + // Whether the input set changed since it was restored from cache (or was freshly recorded), so + // that toCacheObjects/hasNewOrModifiedCacheEntries know it must be persisted. + #inputSetModified = false; + + /** + * Creates a new BuildStageCache instance + * + * @public + * @param {string} projectName Name of the project this stage belongs to + * @param {string} stageId Id of the stage this cache manages + * @param {boolean} stepBased Whether the stage ran the step runner, driving per-step delta tracking + * @param {ResourceRequestManager} [projectRequestManager] Optional pre-existing project request manager from cache + * @param {ResourceRequestManager} [dependencyRequestManager] + * Optional pre-existing dependency request manager from cache + * @param {TaskInputSet} [inputSet] Optional pre-existing task input set from cache + * @param {{gitignore: ResourceRequestManager, noGitignore: ResourceRequestManager}} [rootRequestManagers] + * Optional pre-existing root request managers from cache, keyed by useGitignore + */ + constructor(projectName, stageId, stepBased, projectRequestManager, dependencyRequestManager, + inputSet, rootRequestManagers) { + this.#projectName = projectName; + this.#stageId = stageId; + this.#stepBased = stepBased; + log.verbose(`Initializing BuildStageCache for stage "${stageId}" of project "${this.#projectName}" ` + + `(stepBased=${stepBased})`); + + this.#projectRequestManager = projectRequestManager ?? + new ResourceRequestManager(projectName, stageId, stepBased); + this.#dependencyRequestManager = dependencyRequestManager ?? + new ResourceRequestManager(projectName, stageId, stepBased); + this.#inputSet = inputSet ?? new TaskInputSet(); + // Root requests use full-refresh signatures, not the differential deltas project and dependency + // requests use: a changed root file re-runs the whole stage rather than a differential update. + // This fits the current use case, tracking a few root config files such as tsconfig.json. A + // future use case, bundling many files from outside the UI5 dirs (e.g. node_modules) into the + // build result, would want per-file delta re-runs like project/dependency; that needs a + // changed-root-path signal, list-valued root signatures in the stage delta candidates, and root + // reads threaded into step unit selection, and is left as a separate change. + this.#rootRequestManagers = rootRequestManagers ?? { + gitignore: new ResourceRequestManager(projectName, `${stageId}#root`, false), + noGitignore: new ResourceRequestManager(projectName, `${stageId}#root-no-gitignore`, false), + }; + } + + /** + * Factory method to restore a BuildStageCache from cached data + * + * Deserializes previously cached request managers for both project and dependency resources, + * allowing the stage cache to resume from a prior build state. + * + * @public + * @param {object} options + * @param {string} options.projectName Name of the project + * @param {string} options.stageId Id of the stage + * @param {boolean} options.stepBased Whether the stage ran the step runner, driving per-step delta tracking + * @param {object} options.projectRequests Cached project request manager data + * @param {object} options.dependencyRequests Cached dependency request manager data + * @param {object} [options.inputSet] Cached task input set data + * @param {object} [options.rootRequests] Cached useGitignore:true root request manager data + * @param {object} [options.rootNoGitignoreRequests] Cached useGitignore:false root request manager data + * @returns {BuildStageCache} Restored stage cache instance + */ + static fromCache({ + projectName, stageId, stepBased, projectRequests, dependencyRequests, + inputSet, rootRequests, rootNoGitignoreRequests, + }) { + const projectRequestManager = ResourceRequestManager.fromCache(projectName, stageId, + stepBased, projectRequests); + const dependencyRequestManager = ResourceRequestManager.fromCache(projectName, stageId, + stepBased, dependencyRequests); + // Root managers are optional: absent for stages that made no root reads, and absent in caches + // written before root tracking existed. A missing entry restores a clean empty manager (not a + // fresh dirty one), so a stage without root reads is not needlessly re-persisted. + const rootRequestManagers = { + gitignore: ResourceRequestManager.fromCache( + projectName, `${stageId}#root`, false, rootRequests ?? emptyRequestManagerCache()), + noGitignore: ResourceRequestManager.fromCache( + projectName, `${stageId}#root-no-gitignore`, false, + rootNoGitignoreRequests ?? emptyRequestManagerCache()), + }; + return new BuildStageCache(projectName, stageId, stepBased, + projectRequestManager, dependencyRequestManager, TaskInputSet.fromCache(inputSet), rootRequestManagers); + } + + // ===== METADATA ACCESS ===== + + /** + * Gets the id of the stage + * + * @public + * @returns {string} Stage id + */ + getStageId() { + return this.#stageId; + } + + /** + * Checks whether the stage ran the step runner, which drives per-step delta tracking + * + * A step-based task tracks resource-request deltas per step, so a later build re-runs only the + * changed steps rather than the whole task. + * + * @public + * @returns {boolean} True if the stage ran the step runner + */ + getStepBased() { + return this.#stepBased; + } + + /** + * Checks whether new or modified cache entries exist + * + * Returns true if either the project or dependency request managers have new or + * modified cache entries that need to be persisted. + * + * @public + * @returns {boolean} True if cache entries need to be written + */ + hasNewOrModifiedCacheEntries() { + return this.#projectRequestManager.hasNewOrModifiedCacheEntries() || + this.#dependencyRequestManager.hasNewOrModifiedCacheEntries() || + this.#rootRequestManagers.gitignore.hasNewOrModifiedCacheEntries() || + this.#rootRequestManagers.noGitignore.hasNewOrModifiedCacheEntries() || + this.#inputSetModified; + } + + /** + * Returns the signature of this task's recorded non-resource inputs, computed against the current + * environment and project graph. + * + * Used on cache lookup: each recorded input name is re-evaluated via the given resolver, so the + * returned signature reflects the environment and graph of the build performing the lookup. When + * the task recorded no inputs, a stable empty-input digest is returned. + * + * @public + * @param {function(string, string, (string|undefined)): (string|undefined)} [resolveValue] + * Resolver for the current value of an input (see + * {@link @ui5/project/build/cache/index/TaskInputSet#getSignatureWithCurrentValues}) + * @returns {string} Input signature + */ + getInputSignature(resolveValue) { + return this.#inputSet.getSignatureWithCurrentValues(resolveValue); + } + + /** + * Returns whether this task recorded any root resource requests + * + * @public + * @returns {boolean} + */ + hasRootRequests() { + return this.#rootRequestManagers.gitignore.hasRequests() || + this.#rootRequestManagers.noGitignore.hasRequests(); + } + + /** + * Refreshes both root resource indices against the current project root. + * + * Root files (a tsconfig.json, third-party packages under node_modules) live outside the source + * and dependency readers and are not reported through the incremental change signal, so a full + * refresh runs at the start of every build from cache. Each manager resolves against a root reader + * built with the matching useGitignore flag, since the same recorded glob matches a different + * resource set with the flag on versus off. + * + * @public + * @param {function(boolean): module:@ui5/fs.AbstractReader} getRootReader + * Factory returning a project root reader for the given useGitignore flag + * @returns {Promise} + */ + async refreshRootIndices(getRootReader) { + await Promise.all([ + this.#rootRequestManagers.gitignore.refreshIndices(getRootReader(true)), + this.#rootRequestManagers.noGitignore.refreshIndices(getRootReader(false)), + ]); + } + + /** + * Returns a single signature aggregating the current signatures of both root request sets. + * + * Folded into the task's stage signature so a changed root file misses the cached stage. Unlike + * the project and dependency components, root requests are not delta-tracked: the aggregate is one + * value, so any root change re-runs the whole task. A task with no recorded root requests yields a + * stable digest that stays constant across builds. + * + * @public + * @returns {string} Aggregated root signature + */ + getRootSignature() { + const signatures = [ + ...this.#rootRequestManagers.gitignore.getIndexSignatures(), + ...this.#rootRequestManagers.noGitignore.getIndexSignatures(), + ]; + if (signatures.length === 0) { + return EMPTY_ROOT_SIGNATURE; + } + return crypto.createHash("sha256").update(signatures.sort().join("\0")).digest("hex"); + } + + /** + * Updates project resource indices based on changed resource paths + * + * Processes changed resource paths and updates the project request manager's indices + * accordingly. Only relevant resources (those matching recorded requests) are processed. + * + * @public + * @param {module:@ui5/fs.AbstractReader} projectReader Reader for accessing project resources + * @param {string[]} changedProjectResourcePaths Array of changed project resource paths + * @returns {Promise} True if any index has changed + */ + updateProjectIndices(projectReader, changedProjectResourcePaths) { + return this.#projectRequestManager.updateIndices(projectReader, changedProjectResourcePaths); + } + + /** + * Updates dependency resource indices based on changed resource paths + * + * Processes changed dependency resource paths and updates the dependency request manager's + * indices accordingly. Only relevant resources (those matching recorded requests) are processed. + * + * @public + * @param {module:@ui5/fs.AbstractReader} dependencyReader Reader for accessing dependency resources + * @param {string[]} changedDepResourcePaths Array of changed dependency resource paths + * @returns {Promise} True if any index has changed + */ + updateDependencyIndices(dependencyReader, changedDepResourcePaths) { + return this.#dependencyRequestManager.updateIndices(dependencyReader, changedDepResourcePaths); + } + + /** + * Returns whether this task has any recorded dependency resource requests + * + * @public + * @returns {boolean} + */ + hasDependencyRequests() { + return this.#dependencyRequestManager.hasRequests(); + } + + /** + * Performs a full refresh of the dependency resource index + * + * Since dependency resources may change independently from this project's cache, a full + * refresh of the dependency index is required at the beginning of every build from cache. + * This ensures all dependency resources are current before task execution. + * + * @public + * @param {module:@ui5/fs.AbstractReader} dependencyReader Reader for accessing dependency resources + * @returns {Promise} + */ + refreshDependencyIndices(dependencyReader) { + return this.#dependencyRequestManager.refreshIndices(dependencyReader); + } + + /** + * Gets all project index signatures for this task + * + * Returns signatures from all recorded project-request sets. Each signature represents + * a unique combination of resources, belonging to the current project, that were accessed + * during task execution. These can be used as cache keys for restoring cached task results. + * + * @public + * @returns {string[]} Array of signature strings + * @throws {Error} If resource index is missing for any request set + */ + getProjectIndexSignatures() { + return this.#projectRequestManager.getIndexSignatures(); + } + + /** + * Gets all dependency index signatures for this task + * + * Returns signatures from all recorded dependency-request sets. Each signature represents + * a unique combination of resources, belonging to all dependencies of the current project, + * that were accessed during task execution. These can be used as cache keys for restoring + * cached task results. + * + * @public + * @returns {string[]} Array of signature strings + * @throws {Error} If resource index is missing for any request set + */ + getDependencyIndexSignatures() { + return this.#dependencyRequestManager.getIndexSignatures(); + } + + /** + * Gets all project index delta transitions for differential updates + * + * Returns a map of signature transitions and their associated changed resource paths + * for project resources. Used when tasks support differential updates to identify + * which resources changed between cache states. + * + * @public + * @returns {Map} Map from original signature to delta information + * containing newSignature and changedPaths array + */ + getProjectIndexDeltas() { + return this.#projectRequestManager.getDeltas(); + } + + /** + * Gets all dependency index delta transitions for differential updates + * + * Returns a map of signature transitions and their associated changed resource paths + * for dependency resources. Used when tasks support differential updates to identify + * which dependency resources changed between cache states. + * + * @public + * @returns {Map} Map from original signature to delta information + * containing newSignature and changedPaths array + */ + getDependencyIndexDeltas() { + return this.#dependencyRequestManager.getDeltas(); + } + + /** + * Builds this stage's exact-match signatures from the current index state: the cartesian product of + * the recorded project-request signatures with the recorded dependency-request signatures, each + * paired with the stage's current non-resource input signature and root signature into a full + * [project, dependency, input, root] tuple. + * + * This is the single definition of the stage-signature composition. The delta path in + * {@link @ui5/project/build/cache/ProjectBuildCache} pairs a changed project/dependency signature + * with the same input and root signatures; see {@link #getInputSignature} and + * {@link #getRootSignature} for how those two are re-evaluated against the current environment and + * project root. + * + * @public + * @param {function(string, string, (string|undefined)): (string|undefined)} [resolveInputValue] + * Resolver for the current value of a recorded non-resource input (see {@link #getInputSignature}) + * @returns {string[]} Exact-match stage signatures for the current index state + */ + getStageSignatures(resolveInputValue) { + const inputSignature = this.getInputSignature(resolveInputValue); + const rootSignature = this.getRootSignature(); + const signatures = []; + for (const projectSignature of this.getProjectIndexSignatures()) { + for (const dependencySignature of this.getDependencyIndexSignatures()) { + signatures.push(createStageSignature( + [projectSignature, dependencySignature, inputSignature, rootSignature])); + } + } + return signatures; + } + + /** + * Records resource requests and calculates signatures for the stage + * + * This method: + * 1. Processes project and dependency resource requests + * 2. Searches for exact matches in the request graphs + * 3. If found, returns the existing index signatures + * 4. If not found, creates new request sets and resource indices + * 5. Uses tree derivation when possible to reuse parent indices + * + * The returned signatures uniquely identify the set of resources accessed and their + * content, enabling cache lookup for previously executed stage results. + * + * @public + * @param {object} options + * @param {@ui5/project/build/cache/BuildStageCache~ResourceRequests} options.projectRequestRecording + * Project resource requests (paths and patterns) + * @param {@ui5/project/build/cache/BuildStageCache~ResourceRequests|undefined} + * options.dependencyRequestRecording Dependency resource requests (paths and patterns) + * @param {module:@ui5/fs.AbstractReader} options.projectReader Reader for accessing project resources + * @param {module:@ui5/fs.AbstractReader} options.dependencyReader Reader for accessing dependency resources + * @param {Array<{type: string, name: string, value: string|undefined}>} [options.inputRecording] + * Non-resource inputs (environment variables, TaskUtil interface reads) recorded during stage + * execution + * @param {{gitignore: @ui5/project/build/cache/BuildStageCache~ResourceRequests, + * noGitignore: @ui5/project/build/cache/BuildStageCache~ResourceRequests}} [options.rootRequestRecording] + * Root resource requests, keyed by the useGitignore flag they were read with + * @param {function(boolean): module:@ui5/fs.AbstractReader} [options.getRootReader] + * Factory returning a project root reader for the given useGitignore flag + * @returns {Promise} + * Array containing [projectSignature, dependencySignature, inputSignature, rootSignature] + */ + async recordRequests({ + projectRequestRecording, dependencyRequestRecording, projectReader, dependencyReader, + inputRecording = [], rootRequestRecording, getRootReader, + }) { + const { + setId: projectReqSetId, signature: projectReqSignature + } = await this.#projectRequestManager.addRequests(projectRequestRecording, projectReader); + + let dependencyReqSignature; + if (dependencyRequestRecording) { + const { + setId: depReqSetId, signature: depReqSignature + } = await this.#dependencyRequestManager.addRequests(dependencyRequestRecording, dependencyReader); + + this.#projectRequestManager.addAffiliatedRequestSet(projectReqSetId, depReqSetId); + dependencyReqSignature = depReqSignature; + } else { + dependencyReqSignature = this.#dependencyRequestManager.recordNoRequests(); + } + + // Record the non-resource inputs consumed by this execution. Rebuild the set from the + // recording; if the recorded entry set differs from what was restored/recorded before, flag + // it for persistence. + const newInputSet = new TaskInputSet(inputRecording); + if (newInputSet.getSignature() !== this.#inputSet.getSignature()) { + this.#inputSetModified = true; + } + this.#inputSet = newInputSet; + + // Record root requests against a root reader built with the matching useGitignore flag. A bucket + // with reads records them. A bucket that recorded reads on an earlier build but has none now is + // cleared, so getRootSignature stops folding resources the stage no longer reads and the emptied + // manager is persisted (overwriting the stored request set). A bucket that was always empty is + // left untouched, so a stage without root reads writes no root metadata. + if (rootRequestRecording && getRootReader) { + const recordBucket = (manager, recording, useGitignore) => { + if (recording && (recording.paths.length || recording.patterns.length)) { + return manager.addRequests(recording, getRootReader(useGitignore)); + } + if (manager.hasRequests()) { + manager.clear(); + } + return Promise.resolve(); + }; + await Promise.all([ + recordBucket(this.#rootRequestManagers.gitignore, rootRequestRecording.gitignore, true), + recordBucket(this.#rootRequestManagers.noGitignore, rootRequestRecording.noGitignore, false), + ]); + } + + return [projectReqSignature, dependencyReqSignature, this.#inputSet.getSignature(), this.getRootSignature()]; + } + + /** + * Serializes the task cache to plain objects for persistence + * + * Exports both project and dependency resource request graphs in a format suitable + * for JSON serialization. The serialized data can be passed to fromCache() to restore + * the cache state. Returns undefined for request managers with no new or modified entries. + * + * @public + * @returns {Array} Array containing + * [projectCacheObject, dependencyCacheObject, inputCacheObject, + * rootCacheObject, rootNoGitignoreCacheObject] + */ + toCacheObjects() { + const {gitignore, noGitignore} = this.#rootRequestManagers; + return [ + this.#projectRequestManager.toCacheObject(), + this.#dependencyRequestManager.toCacheObject(), + this.#inputSet.isEmpty() ? undefined : this.#inputSet.toCacheObject(), + // Persist a root manager that recorded requests, or one cleared this build so the now-empty + // state overwrites the stored request set. A manager that was always empty writes no root + // metadata, keeping a stage without root reads free of root rows. + gitignore.hasRequests() || gitignore.wasCleared() ? gitignore.toCacheObject() : undefined, + noGitignore.hasRequests() || noGitignore.wasCleared() ? noGitignore.toCacheObject() : undefined, + ]; + } +} diff --git a/packages/project/lib/build/cache/BuildTaskCache.js b/packages/project/lib/build/cache/BuildTaskCache.js deleted file mode 100644 index d01410f9e50..00000000000 --- a/packages/project/lib/build/cache/BuildTaskCache.js +++ /dev/null @@ -1,294 +0,0 @@ -import {getLogger} from "@ui5/logger"; -import ResourceRequestManager from "./ResourceRequestManager.js"; -const log = getLogger("build:cache:BuildTaskCache"); - -/** - * @typedef {object} @ui5/project/build/cache/BuildTaskCache~ResourceRequests - * @property {Set} paths Specific resource paths that were accessed - * @property {Set} patterns Glob patterns used to access resources - */ - -/** - * Manages the build cache for a single task - * - * This class tracks all resources accessed by a task (both project and dependency resources) - * and maintains a graph of resource request sets. Each request set represents a unique - * combination of resource accesses, enabling efficient cache invalidation and reuse. - * - * Key features: - * - Tracks resource reads using paths and glob patterns - * - Maintains resource indices for different request combinations - * - Supports incremental updates when resources change - * - Provides cache invalidation based on changed resources - * - Serializes/deserializes cache metadata for persistence - * - * The request graph allows derived request sets (when a task reads additional resources) - * to reuse existing resource indices, optimizing both memory and computation. - * - * @class - */ -export default class BuildTaskCache { - #projectName; - #taskName; - #supportsDifferentialBuilds; - - #projectRequestManager; - #dependencyRequestManager; - - /** - * Creates a new BuildTaskCache instance - * - * @public - * @param {string} projectName Name of the project this task belongs to - * @param {string} taskName Name of the task this cache manages - * @param {boolean} supportsDifferentialBuilds Whether the task supports differential updates - * @param {ResourceRequestManager} [projectRequestManager] Optional pre-existing project request manager from cache - * @param {ResourceRequestManager} [dependencyRequestManager] - * Optional pre-existing dependency request manager from cache - */ - constructor(projectName, taskName, supportsDifferentialBuilds, projectRequestManager, dependencyRequestManager) { - this.#projectName = projectName; - this.#taskName = taskName; - this.#supportsDifferentialBuilds = supportsDifferentialBuilds; - log.verbose(`Initializing BuildTaskCache for task "${taskName}" of project "${this.#projectName}" ` + - `(supportsDifferentialBuilds=${supportsDifferentialBuilds})`); - - this.#projectRequestManager = projectRequestManager ?? - new ResourceRequestManager(projectName, taskName, supportsDifferentialBuilds); - this.#dependencyRequestManager = dependencyRequestManager ?? - new ResourceRequestManager(projectName, taskName, supportsDifferentialBuilds); - } - - /** - * Factory method to restore a BuildTaskCache from cached data - * - * Deserializes previously cached request managers for both project and dependency resources, - * allowing the task cache to resume from a prior build state. - * - * @public - * @param {string} projectName Name of the project - * @param {string} taskName Name of the task - * @param {boolean} supportsDifferentialBuilds Whether the task supports differential updates - * @param {object} projectRequests Cached project request manager data - * @param {object} dependencyRequests Cached dependency request manager data - * @returns {BuildTaskCache} Restored task cache instance - */ - static fromCache(projectName, taskName, supportsDifferentialBuilds, projectRequests, dependencyRequests) { - const projectRequestManager = ResourceRequestManager.fromCache(projectName, taskName, - supportsDifferentialBuilds, projectRequests); - const dependencyRequestManager = ResourceRequestManager.fromCache(projectName, taskName, - supportsDifferentialBuilds, dependencyRequests); - return new BuildTaskCache(projectName, taskName, supportsDifferentialBuilds, - projectRequestManager, dependencyRequestManager); - } - - // ===== METADATA ACCESS ===== - - /** - * Gets the name of the task - * - * @public - * @returns {string} Task name - */ - getTaskName() { - return this.#taskName; - } - - /** - * Checks whether the task supports differential updates - * - * Tasks that support differential updates can use incremental cache invalidation, - * processing only changed resources rather than rebuilding from scratch. - * - * @public - * @returns {boolean} True if differential updates are supported - */ - getSupportsDifferentialBuilds() { - return this.#supportsDifferentialBuilds; - } - - /** - * Checks whether new or modified cache entries exist - * - * Returns true if either the project or dependency request managers have new or - * modified cache entries that need to be persisted. - * - * @public - * @returns {boolean} True if cache entries need to be written - */ - hasNewOrModifiedCacheEntries() { - return this.#projectRequestManager.hasNewOrModifiedCacheEntries() || - this.#dependencyRequestManager.hasNewOrModifiedCacheEntries(); - } - - /** - * Updates project resource indices based on changed resource paths - * - * Processes changed resource paths and updates the project request manager's indices - * accordingly. Only relevant resources (those matching recorded requests) are processed. - * - * @public - * @param {module:@ui5/fs.AbstractReader} projectReader Reader for accessing project resources - * @param {string[]} changedProjectResourcePaths Array of changed project resource paths - * @returns {Promise} True if any index has changed - */ - updateProjectIndices(projectReader, changedProjectResourcePaths) { - return this.#projectRequestManager.updateIndices(projectReader, changedProjectResourcePaths); - } - - /** - * Updates dependency resource indices based on changed resource paths - * - * Processes changed dependency resource paths and updates the dependency request manager's - * indices accordingly. Only relevant resources (those matching recorded requests) are processed. - * - * @public - * @param {module:@ui5/fs.AbstractReader} dependencyReader Reader for accessing dependency resources - * @param {string[]} changedDepResourcePaths Array of changed dependency resource paths - * @returns {Promise} True if any index has changed - */ - updateDependencyIndices(dependencyReader, changedDepResourcePaths) { - return this.#dependencyRequestManager.updateIndices(dependencyReader, changedDepResourcePaths); - } - - /** - * Returns whether this task has any recorded dependency resource requests - * - * @public - * @returns {boolean} - */ - hasDependencyRequests() { - return this.#dependencyRequestManager.hasRequests(); - } - - /** - * Performs a full refresh of the dependency resource index - * - * Since dependency resources may change independently from this project's cache, a full - * refresh of the dependency index is required at the beginning of every build from cache. - * This ensures all dependency resources are current before task execution. - * - * @public - * @param {module:@ui5/fs.AbstractReader} dependencyReader Reader for accessing dependency resources - * @returns {Promise} - */ - refreshDependencyIndices(dependencyReader) { - return this.#dependencyRequestManager.refreshIndices(dependencyReader); - } - - /** - * Gets all project index signatures for this task - * - * Returns signatures from all recorded project-request sets. Each signature represents - * a unique combination of resources, belonging to the current project, that were accessed - * during task execution. These can be used as cache keys for restoring cached task results. - * - * @public - * @returns {string[]} Array of signature strings - * @throws {Error} If resource index is missing for any request set - */ - getProjectIndexSignatures() { - return this.#projectRequestManager.getIndexSignatures(); - } - - /** - * Gets all dependency index signatures for this task - * - * Returns signatures from all recorded dependency-request sets. Each signature represents - * a unique combination of resources, belonging to all dependencies of the current project, - * that were accessed during task execution. These can be used as cache keys for restoring - * cached task results. - * - * @public - * @returns {string[]} Array of signature strings - * @throws {Error} If resource index is missing for any request set - */ - getDependencyIndexSignatures() { - return this.#dependencyRequestManager.getIndexSignatures(); - } - - /** - * Gets all project index delta transitions for differential updates - * - * Returns a map of signature transitions and their associated changed resource paths - * for project resources. Used when tasks support differential updates to identify - * which resources changed between cache states. - * - * @public - * @returns {Map} Map from original signature to delta information - * containing newSignature and changedPaths array - */ - getProjectIndexDeltas() { - return this.#projectRequestManager.getDeltas(); - } - - /** - * Gets all dependency index delta transitions for differential updates - * - * Returns a map of signature transitions and their associated changed resource paths - * for dependency resources. Used when tasks support differential updates to identify - * which dependency resources changed between cache states. - * - * @public - * @returns {Map} Map from original signature to delta information - * containing newSignature and changedPaths array - */ - getDependencyIndexDeltas() { - return this.#dependencyRequestManager.getDeltas(); - } - - /** - * Records resource requests and calculates signatures for the task - * - * This method: - * 1. Processes project and dependency resource requests - * 2. Searches for exact matches in the request graphs - * 3. If found, returns the existing index signatures - * 4. If not found, creates new request sets and resource indices - * 5. Uses tree derivation when possible to reuse parent indices - * - * The returned signatures uniquely identify the set of resources accessed and their - * content, enabling cache lookup for previously executed task results. - * - * @public - * @param {@ui5/project/build/cache/BuildTaskCache~ResourceRequests} projectRequestRecording - * Project resource requests (paths and patterns) - * @param {@ui5/project/build/cache/BuildTaskCache~ResourceRequests|undefined} dependencyRequestRecording - * Dependency resource requests (paths and patterns) - * @param {module:@ui5/fs.AbstractReader} projectReader Reader for accessing project resources - * @param {module:@ui5/fs.AbstractReader} dependencyReader Reader for accessing dependency resources - * @returns {Promise} Array containing [projectSignature, dependencySignature] - */ - async recordRequests(projectRequestRecording, dependencyRequestRecording, projectReader, dependencyReader) { - const { - setId: projectReqSetId, signature: projectReqSignature - } = await this.#projectRequestManager.addRequests(projectRequestRecording, projectReader); - - let dependencyReqSignature; - if (dependencyRequestRecording) { - const { - setId: depReqSetId, signature: depReqSignature - } = await this.#dependencyRequestManager.addRequests(dependencyRequestRecording, dependencyReader); - - this.#projectRequestManager.addAffiliatedRequestSet(projectReqSetId, depReqSetId); - dependencyReqSignature = depReqSignature; - } else { - dependencyReqSignature = this.#dependencyRequestManager.recordNoRequests(); - } - return [projectReqSignature, dependencyReqSignature]; - } - - /** - * Serializes the task cache to plain objects for persistence - * - * Exports both project and dependency resource request graphs in a format suitable - * for JSON serialization. The serialized data can be passed to fromCache() to restore - * the cache state. Returns undefined for request managers with no new or modified entries. - * - * @public - * @returns {Array} Array containing [projectCacheObject, dependencyCacheObject] - */ - toCacheObjects() { - return [this.#projectRequestManager.toCacheObject(), this.#dependencyRequestManager.toCacheObject()]; - } -} diff --git a/packages/project/lib/build/cache/CacheManager.js b/packages/project/lib/build/cache/CacheManager.js index d1baec61eb0..8cab156e517 100644 --- a/packages/project/lib/build/cache/CacheManager.js +++ b/packages/project/lib/build/cache/CacheManager.js @@ -11,7 +11,7 @@ const log = getLogger("build:cache:CacheManager"); const cacheManagerInstances = new Map(); // Cache version for compatibility management -export const CACHE_VERSION = "v0_7"; +export const CACHE_VERSION = "v0_9"; /** * Manages persistence for the build cache using a unified SQLite-backed storage @@ -154,31 +154,31 @@ export default class CacheManager { } /** - * Reads task metadata from cache + * Reads stage request metadata from cache * * @public * @param {string} projectId Project identifier * @param {string} buildSignature Build signature hash - * @param {string} taskName Task name + * @param {string} stageId Stage id * @param {string} type "project" or "dependency" - * @returns {object|null} Parsed task metadata or null if not found + * @returns {object|null} Parsed stage metadata or null if not found */ - readTaskMetadata(projectId, buildSignature, taskName, type) { - return this.#storage.readTaskMetadata(projectId, buildSignature, taskName, type); + readTaskMetadata(projectId, buildSignature, stageId, type) { + return this.#storage.readTaskMetadata(projectId, buildSignature, stageId, type); } /** - * Writes task metadata to cache + * Writes stage request metadata to cache * * @public * @param {string} projectId Project identifier * @param {string} buildSignature Build signature hash - * @param {string} taskName Task name + * @param {string} stageId Stage id * @param {string} type "project" or "dependency" - * @param {object} metadata Task metadata object to serialize + * @param {object} metadata Stage metadata object to serialize */ - writeTaskMetadata(projectId, buildSignature, taskName, type, metadata) { - this.#storage.writeTaskMetadata(projectId, buildSignature, taskName, type, metadata); + writeTaskMetadata(projectId, buildSignature, stageId, type, metadata) { + this.#storage.writeTaskMetadata(projectId, buildSignature, stageId, type, metadata); } /** diff --git a/packages/project/lib/build/cache/ProjectBuildCache.js b/packages/project/lib/build/cache/ProjectBuildCache.js index 7f9842da87e..0543a7c7327 100644 --- a/packages/project/lib/build/cache/ProjectBuildCache.js +++ b/packages/project/lib/build/cache/ProjectBuildCache.js @@ -4,9 +4,10 @@ import {gzip} from "node:zlib"; import {Readable} from "node:stream"; import crypto from "node:crypto"; import os from "node:os"; -import BuildTaskCache from "./BuildTaskCache.js"; +import BuildStageCache from "./BuildStageCache.js"; import StageCache from "./StageCache.js"; import ResourceIndex from "./index/ResourceIndex.js"; +import {createStageSignature, splitStageSignature, STAGE_SIG_DEPENDENCY_INDEX} from "./stageSignature.js"; import {isResourceUnchanged} from "./utils.js"; const log = getLogger("build:cache:ProjectBuildCache"); import Cache from "./Cache.js"; @@ -50,17 +51,21 @@ export const RESULT_CACHE_STATES = Object.freeze({ * Map of resource paths to their tags that were set or cleared during this stage's execution, for project tags * @property {Map>} buildTagOperations * Map of resource paths to their tags that were set or cleared during this stage's execution, for build tags + * @property {Map} [stepInvocationData] A step-based stage's per-key invocation data, + * restored from the same cache row as the stage output so the two stay paired under one signature */ export default class ProjectBuildCache { - #taskCache = new Map(); + #stageCaches = new Map(); #stageCache = new StageCache(); - #prefetchedStageReads; + // Stage ids in execution order, as established by setTasks. + #stageOrder = []; #project; #buildSignature; #cacheManager; #cacheMode; + #resolveInputValue; #currentProjectReader; #currentDependencyReader; #sourceIndex; @@ -69,6 +74,12 @@ export default class ProjectBuildCache { #cachedResultSignature; #currentResultSignature; + // Aggregated root resource signature established when the cache was last validated or built in this + // session. A mismatch on the next validateCache means a root file (a tsconfig.json, a bundled + // node_modules package) changed since, forcing result-cache revalidation even when no source or + // dependency resource changed (root files are not reported through the incremental change signal). + #cachedRootAggregateSignature; + // Dependency-set identity: a hash over the project's transitive dependency ids, computed by the // caller from the graph and passed into validateCache. #cachedDependencySetIdentity is restored // from the persisted source index, #currentDependencySetIdentity reflects the current graph. A @@ -80,7 +91,11 @@ export default class ProjectBuildCache { // Pending changes #changedProjectSourcePaths = []; #changedDependencyResourcePaths = []; + // Written result paths, consumed in insertion order by updateProjectIndices. The parallel Set is + // the membership index: the list grows to the project's full written-resource count and is appended + // to once per written resource per stage, so an Array.includes membership test would be O(n squared). #writtenResultResourcePaths = []; + #writtenResultResourcePathSet = new Set(); // Set of integrity hashes known to already exist in CAS from restored stage metadata. // Populated during the restore phase, consulted during writes to skip redundant CAS lookups. @@ -94,6 +109,24 @@ export default class ProjectBuildCache { #combinedIndexState = INDEX_STATES.RESTORING_PROJECT_INDICES; #resultCacheState = RESULT_CACHE_STATES.PENDING_VALIDATION; + // Per-stage step-runner invocation data (see lib/build/helpers/StepRunner.js), keyed by stage id. + // Each value is a Map of key identity -> {reads, dependencyReads, writes, inputs, needsInputs, + // tagOperations, returns} recorded during the stage's last run. It lets a delta build map a changed + // input back to the key that read it, fold newly-observed reads into the stage's request graph, drop + // outputs a key no longer produces, and rebuild a cached key's returned resource(s) from the CAS. + // + // The persisted copy travels inside the stage's own stage_metadata row (keyed by stage signature, + // see #prepareStageCache / #processStageCacheMetadata), so the per-key map can never pair with a + // different run's stage output. This map is the per-build working copy: a cache lookup stashes the + // matched stage's map here (prepareStageExecutionAndValidateCache) so getStepInvocationData returns + // the signature-matched "previous" data, and a stage that re-records overwrites it via + // setStepInvocationData. + #stepInvocationData = new Map(); + + // Compressed CAS rows for resources returned by the step runner, buffered per unit and flushed in + // one transaction per step via the return value store's flush() (see #flushStepReturns). + #pendingStepReturnCasRows = []; + /** * Creates a new ProjectBuildCache instance * @@ -105,14 +138,19 @@ export default class ProjectBuildCache { * @param {string} buildSignature Build signature for the current build * @param {object|null} cacheManager Cache manager instance for reading/writing cache data * @param {string} cacheMode Cache mode to use for building UI5 projects + * @param {function(string, string): (string|undefined)} [resolveInputValue] + * Resolver for the current value of a recorded non-resource task input, given its type and + * name. Provided by the ProjectBuildContext (which can reach the project graph). When omitted, + * only environment-variable inputs are re-read (from process.env). */ - constructor(project, buildSignature, cacheManager, cacheMode) { + constructor(project, buildSignature, cacheManager, cacheMode, resolveInputValue) { log.verbose( `ProjectBuildCache for project ${project.getName()} uses build signature ${buildSignature}`); this.#project = project; this.#buildSignature = buildSignature; this.#cacheManager = cacheManager; this.#cacheMode = cacheMode; + this.#resolveInputValue = resolveInputValue; } /** @@ -154,7 +192,7 @@ export default class ProjectBuildCache { * project build: discards any in-memory StageCache entries left over from a prior aborted * build (successful builds flush the queue in writeCache, so this is a no-op in the common * case) and captures the current project and dependency readers for later use by - * recordTaskResult. + * recordStageResult. * * @public * @param {@ui5/fs/AbstractReader} dependencyReader Reader for dependency resources, used to @@ -243,6 +281,29 @@ export default class ProjectBuildCache { this.#combinedIndexState = INDEX_STATES.FRESH; } + // Root resources (a tsconfig.json, third-party packages a task bundles from node_modules) live + // outside the source and dependency readers and are not reported through the incremental change + // signal. Refresh their indices against the current project root and, if their aggregate + // signature moved since the cache was last validated, force result-cache revalidation so a root + // change is not skipped when no source or dependency resource changed. + if (this.#combinedIndexState === INDEX_STATES.FRESH && this.#anyTaskHasRootRequests()) { + const rootStart = performance.now(); + await this.#refreshRootIndices(); + const rootAggregate = this.#getAggregatedRootSignature(); + if (this.#cachedRootAggregateSignature !== undefined && + rootAggregate !== this.#cachedRootAggregateSignature) { + log.verbose(`Root resources changed for project ${this.#project.getName()}, ` + + `revalidating result cache`); + this.#resultCacheState = RESULT_CACHE_STATES.PENDING_VALIDATION; + } + this.#cachedRootAggregateSignature = rootAggregate; + if (log.isLevelEnabled("perf")) { + log.perf( + `Refreshed root indices for project ${this.#project.getName()} ` + + `in ${(performance.now() - rootStart).toFixed(2)} ms`); + } + } + if (this.#resultCacheState === RESULT_CACHE_STATES.PENDING_VALIDATION) { log.verbose(`Project ${this.#project.getName()} cache requires validation due to detected changes.`); const findStart = performance.now(); @@ -289,10 +350,10 @@ export default class ProjectBuildCache { let depIndicesChanged = false; if (this.#changedDependencyResourcePaths.length) { const depStart = performance.now(); - const tasksWithDepRequests = Array.from(this.#taskCache.values()) - .filter((taskCache) => taskCache.hasDependencyRequests()); - await Promise.all(tasksWithDepRequests.map(async (taskCache) => { - const changed = await taskCache + const tasksWithDepRequests = Array.from(this.#stageCaches.values()) + .filter((stageCache) => stageCache.hasDependencyRequests()); + await Promise.all(tasksWithDepRequests.map(async (stageCache) => { + const changed = await stageCache .updateDependencyIndices(dependencyReader, this.#changedDependencyResourcePaths); if (changed) { depIndicesChanged = true; @@ -303,7 +364,7 @@ export default class ProjectBuildCache { `#flushPendingChanges updateDependencyIndices for project ${this.#project.getName()} ` + `completed in ${(performance.now() - depStart).toFixed(2)} ms ` + `(${this.#changedDependencyResourcePaths.length} changed paths, ` + - `${tasksWithDepRequests.length}/${this.#taskCache.size} tasks, changed=${depIndicesChanged})`); + `${tasksWithDepRequests.length}/${this.#stageCaches.size} tasks, changed=${depIndicesChanged})`); } } @@ -328,10 +389,10 @@ export default class ProjectBuildCache { * @returns {Promise} */ async _refreshDependencyIndices(dependencyReader) { - const tasksWithDepRequests = Array.from(this.#taskCache.values()) - .filter((taskCache) => taskCache.hasDependencyRequests()); - await Promise.all(tasksWithDepRequests.map(async (taskCache) => { - await taskCache.refreshDependencyIndices(dependencyReader); + const tasksWithDepRequests = Array.from(this.#stageCaches.values()) + .filter((stageCache) => stageCache.hasDependencyRequests()); + await Promise.all(tasksWithDepRequests.map(async (stageCache) => { + await stageCache.refreshDependencyIndices(dependencyReader); })); // Reset pending dependency changes since indices are fresh now anyways this.#changedDependencyResourcePaths = []; @@ -423,23 +484,23 @@ export default class ProjectBuildCache { /** * Imports cached stages and sets them in the project * - * @param {Object} stageSignatures Map of stage names to their signatures + * @param {Object} stageSignatures Map of stage ids to their signatures * @returns {string[]} Array of resource paths written by all imported stages */ #importStages(stageSignatures) { - const stageNames = Object.keys(stageSignatures); + const stageIds = Object.keys(stageSignatures); if (this.#project.getProjectResources().getStage()?.getId() === "initial") { // Only initialize stages once - this.#project.getProjectResources().initStages(stageNames); + this.#project.getProjectResources().initStages(stageIds); } - const importedStages = stageNames.map((stageName) => { - const stageSignature = stageSignatures[stageName]; - const stageCache = this.#findStageCache(stageName, [stageSignature]); + const importedStages = stageIds.map((stageId) => { + const stageSignature = stageSignatures[stageId]; + const stageCache = this.#findStageCache(stageId, [stageSignature]); if (!stageCache) { throw new Error(`Inconsistent result cache: Could not find cached stage ` + - `${stageName} with signature ${stageSignature} for project ${this.#project.getName()}`); + `${stageId} with signature ${stageSignature} for project ${this.#project.getName()}`); } - return [stageName, stageCache]; + return [stageId, stageCache]; }); this.#project.getProjectResources().useResultStage(); @@ -450,15 +511,16 @@ export default class ProjectBuildCache { const isInitialImport = this.#currentStageSignatures.size === 0; const writtenResourcePaths = new Set(); - for (const [stageName, stageCache] of importedStages) { + for (const [stageId, stageCache] of importedStages) { // Check whether the stage differs form the one currently in use - if (this.#currentStageSignatures.get(stageName)?.join("-") !== stageCache.signature) { + const currentStageTuple = this.#currentStageSignatures.get(stageId); + if ((currentStageTuple && createStageSignature(currentStageTuple)) !== stageCache.signature) { // Set stage - this.#project.getProjectResources().setStage(stageName, stageCache.stage, + this.#project.getProjectResources().setStage(stageId, stageCache.stage, stageCache.projectTagOperations, stageCache.buildTagOperations); // Store signature for later use in result stage signature calculation - this.#currentStageSignatures.set(stageName, stageCache.signature.split("-")); + this.#currentStageSignatures.set(stageId, splitStageSignature(stageCache.signature)); if (!isInitialImport) { // Cached stage differs from the previous one @@ -482,22 +544,44 @@ export default class ProjectBuildCache { } /** - * Calculates all possible result stage signatures based on current state + * Calculates all possible result stage signatures based on current state. + * + * A result signature is the tuple [source, combinedDependency, aggregatedInput, aggregatedRoot]. The + * dependency component is a cartesian product over the per-stage dependency-signature lists, so there + * is one candidate per combination. * * @returns {string[]} Array of possible result stage signatures */ #getPossibleResultStageSignatures() { const projectSourceSignature = this.#sourceIndex.getSignature(); - const taskDependencySignatures = []; - for (const taskCache of this.#taskCache.values()) { - taskDependencySignatures.push(taskCache.getDependencyIndexSignatures()); - } + // Derive the per-stage dependency-signature lists from the single stage order, so this lookup and + // #getResultStageSignature (the store side) always walk the same stages in the same order. + // createDependencySignature is positional and length-sensitive, so a divergence here would store + // a result signature no later lookup could reproduce (F2). + const taskDependencySignatures = this.#stageOrder.map((stageId) => { + const stageCache = this.#stageCaches.get(stageId); + if (!stageCache) { + throw new Error( + `Inconsistent stage state in project ${this.#project.getName()}: stage ${stageId} is ` + + `in the stage order but has no task cache`); + } + return stageCache.getDependencyIndexSignatures(); + }); const dependencySignaturesCombinations = cartesianProduct(taskDependencySignatures); + // The aggregated input and root signatures are single current values (not sets of cached + // alternatives), so they apply to every dependency combination as constants. Each is its own slot + // of the result-signature tuple, so a changed input or root file invalidates the project-level + // result cache and the per-project build is not skipped wholesale (the result-cache check runs + // before the per-stage cache checks). + const aggregatedInputSignature = this.#getAggregatedInputSignature(); + const aggregatedRootSignature = this.#getAggregatedRootSignature(); + return dependencySignaturesCombinations.map((dependencySignatures) => { const combinedDepSignature = createDependencySignature(dependencySignatures); - return createStageSignature(projectSourceSignature, combinedDepSignature); + return createStageSignature( + [projectSourceSignature, combinedDepSignature, aggregatedInputSignature, aggregatedRootSignature]); }); } @@ -508,210 +592,295 @@ export default class ProjectBuildCache { */ #getResultStageSignature() { const projectSourceSignature = this.#sourceIndex.getSignature(); - const dependencySignatures = []; - for (const [, depSignature] of this.#currentStageSignatures.values()) { - dependencySignatures.push(depSignature); - } + // Walk #stageOrder (not #currentStageSignatures insertion order) so this stored signature's + // dependency component matches the candidate list #getPossibleResultStageSignatures computes on + // the next build. A stage missing from #currentStageSignatures is a clear invariant violation + // rather than a silently shortened, never-matching dependency list (F2). + const dependencySignatures = this.#stageOrder.map((stageId) => { + const stageTuple = this.#currentStageSignatures.get(stageId); + if (!stageTuple) { + throw new Error( + `Inconsistent stage state in project ${this.#project.getName()}: stage ${stageId} has ` + + `no current stage signature`); + } + return stageTuple[STAGE_SIG_DEPENDENCY_INDEX]; + }); const combinedDepSignature = createDependencySignature(dependencySignatures); - return createStageSignature(projectSourceSignature, combinedDepSignature); + const aggregatedInputSignature = this.#getAggregatedInputSignature(); + const aggregatedRootSignature = this.#getAggregatedRootSignature(); + return createStageSignature( + [projectSourceSignature, combinedDepSignature, aggregatedInputSignature, aggregatedRootSignature]); + } + + /** + * Aggregates the current non-resource input signatures (e.g. recorded env-var usage) across all task + * caches into a single signature, re-evaluated against the current environment and graph. + * + * It is one slot of the result stage signature (root resources are a sibling slot via + * {@link #getAggregatedRootSignature}), so a changed input invalidates the project-level result cache + * and the per-project build is not skipped wholesale (the result-cache check runs before the + * per-stage cache checks). Order-independent: the per-stage signatures are sorted before hashing. + * + * @returns {string} Aggregated input signature + */ + #getAggregatedInputSignature() { + const inputSignatures = []; + for (const stageCache of this.#stageCaches.values()) { + inputSignatures.push(stageCache.getInputSignature(this.#resolveInputValue)); + } + return crypto.createHash("sha256").update(inputSignatures.sort().join("\0")).digest("hex"); + } + + /** + * Returns a factory for project root readers, used to re-materialize recorded root resource + * requests. The useGitignore flag must match the one the request was recorded with, since it + * changes which resources a glob matches. + * + * @returns {function(boolean): @ui5/fs/AbstractReader} Root reader factory + */ + #getRootReaderFactory() { + return (useGitignore) => this.#project.getRootReader({useGitignore}); + } + + /** + * Whether any task cache recorded root resource requests. + * + * @returns {boolean} + */ + #anyTaskHasRootRequests() { + for (const stageCache of this.#stageCaches.values()) { + if (stageCache.hasRootRequests()) { + return true; + } + } + return false; + } + + /** + * Refreshes the root resource indices of every task cache that recorded root requests, resolving + * them against the current project root. Bounded by what the tasks requested. + * + * @returns {Promise} + */ + async #refreshRootIndices() { + const getRootReader = this.#getRootReaderFactory(); + await Promise.all(Array.from(this.#stageCaches.values()) + .filter((stageCache) => stageCache.hasRootRequests()) + .map((stageCache) => stageCache.refreshRootIndices(getRootReader))); + } + + /** + * Aggregates the current root signatures across all task caches into one signature, used to detect + * whether any recorded root file changed since the cache was last validated. + * + * @returns {string} Aggregated root signature + */ + #getAggregatedRootSignature() { + const rootSignatures = []; + for (const stageCache of this.#stageCaches.values()) { + rootSignatures.push(stageCache.getRootSignature()); + } + return crypto.createHash("sha256").update(rootSignatures.sort().join("\0")).digest("hex"); } // ===== TASK MANAGEMENT ===== /** - * Prepares a task for execution by switching to its stage and checking for cached results + * Prepares a stage for execution by switching to it and checking for cached results * * This method: - * 1. Switches the project to the task's stage - * 2. Updates task indices if the task has been invalidated - * 3. Attempts to find a cached stage for the task - * 4. Returns whether the task needs to be executed + * 1. Switches the project to the stage + * 2. Updates the stage's indices if it has been invalidated + * 3. Attempts to find a cached stage + * 4. Returns whether the stage needs to be (re-)executed + * + * A legacy task has a single stage (stepName omitted); a step-based task calls this once + * per step, each step being its own stage. * * @public * @param {string} taskName Name of the task to prepare + * @param {string} [stepName] Name of the step, for a step-based task's per-step stage * @returns {Promise} - * True if task can use cache, false if task needs execution, + * True if the stage can use cache, false if it needs execution, * or an object with cache information for differential updates */ - async prepareTaskExecutionAndValidateCache(taskName) { - const stageName = this.#getStageNameForTask(taskName); - const taskCache = this.#taskCache.get(taskName); - // Store current project reader (= state of the previous stage) for later use (e.g. in recordTaskResult) + async prepareStageExecutionAndValidateCache(taskName, stepName) { + const stageId = this.#stageIdFor(taskName, stepName); + const stageCache = this.#stageCaches.get(stageId); + // Store current project reader (= state of the previous stage) for later use (e.g. in recordStageResult) this.#currentProjectReader = this.#project.getReader(); // Switch project to new stage - this.#project.getProjectResources().useStage(stageName); - log.verbose(`Preparing task execution for task ${taskName} in project ${this.#project.getName()}...`); - if (!taskCache) { - log.verbose(`No task cache found`); + this.#project.getProjectResources().useStage(stageId); + log.verbose(`Preparing execution for stage ${stageId} in project ${this.#project.getName()}...`); + if (!stageCache) { + log.verbose(`No stage cache found`); + // No cached stage to restore from: this build has no "previous" per-key data for the stage. + this.#stepInvocationData.set(stageId, undefined); return false; } if (this.#writtenResultResourcePaths.length) { - // Update task indices based on source changes and changes from by previous tasks + // Update stage indices based on source changes and changes from previous stages. + // + // The list passed here is the paths accumulated so far this build (source changes plus every + // earlier stage's writes), not the whole build's final written set: it grows as stages run, + // so stage N receives exactly the changes from stages 0..N-1. A finer per-stage delta (only + // the increment since the previous stage) is not safely derivable, because this stage's cached + // index baseline is the previous build's final state, so it must see every change since then, + // not only the last stage's. updateIndices early-exits when the stage recorded no requests and + // otherwise matches only the paths its recorded requests cover, so the accumulated list is not + // re-scanned in full for stages that read little. const updateProjectIndicesStart = performance.now(); - await taskCache.updateProjectIndices(this.#currentProjectReader, this.#writtenResultResourcePaths); + await stageCache.updateProjectIndices(this.#currentProjectReader, this.#writtenResultResourcePaths); if (log.isLevelEnabled("perf")) { log.perf( - `Updated project indices for task ${taskName} in project ${this.#project.getName()} ` + + `Updated project indices for stage ${stageId} in project ${this.#project.getName()} ` + `in ${(performance.now() - updateProjectIndicesStart).toFixed(2)} ms`); } } // TODO: Implement: // After index update, try to find cached stages for the new signatures - // let stageSignatures = taskCache.getAffiliatedSignaturePairs(); - - const projectSignatures = taskCache.getProjectIndexSignatures(); - const dependencySignatures = taskCache.getDependencyIndexSignatures(); - const stageSignatures = combineTwoArraysFast( - projectSignatures, - dependencySignatures, - ).map((signaturePair) => { - return createStageSignature(...signaturePair); - }); - - const stageCache = this.#findStageCache(stageName, stageSignatures); - const oldStageSig = this.#currentStageSignatures.get(stageName)?.join("-"); - if (stageCache) { - this.#project.getProjectResources().setStage(stageName, stageCache.stage, - stageCache.projectTagOperations, stageCache.buildTagOperations); - - // Check whether the stage actually changed - if (stageCache.signature !== oldStageSig) { + // let stageSignatures = stageCache.getAffiliatedSignaturePairs(); + + // A stage signature is the [project, dependency, input, root] tuple. The exact-match candidates + // are the cartesian product of the recorded project and dependency index signatures, each paired + // with the current input and root signatures (BuildStageCache.getStageSignatures). The input and + // root signatures are re-evaluated against the current environment, graph, and project root, so a + // changed input or root file misses the cached stage. Root indices were refreshed in validateCache + // before this build's tasks run. + const inputSignature = stageCache.getInputSignature(this.#resolveInputValue); + const rootSignature = stageCache.getRootSignature(); + const stageSignatures = stageCache.getStageSignatures(this.#resolveInputValue); + + const cachedStage = this.#findStageCache(stageId, stageSignatures); + const oldStageTuple = this.#currentStageSignatures.get(stageId); + const oldStageSig = oldStageTuple && createStageSignature(oldStageTuple); + if (cachedStage) { + this.#project.getProjectResources().setStage(stageId, cachedStage.stage, + cachedStage.projectTagOperations, cachedStage.buildTagOperations); + + // Stash the matched stage's per-key map as this build's "previous" data, so + // getStepInvocationData returns the map recorded under exactly this signature rather than the + // most-recently-written one (see the stepInvocationData field note). + this.#stepInvocationData.set(stageId, cachedStage.stepInvocationData); + + // Skip propagation when the cached stage matches the previous one + if (cachedStage.signature !== oldStageSig) { // Store new stage signature for later use in result stage signature calculation - this.#currentStageSignatures.set(stageName, stageCache.signature.split("-")); + this.#currentStageSignatures.set(stageId, splitStageSignature(cachedStage.signature)); // Cached stage likely differs from the previous one (if any) // Add all resources written by the cached stage to the set of written/potentially changed resources - for (const resourcePath of stageCache.writtenResourcePaths) { - if (!this.#writtenResultResourcePaths.includes(resourcePath)) { - this.#writtenResultResourcePaths.push(resourcePath); - } + for (const resourcePath of cachedStage.writtenResourcePaths) { + this.#addWrittenResultResourcePath(resourcePath); } } - return true; // No need to execute the task + return true; // No need to execute the stage } else { - log.verbose(`No cached stage found for task ${taskName} in project ${this.#project.getName()}. ` + + log.verbose(`No cached stage found for stage ${stageId} in project ${this.#project.getName()}. ` + `Attempting to find delta cached stage...`); - // TODO: Optimize this crazy thing - const projectDeltas = taskCache.getProjectIndexDeltas(); - const depDeltas = taskCache.getDependencyIndexDeltas(); - - // Combine deltas of project stages with cached dependency signatures - const projDeltaSignatures = combineTwoArraysFast( - Array.from(projectDeltas.keys()), - dependencySignatures, - ).map((signaturePair) => { - return createStageSignature(...signaturePair); - }); - // Combine deltas of dependency stages with cached project signatures - const depDeltaSignatures = combineTwoArraysFast( - projectSignatures, - Array.from(depDeltas.keys()), - ).map((signaturePair) => { - return createStageSignature(...signaturePair); - }); - // Combine deltas of both project and dependency stages - const deltaDeltaSignatures = combineTwoArraysFast( - Array.from(projectDeltas.keys()), - Array.from(depDeltas.keys()), - ).map((signaturePair) => { - return createStageSignature(...signaturePair); - }); - const deltaSignatures = [...projDeltaSignatures, ...depDeltaSignatures, ...deltaDeltaSignatures]; - const deltaStageCache = this.#findStageCache(stageName, deltaSignatures); + const projectDeltas = stageCache.getProjectIndexDeltas(); + const depDeltas = stageCache.getDependencyIndexDeltas(); + const projectSignatures = stageCache.getProjectIndexSignatures(); + const dependencySignatures = stageCache.getDependencyIndexSignatures(); + + // Build the delta candidates and carry each one's provenance alongside it: the resolved new + // project and dependency components and the changed-path lists. The winner is looked up by its + // full signature, so no component is ever reverse-mapped out of the tuple (reverse mapping is + // how a dependency-only delta used to combine the unchanged project component a second time). + // Three candidate families, keeping the order the single lookup list had before: + // - project deltas x current dependency signatures (project changed, dependency unchanged) + // - current project signatures x dependency deltas (dependency changed, project unchanged) + // - project deltas x dependency deltas (both changed) + const deltaSignatures = []; + const provenanceBySignature = new Map(); + const addDeltaCandidate = (projectSig, projectDeltaInfo, dependencySig, dependencyDeltaInfo) => { + const signature = createStageSignature( + [projectSig, dependencySig, inputSignature, rootSignature]); + deltaSignatures.push(signature); + provenanceBySignature.set(signature, { + newProjectSig: projectDeltaInfo?.newSignature ?? projectSig, + newDependencySig: dependencyDeltaInfo?.newSignature ?? dependencySig, + changedProjectResourcePaths: projectDeltaInfo?.changedPaths ?? [], + changedDependencyResourcePaths: dependencyDeltaInfo?.changedPaths ?? [], + }); + }; + for (const [projectSig, projectDeltaInfo] of projectDeltas) { + for (const dependencySig of dependencySignatures) { + addDeltaCandidate(projectSig, projectDeltaInfo, dependencySig, undefined); + } + } + for (const projectSig of projectSignatures) { + for (const [dependencySig, dependencyDeltaInfo] of depDeltas) { + addDeltaCandidate(projectSig, undefined, dependencySig, dependencyDeltaInfo); + } + } + for (const [projectSig, projectDeltaInfo] of projectDeltas) { + for (const [dependencySig, dependencyDeltaInfo] of depDeltas) { + addDeltaCandidate(projectSig, projectDeltaInfo, dependencySig, dependencyDeltaInfo); + } + } + + const deltaStageCache = this.#findStageCache(stageId, deltaSignatures); if (deltaStageCache) { - // Store dependency signature for later use in result stage signature calculation - const [foundProjectSig, foundDepSig] = deltaStageCache.signature.split("-"); + const provenance = provenanceBySignature.get(deltaStageCache.signature); + + // Stash the matched (previous-signature) stage's per-key map so #selectStepsToRun and + // #computeStaleOutputs run against the data recorded under the restored signature. + this.#stepInvocationData.set(stageId, deltaStageCache.stepInvocationData); - // Check whether the stage actually changed + // Skip propagation when the cached stage matches the previous one if (oldStageSig !== deltaStageCache.signature) { // Cached stage likely differs from the previous one (if any) // Add all resources written by the cached stage to the set of written/potentially changed resources for (const resourcePath of deltaStageCache.writtenResourcePaths) { - if (!this.#writtenResultResourcePaths.includes(resourcePath)) { - this.#writtenResultResourcePaths.push(resourcePath); - } + this.#addWrittenResultResourcePath(resourcePath); } } - // Create new signature and determine changed resource paths - const projectDeltaInfo = projectDeltas.get(foundProjectSig); - const dependencyDeltaInfo = depDeltas.get(foundDepSig); - - const newProjSig = projectDeltaInfo?.newSignature ?? foundProjectSig; - const newDepSig = dependencyDeltaInfo?.newSignature ?? foundDepSig; - const newSignature = createStageSignature(newProjSig, newDepSig); - this.#currentStageSignatures.set(stageName, [newProjSig, newDepSig]); + // Pair the delta's resolved project and dependency components with the current input and + // root signatures. For a dependency-only delta the project component is the one the stage + // was recorded under, carried through unchanged. + const newStageTuple = + [provenance.newProjectSig, provenance.newDependencySig, inputSignature, rootSignature]; + const newSignature = createStageSignature(newStageTuple); + this.#currentStageSignatures.set(stageId, newStageTuple); log.verbose( - `Using delta cached stage for task ${taskName} in project ${this.#project.getName()} ` + + `Using delta cached stage for stage ${stageId} in project ${this.#project.getName()} ` + `with original signature ${deltaStageCache.signature} (now ${newSignature}) ` + - `and ${projectDeltaInfo?.changedPaths.length ?? "unknown"} changed project resource paths and ` + - `${dependencyDeltaInfo?.changedPaths.length ?? "unknown"} changed dependency resource paths.`); + `and ${provenance.changedProjectResourcePaths.length} changed project resource paths and ` + + `${provenance.changedDependencyResourcePaths.length} changed dependency resource paths.`); return { previousStageCache: deltaStageCache, newSignature: newSignature, - changedProjectResourcePaths: projectDeltaInfo?.changedPaths ?? [], - changedDependencyResourcePaths: dependencyDeltaInfo?.changedPaths ?? [] + changedProjectResourcePaths: provenance.changedProjectResourcePaths, + changedDependencyResourcePaths: provenance.changedDependencyResourcePaths }; } } + // No cached stage matched (neither an exact signature nor a delta): the stage runs in full against + // a fresh writer, so it has no restorable "previous" per-key map. + this.#stepInvocationData.set(stageId, undefined); return false; // Task needs to be executed } /** - * Pre-fetches stage cache metadata from persistent storage for the given task. - * Results are stored internally and consumed by #findStageCache when called later. + * Reopens a step's stage with a fresh live writer so it can be re-run after a full cache hit that the + * {@link StepRunner} determined must re-execute (a consumed needs return changed). The + * full hit had installed the cached read-only stage via {@link #findStageCache} + + * setStage; this swaps in a writable stage. The re-run records through the normal + * {@link #recordStageResult} full path, which recomputes the stage signature and overwrites the + * eagerly-stored full-hit signature. * * @public - * @param {string} taskName Task name to prefetch cache for + * @param {string} taskName Name of the task + * @param {string} [stepName] Name of the step, for a step-based task's per-step stage */ - prefetchStageCache(taskName) { - const taskCache = this.#taskCache.get(taskName); - if (!taskCache) { - return; - } - const stageName = this.#getStageNameForTask(taskName); - - // Compute possible signatures from current index state - const projectSignatures = taskCache.getProjectIndexSignatures(); - const dependencySignatures = taskCache.getDependencyIndexSignatures(); - const stageSignatures = combineTwoArraysFast( - projectSignatures, - dependencySignatures, - ).map((signaturePair) => { - return createStageSignature(...signaturePair); - }); - - if (!stageSignatures.length) { - return; - } - - // Filter out signatures already in memory - const uncachedSignatures = stageSignatures.filter((sig) => - !this.#stageCache.getCacheForSignature(stageName, sig)); - - if (!uncachedSignatures.length) { - return; - } - - // Batch-check which signatures actually exist in the DB - const existingSignatures = this.#cacheManager.findExistingStageSignatures( - this.#project.getId(), this.#buildSignature, stageName, uncachedSignatures); - - if (!existingSignatures.length) { - return; - } - - // Only read signatures that exist - const prefetchMap = new Map(); - for (const sig of existingSignatures) { - prefetchMap.set(sig, this.#cacheManager.readStageCache( - this.#project.getId(), this.#buildSignature, stageName, sig)); - } - this.#prefetchedStageReads = this.#prefetchedStageReads ?? new Map(); - this.#prefetchedStageReads.set(stageName, prefetchMap); + reopenStageForRerun(taskName, stepName) { + const stageId = this.#stageIdFor(taskName, stepName); + this.#project.getProjectResources().reopenStage(stageId); } /** @@ -720,70 +889,52 @@ export default class ProjectBuildCache { * Checks both in-memory stage cache and persistent cache storage for a matching * stage signature. Returns the first matching cached stage found. * - * @param {string} stageName Name of the stage to find + * @param {string} stageId Name of the stage to find * @param {string[]} stageSignatures Possible signatures for the stage * @returns {@ui5/project/build/cache/ProjectBuildCache~StageCacheEntry|undefined} * Cached stage entry or undefined if not found */ - #findStageCache(stageName, stageSignatures) { + #findStageCache(stageId, stageSignatures) { if (!stageSignatures.length) { return; } // Check cache exists and ensure it's still valid before using it - log.verbose(`Looking for cached stage for task ${stageName} in project ${this.#project.getName()} ` + + log.verbose(`Looking for cached stage for stage in project ${this.#project.getName()} ` + `with ${stageSignatures.length} possible signatures:\n - ${stageSignatures.join("\n - ")}`); for (const stageSignature of stageSignatures) { - const stageCache = this.#stageCache.getCacheForSignature(stageName, stageSignature); + const stageCache = this.#stageCache.getCacheForSignature(stageId, stageSignature); if (stageCache) { return stageCache; } } - // Check prefetched data - const prefetchMap = this.#prefetchedStageReads?.get(stageName); - if (prefetchMap) { - this.#prefetchedStageReads.delete(stageName); - for (const stageSignature of stageSignatures) { - const stageMetadata = prefetchMap.get(stageSignature); - if (stageMetadata) { - log.verbose(`Found prefetched cached stage for task ${stageName} ` + - `with signature ${stageSignature}`); - return this.#processStageCacheMetadata(stageName, stageSignature, stageMetadata); - } - } - // Filter out already-checked signatures from disk lookup - stageSignatures = stageSignatures.filter((sig) => !prefetchMap.has(sig)); - if (!stageSignatures.length) { - return; - } - } - // Batch-check which signatures exist, then read only the first match const existingSignatures = this.#cacheManager.findExistingStageSignatures( - this.#project.getId(), this.#buildSignature, stageName, stageSignatures); + this.#project.getId(), this.#buildSignature, stageId, stageSignatures); if (!existingSignatures.length) { return; } const stageSignature = existingSignatures[0]; const stageMetadata = this.#cacheManager.readStageCache( - this.#project.getId(), this.#buildSignature, stageName, stageSignature); + this.#project.getId(), this.#buildSignature, stageId, stageSignature); if (!stageMetadata) { return; } - log.verbose(`Found cached stage for task ${stageName} with signature ${stageSignature}`); - return this.#processStageCacheMetadata(stageName, stageSignature, stageMetadata); + log.verbose(`Found cached stage for stage with signature ${stageSignature}`); + return this.#processStageCacheMetadata(stageId, stageSignature, stageMetadata); } /** * Processes stage cache metadata into a stage cache entry * - * @param {string} stageName Name of the stage + * @param {string} stageId Name of the stage * @param {string} stageSignature Signature of the stage * @param {object} stageMetadata Raw metadata from cache * @returns {object} Stage cache entry */ - #processStageCacheMetadata(stageName, stageSignature, stageMetadata) { - const {resourceMapping, resourceMetadata, projectTagOperations, buildTagOperations} = stageMetadata; + #processStageCacheMetadata(stageId, stageSignature, stageMetadata) { + const {resourceMapping, resourceMetadata, projectTagOperations, buildTagOperations, + stepInvocationData} = stageMetadata; let writtenResourcePaths; let stageReader; if (resourceMapping) { @@ -792,7 +943,7 @@ export default class ProjectBuildCache { const readers = resourceMetadata.map((metadata) => { writtenResourcePaths.push(...Object.keys(metadata)); return this.#createReaderForStageCache( - stageName, stageSignature, metadata); + stageId, stageSignature, metadata); }); const writerMapping = Object.create(null); @@ -805,12 +956,12 @@ export default class ProjectBuildCache { } stageReader = createWriterCollection({ - name: `Restored cached stage ${stageName} for project ${this.#project.getName()}`, + name: `Restored cached stage ${stageId} for project ${this.#project.getName()}`, writerMapping, }); } else { writtenResourcePaths = Object.keys(resourceMetadata); - stageReader = this.#createReaderForStageCache(stageName, stageSignature, resourceMetadata); + stageReader = this.#createReaderForStageCache(stageId, stageSignature, resourceMetadata); } this.#collectKnownIntegrities(resourceMetadata); @@ -821,6 +972,8 @@ export default class ProjectBuildCache { writtenResourcePaths, projectTagOperations: tagOpsToMap(projectTagOperations), buildTagOperations: tagOpsToMap(buildTagOperations), + // Persisted as [[keyId, entry], ...] pairs (JSON has no Map); undefined for a legacy stage. + stepInvocationData: stepInvocationData ? new Map(stepInvocationData) : undefined, }; } @@ -835,29 +988,236 @@ export default class ProjectBuildCache { * * @public * @param {string} taskName Name of the executed task - * @param {@ui5/project/build/cache/BuildTaskCache~ResourceRequests} projectResourceRequests + * @param {@ui5/project/build/cache/BuildStageCache~ResourceRequests} projectResourceRequests * Resource requests for project resources - * @param {@ui5/project/build/cache/BuildTaskCache~ResourceRequests|undefined} dependencyResourceRequests + * @param {@ui5/project/build/cache/BuildStageCache~ResourceRequests|undefined} dependencyResourceRequests * Resource requests for dependency resources * @param {object} cacheInfo Cache information for differential updates - * @param {boolean} supportsDifferentialBuilds Whether the task supports differential updates + * @param {Array<{type: string, name: string, value: string|undefined}>} [inputRecording] + * Non-resource inputs (environment variables, TaskUtil interface reads) recorded during task + * execution + * @param {{gitignore: @ui5/project/build/cache/BuildStageCache~ResourceRequests, + * noGitignore: @ui5/project/build/cache/BuildStageCache~ResourceRequests}} [rootResourceRequests] + * Resource requests read through the project's root reader, keyed by useGitignore * @returns {Promise} The resource paths written by the task, * or undefined if caching is disabled */ - async recordTaskResult( - taskName, projectResourceRequests, dependencyResourceRequests, cacheInfo, supportsDifferentialBuilds + /** + * Returns the step-runner invocation data for a step's stage on its previous run, or + * undefined if the stage has none (first build, a stage that ran no keys, or a full miss + * with no cached stage to restore from). The value is the per-key map of the stage that + * {@link #prepareStageExecutionAndValidateCache} matched for this build (stashed there under the exact + * signature it was recorded under), or the map a running stage recorded via + * {@link #setStepInvocationData}. Keyed by stage id: each step-based task's step is its own stage, so + * the data is that step's per-key map alone. + * + * @param {string} stageId Stage id + * @returns {Map|undefined} That stage's per-key data + * {reads, dependencyReads, writes, inputs, needsInputs, tagOperations, returns} + */ + getStepInvocationData(stageId) { + return this.#stepInvocationData.get(stageId); + } + + /** + * Stores the step-runner invocation data a step's stage recorded on this build. {@link #recordStageResult} + * reads it back to pair it with the stage under its new signature (persisted inside the stage's own + * {@link #prepareStageCache} payload), so the per-key map and the stage output stay keyed together. + * + * @param {string} stageId Stage id + * @param {Map} invocationData That stage's per-key data + * {reads, dependencyReads, writes, inputs, needsInputs, tagOperations, returns} + */ + setStepInvocationData(stageId, invocationData) { + this.#stepInvocationData.set(stageId, invocationData); + } + + /** + * Returns the CAS-backed store the {@link StepRunner} driver uses to persist and rebuild callback + * return values. store buffers the resources' content for the CAS (deduped) and returns + * path-aligned descriptors; flush writes the content buffered since the last flush in a + * single transaction, called once per step by the driver; restore rebuilds a resource + * from such a descriptor on a delta build without re-running the step. + * + * @returns {{store: Function, flush: Function, restore: Function}} The return value store + */ + getStepReturnValueStore() { + return { + store: (resources) => this.#storeStepReturns(resources), + flush: () => this.#flushStepReturns(), + restore: (descriptor) => this.#restoreStepReturn(descriptor), + }; + } + + /** + * Returns the resolver the {@link StepRunner} driver uses to re-derive the current value of a step's + * recorded non-resource input on a delta build (the same resolver the task-level input lookup uses, + * reaching process.env and the current project graph). A step whose input no longer + * resolves to its stored value is re-run. undefined when the cache was built without one. + * + * @returns {function(string, string): (string|undefined)|undefined} The input value resolver + */ + getResolveInputValue() { + return this.#resolveInputValue; + } + + /** + * Persists the content of resources a step returned and describes them for later + * reconstruction. Content goes through the same compression and dedup pipeline as stage resources; + * the compressed rows are buffered in {@link #pendingStepReturnCasRows} and written by + * {@link #flushStepReturns}, which the driver calls once per step so a step returning many units + * costs one transaction rather than one per unit. The descriptors are recorded in the step's + * invocation data. The CAS write uses INSERT OR IGNORE, so content shared with a stage output is + * stored once and a build that later fails before the flush leaves nothing behind. + * + * @param {@ui5/fs/Resource[]} resources Resources a step returned, in return order + * @returns {Promise>} Descriptors {path, integrity, size, lastModified, inode} + * aligned to resources + */ + async #storeStepReturns(resources) { + // Reuse the stage-resource pipeline for compression and CAS dedup; a returned resource whose path + // collides with a written output is stored once by integrity and rebuilt independently of that + // output. + const {resourceMetadata, casRows} = await this.#prepareStageResources(resources, "stepReturn"); + for (const row of casRows) { + this.#pendingStepReturnCasRows.push(row); + } + this.#collectKnownIntegrities(resourceMetadata); + // Build descriptors from the metadata #prepareStageResources already computed (integrity, size, + // lastModified, inode per path) rather than re-reading each resource. resourceMetadata is keyed by + // original path; the descriptor path is the current path, which differ only for a renamed resource. + return resources.map((res) => { + const {integrity, size, lastModified, inode} = resourceMetadata[res.getOriginalPath()]; + return {path: res.getPath(), integrity, size, lastModified, inode}; + }); + } + + /** + * Writes the step-return CAS rows buffered since the last flush in a single transaction. The driver + * calls this once per step (after the step's units have stored their returns), so one step costs one + * transaction regardless of how many units returned resources. A no-op when nothing was buffered. + */ + #flushStepReturns() { + if (!this.#pendingStepReturnCasRows.length) { + return; + } + const rows = this.#pendingStepReturnCasRows; + this.#pendingStepReturnCasRows = []; + this.#cacheManager.transaction(() => { + for (const {integrity, compressedBuffer} of rows) { + this.#cacheManager.putCompressedContent(integrity, compressedBuffer); + } + }); + } + + /** + * Rebuilds a resource a step returned on a previous build, reading its content from the + * CAS by integrity. Mirrors the CAS-backed resources of {@link #createReaderForStageCache}, but + * treats lastModified and inode as optional: returned resources are + * usually fresh build outputs that never had filesystem metadata. + * + * @param {object} descriptor Return descriptor recorded by {@link #storeStepReturns} + * @param {string} descriptor.path Virtual path of the returned resource + * @param {string} descriptor.integrity Content integrity, the CAS lookup key + * @param {number} [descriptor.size] Byte size + * @param {number} [descriptor.lastModified] Last-modified timestamp, if the resource had one + * @param {number} [descriptor.inode] Inode of the original resource, if known + * @returns {@ui5/fs/Resource} The reconstructed resource + */ + #restoreStepReturn({path, integrity, size, lastModified, inode}) { + if (!integrity) { + throw new Error( + `Incomplete step return descriptor for resource ${path} ` + + `in project ${this.#project.getName()}: missing integrity`); + } + return createResource({ + path, + sourceMetadata: { + adapter: "CAS_SQLITE", + contentModified: false, + }, + createStream: () => Readable.from(this.#cacheManager.readContent(integrity)), + createBuffer: () => this.#cacheManager.readContent(integrity), + byteSize: size, + lastModified, + integrity, + inode, + project: this.#project, + }); + } + + /** + * Re-records a map step's stage complete request set on a delta build and returns the resulting + * [projectSignature, dependencySignature] pair, so {@link #recordStageResult} can re-key the stage on + * it (see open-gaps §7). The request set fed in already unions the delta's monitored requests with + * every key's persisted reads (assembled by the driver and the TaskRunner), so recording it keys + * the stage exactly as a full build would. + * + * @param {string} stageId Executed stage id + * @param {@ui5/project/build/cache/BuildStageCache} stageCache The stage's cache + * @param {object} projectResourceRequests Complete project requests (paths + patterns) + * @param {object} dependencyResourceRequests Complete dependency requests, if the stage reads dependencies + * @param {Array} inputRecording Recorded non-resource inputs + * @param {object} rootResourceRequests Recorded root requests + * @returns {Promise} The [project, dependency, input, root] stage-signature tuple + */ + async #foldStepReads( + stageId, stageCache, projectResourceRequests, dependencyResourceRequests, inputRecording, rootResourceRequests ) { + return stageCache.recordRequests({ + projectRequestRecording: projectResourceRequests, + dependencyRequestRecording: dependencyResourceRequests, + projectReader: this.#currentProjectReader, + dependencyReader: this.#currentDependencyReader, + inputRecording, + rootRequestRecording: rootResourceRequests, + getRootReader: this.#getRootReaderFactory(), + }); + } + + /** + * Records the result of a stage execution and updates the cache. + * + * @public + * @param {object} options + * @param {string} options.taskName Name of the executed task + * @param {@ui5/project/build/cache/BuildStageCache~ResourceRequests} options.projectResourceRequests + * Resource requests for project resources + * @param {@ui5/project/build/cache/BuildStageCache~ResourceRequests|undefined} + * options.dependencyResourceRequests Resource requests for dependency resources + * @param {object} [options.cacheInfo] Delta cache verdict for differential updates, or undefined for a + * full execution. Treated as read-only: the effective changed-path list is passed separately via + * changedProjectResourcePaths rather than mutated onto this object. + * @param {Array<{type: string, name: string, value: string|undefined}>} [options.inputRecording] + * Non-resource inputs (environment variables, TaskUtil interface reads) recorded during execution + * @param {{gitignore: @ui5/project/build/cache/BuildStageCache~ResourceRequests, + * noGitignore: @ui5/project/build/cache/BuildStageCache~ResourceRequests}} [options.rootResourceRequests] + * Resource requests read through the project's root reader, keyed by useGitignore + * @param {boolean} [options.stepBased=false] Whether the stage ran the step runner + * @param {string} [options.stepName] Name of the step, for a step-based task's per-step stage + * @param {string[]} [options.changedProjectResourcePaths] On a delta merge, the project resource paths + * to drop from the carried-forward stage: the verdict's own changed paths plus any stale outputs the + * caller derived. Defaults to the verdict's changedProjectResourcePaths. + * @returns {Promise} The resource paths written by the stage, + * or undefined if caching is disabled + */ + async recordStageResult({ + taskName, projectResourceRequests, dependencyResourceRequests, cacheInfo, + inputRecording = [], rootResourceRequests, stepBased = false, stepName, + changedProjectResourcePaths, + }) { if (this.#cacheMode === Cache.Off) { return; } const recordStart = performance.now(); - if (!this.#taskCache.has(taskName)) { - // Initialize task cache - this.#taskCache.set(taskName, - new BuildTaskCache(this.#project.getName(), taskName, supportsDifferentialBuilds)); + const stageId = this.#stageIdFor(taskName, stepName); + if (!this.#stageCaches.has(stageId)) { + // Initialize stage cache + this.#stageCaches.set(stageId, + new BuildStageCache(this.#project.getName(), stageId, stepBased)); } - log.verbose(`Recording results of task ${taskName} in project ${this.#project.getName()}...`); - const taskCache = this.#taskCache.get(taskName); + log.verbose(`Recording results of stage ${stageId} in project ${this.#project.getName()}...`); + const stageCache = this.#stageCaches.get(stageId); // Identify resources written by task const stage = this.#project.getProjectResources().getStage(); @@ -905,8 +1265,11 @@ export default class ProjectBuildCache { } // Paths flagged changed but not re-emitted by the delta task: their source // is gone or excluded, so replaying the previous stage's copy would - // resurrect content that no longer belongs in the output. - const changedProjectResourcePaths = new Set(cacheInfo.changedProjectResourcePaths ?? []); + // resurrect content that no longer belongs in the output. The caller passes the effective + // list (the verdict's changed paths plus any stale outputs it derived); fall back to the + // verdict's own list when the caller passes none. + const changedPathSet = new Set( + changedProjectResourcePaths ?? cacheInfo.changedProjectResourcePaths ?? []); // Set form for the membership check below; the array is retained for the // ordered downstream uses (recordStageCache, verbose counts). const writtenResourcePathSet = new Set(writtenResourcePaths); @@ -919,7 +1282,7 @@ export default class ProjectBuildCache { if (writtenResourcePathSet.has(path)) { continue; // Delta re-emitted this path; skip } - if (changedProjectResourcePaths.has(path)) { + if (changedPathSet.has(path)) { // Flagged changed but not written back by the delta task. // Drop the stale copy from the merge. droppedCount++; @@ -930,54 +1293,71 @@ export default class ProjectBuildCache { } if (log.isLevelEnabled("perf")) { log.perf( - `recordTaskResult delta merge for task ${taskName} ` + + `recordStageResult delta merge for task ${taskName} ` + `in project ${this.#project.getName()} completed in ` + `${(performance.now() - mergeStart).toFixed(2)} ms ` + `(${previousWrittenResources.length} previous, ${mergedCount} merged, ` + `${droppedCount} dropped)`); } + + if (stepBased) { + // A map step's stage carries an internal key-delta: the delta merge above carried its + // not-re-run keys' output forward and dropped stale output. But cacheInfo.newSignature keys + // the stage on the delta's partial request node, which does not track a read first observed + // on this build (a marker probe, a source map pulled in by a re-run key). Re-key on the + // stage's complete read set instead, exactly as the full-build branch does, so the next + // build looks the stage up under a signature that tracks every current input (open-gaps §7). + const foldedStageTuple = await this.#foldStepReads( + stageId, stageCache, projectResourceRequests, dependencyResourceRequests, + inputRecording, rootResourceRequests); + this.#currentStageSignatures.set(stageId, foldedStageTuple); + stageSignature = createStageSignature(foldedStageTuple); + } } else { - // Calculate signature for executed task + // Calculate signature for executed stage const recordReqStart = performance.now(); - const currentSignaturePair = await taskCache.recordRequests( - projectResourceRequests, - dependencyResourceRequests, - this.#currentProjectReader, - this.#currentDependencyReader - ); + const stageSignatureTuple = await stageCache.recordRequests({ + projectRequestRecording: projectResourceRequests, + dependencyRequestRecording: dependencyResourceRequests, + projectReader: this.#currentProjectReader, + dependencyReader: this.#currentDependencyReader, + inputRecording, + rootRequestRecording: rootResourceRequests, + getRootReader: this.#getRootReaderFactory(), + }); if (log.isLevelEnabled("perf")) { log.perf( - `recordTaskResult recordRequests for task ${taskName} ` + + `recordStageResult recordRequests for stage ${stageId} ` + `in project ${this.#project.getName()} completed in ` + `${(performance.now() - recordReqStart).toFixed(2)} ms`); } - // If provided, set dependency signature for later use in result stage signature calculation - const stageName = this.#getStageNameForTask(taskName); - this.#currentStageSignatures.set(stageName, currentSignaturePair); - stageSignature = createStageSignature(...currentSignaturePair); + // recordRequests returns the [project, dependency, input, root] stage-signature tuple directly. + this.#currentStageSignatures.set(stageId, stageSignatureTuple); + stageSignature = createStageSignature(stageSignatureTuple); } - log.verbose(`Caching stage for task ${taskName} in project ${this.#project.getName()} ` + + log.verbose(`Caching stage ${stageId} in project ${this.#project.getName()} ` + `with signature ${stageSignature}`); - // Store resulting stage in stage cache + // Store resulting stage in stage cache. The step runner set the stage's per-key map via + // setStepInvocationData immediately before this call (undefined for a legacy task), so it travels + // with the stage under this signature and is persisted inside the stage's own metadata row. this.#stageCache.addSignature( - this.#getStageNameForTask(taskName), stageSignature, this.#project.getProjectResources().getStage(), - writtenResourcePaths, projectTagOperations, buildTagOperations); + stageId, stageSignature, this.#project.getProjectResources().getStage(), + writtenResourcePaths, projectTagOperations, buildTagOperations, + this.#stepInvocationData.get(stageId)); // Update task cache with new metadata - log.verbose(`Task ${taskName} produced ${writtenResourcePaths.length} resources`); + log.verbose(`Stage ${stageId} produced ${writtenResourcePaths.length} resources`); for (const resourcePath of writtenResourcePaths) { - if (!this.#writtenResultResourcePaths.includes(resourcePath)) { - this.#writtenResultResourcePaths.push(resourcePath); - } + this.#addWrittenResultResourcePath(resourcePath); } // Reset current project reader this.#currentProjectReader = null; if (log.isLevelEnabled("perf")) { log.perf( - `recordTaskResult for task ${taskName} in project ${this.#project.getName()} ` + + `recordStageResult for task ${taskName} in project ${this.#project.getName()} ` + `completed in ${(performance.now() - recordStart).toFixed(2)} ms ` + `(${writtenResourcePaths.length} written resources, delta=${!!cacheInfo})`); } @@ -985,15 +1365,27 @@ export default class ProjectBuildCache { } /** - * Returns the task cache for a specific task + * Returns the stage cache for a task's stage, or a step's stage for a step-based task. + * + * The parameter is a task name (plus optional step name), resolved to its stage id via the same + * mapping the rest of the cache uses. Passing an already-composed stage id (one in the + * task/ namespace) is a caller mistake and throws, rather than being silently accepted. + * A task name with no recorded stage returns undefined. * * @public * @param {string} taskName Name of the task - * @returns {@ui5/project/build/cache/BuildTaskCache|undefined} - * The task cache or undefined if not found + * @param {string} [stepName] Name of the step, for a step-based task's per-step stage + * @returns {@ui5/project/build/cache/BuildStageCache|undefined} + * The stage cache or undefined if not found + * @throws {Error} If a composed stage id is passed in place of a task name */ - getTaskCache(taskName) { - return this.#taskCache.get(taskName); + getStageCache(taskName, stepName) { + if (taskName.startsWith("task/")) { + throw new Error( + `getStageCache expects a task name, but received the stage id '${taskName}'. ` + + `Pass the task name (and optional step name) instead.`); + } + return this.#stageCaches.get(this.#stageIdFor(taskName, stepName)); } /** @@ -1039,21 +1431,47 @@ export default class ProjectBuildCache { } /** - * Initializes project stages for the given tasks + * Initializes project stages for the given tasks. * - * Creates stage names for each task and initializes them in the project. - * This must be called before task execution begins. + * A legacy task contributes one stage; a step-based task contributes one stage per step, in + * step order, so each step is cached and folded into the result-stage signature independently. The step + * names are discovered by the caller from the task factory before the build runs. * * @public - * @param {string[]} taskNames Array of task names to initialize stages for + * @param {Array<{taskName: string, stepNames: string[]|undefined}>} tasks Tasks to initialize stages for, in + * execution order. stepNames (in step order) is present for a step-based task. */ - setTasks(taskNames) { - const stageNames = taskNames.map((taskName) => this.#getStageNameForTask(taskName)); - this.#project.getProjectResources().initStages(stageNames); + setTasks(tasks) { + const stageIds = []; + for (const {taskName, stepNames} of tasks) { + if (stepNames && stepNames.length) { + for (const stepName of stepNames) { + stageIds.push(this.#stageIdFor(taskName, stepName)); + } + } else { + stageIds.push(this.#stageIdFor(taskName)); + } + } + this.#project.getProjectResources().initStages(stageIds); + // Remember the order so the dependency signature is composed over stages deterministically. + this.#stageOrder = stageIds; // TODO: Rename function? We simply use it to have a point in time right before the project is built } + /** + * Returns the stage id for a task's single stage (legacy) or a step's stage (step-based). Lets the + * TaskRunner address a step's stage for its per-stage cache lookups. + * + * @public + * @param {string} taskName Task name + * @param {string} [stepName] Step name, for a step-based task's per-step stage + * @returns {string} Stage id + */ + getStageId(taskName, stepName) { + return this.#stageIdFor(taskName, stepName); + } + /** * Re-reads all source files from disk and compares them against the source index * to detect whether any source files were modified, added, or deleted during the build. @@ -1280,13 +1698,26 @@ export default class ProjectBuildCache { // Makes the next build re-run initSourceIndex (before validateCache), which re-globs the // source tree from scratch. See the initSourceIndex guard. this.#combinedIndexState = INDEX_STATES.RESTORING_PROJECT_INDICES; - this.#taskCache.clear(); + this.#stageCaches.clear(); + // #stepInvocationData is this build's working copy of the per-key maps (a lookup stashes the matched + // stage's map here, a run overwrites it). A failed build leaves its partial map behind. Clear it so + // the next build re-stashes the signature-matched map from the restored stage (the persisted copy + // lives inside each stage_metadata row and is re-read when #findStageCache matches). Without this, a + // long-lived consumer (ui5 serve) would pair the partial map with the next rebuild's stage, corrupting + // step selection and stale-output derivation. + this.#stepInvocationData.clear(); + // Return CAS rows buffered by a step that stored returns but whose build then aborted before the + // per-step flush: drop them, matching the cleared invocation data that would have referenced them. + this.#pendingStepReturnCasRows = []; // Reset the result cache state. A prior validateCache may have left it at NO_CACHE or // FRESH_AND_IN_USE, but the next build asserts PENDING_VALIDATION after restoring the // dependency index. this.#resultCacheState = RESULT_CACHE_STATES.PENDING_VALIDATION; // initSourceIndex does not touch this one, so reset it here. this.#changedDependencyResourcePaths = []; + // Root managers are held on the (now cleared) task caches; drop the remembered aggregate so the + // re-initialized caches re-establish it on the next validateCache. + this.#cachedRootAggregateSignature = undefined; // Clear per-build state so a failed build does not leak into the next one. // #currentResultSignature drives the #findResultCache early return; #currentStageSignatures // drives the isInitialImport/setStage guards in #importStages. @@ -1359,10 +1790,14 @@ export default class ProjectBuildCache { this.#resultCacheState = RESULT_CACHE_STATES.FRESH_AND_IN_USE; const changedPaths = this.#writtenResultResourcePaths; + // Record the root aggregate this build resolved against, so a later in-session validateCache can + // detect a root file changing without a source or dependency change. + this.#cachedRootAggregateSignature = this.#getAggregatedRootSignature(); + this.#currentResultSignature = this.#getResultStageSignature(); // Reset updated resource paths - this.#writtenResultResourcePaths = []; + this.#setWrittenResultResourcePaths([]); if (log.isLevelEnabled("perf")) { log.perf( `allTasksCompleted for project ${this.#project.getName()} ` + @@ -1377,13 +1812,41 @@ export default class ProjectBuildCache { } /** - * Generates the stage name for a given task + * Appends a written result resource path, keeping the parallel membership Set in sync. A path + * already recorded is ignored, so the ordered list stays free of duplicates without an O(n) scan. + * + * @param {string} resourcePath Resource path written by a stage or detected as a source change + */ + #addWrittenResultResourcePath(resourcePath) { + if (!this.#writtenResultResourcePathSet.has(resourcePath)) { + this.#writtenResultResourcePathSet.add(resourcePath); + this.#writtenResultResourcePaths.push(resourcePath); + } + } + + /** + * Replaces the written result resource paths and rebuilds the parallel membership Set from them. + * + * @param {string[]} paths New written result resource paths. The array is adopted by reference. + */ + #setWrittenResultResourcePaths(paths) { + this.#writtenResultResourcePaths = paths; + this.#writtenResultResourcePathSet = new Set(paths); + } + + /** + * Generates the stage id for a task, or for a single step of a step-based task. + * + * A legacy task maps to one stage task/{taskName}. A step-based task maps to one stage + * per step task/{taskName}::step/{stepName}, so each step is cached, validated, and folded + * into the result-stage signature independently. * * @param {string} taskName Name of the task - * @returns {string} Stage name in the format "task/{taskName}" + * @param {string} [stepName] Name of the step, for a step-based task's per-step stage + * @returns {string} Stage id */ - #getStageNameForTask(taskName) { - return `task/${taskName}`; + #stageIdFor(taskName, stepName) { + return stepName === undefined ? `task/${taskName}` : `task/${taskName}::step/${stepName}`; } /** @@ -1430,29 +1893,53 @@ export default class ProjectBuildCache { } } - // Import task caches - const buildTaskCaches = await Promise.all( - indexCache.tasks.map(async ([taskName, supportsDifferentialBuilds]) => { + // Import stage caches (one entry per stage: a legacy task's single stage, or a step-based + // task's per-step stages). + const buildStageCaches = await Promise.all( + indexCache.tasks.map(async ([stageId, stepBased]) => { const projectRequests = this.#cacheManager.readTaskMetadata( - this.#project.getId(), this.#buildSignature, taskName, "project"); + this.#project.getId(), this.#buildSignature, stageId, "project"); if (!projectRequests) { - throw new Error(`Failed to load project request cache for task ` + - `${taskName} in project ${this.#project.getName()}`); + throw new Error(`Failed to load project request cache for stage ` + + `${stageId} in project ${this.#project.getName()}`); } const dependencyRequests = this.#cacheManager.readTaskMetadata( - this.#project.getId(), this.#buildSignature, taskName, "dependencies"); + this.#project.getId(), this.#buildSignature, stageId, "dependencies"); if (!dependencyRequests) { - throw new Error(`Failed to load dependency request cache for task ` + - `${taskName} in project ${this.#project.getName()}`); + throw new Error(`Failed to load dependency request cache for stage ` + + `${stageId} in project ${this.#project.getName()}`); } - return BuildTaskCache.fromCache(this.#project.getName(), taskName, !!supportsDifferentialBuilds, - projectRequests, dependencyRequests); + // Input metadata (e.g. recorded env-var usage) is optional: absent for stages that + // declared no non-resource inputs, and absent in caches written before input + // tracking existed. + const inputTree = this.#cacheManager.readTaskMetadata( + this.#project.getId(), this.#buildSignature, stageId, "input"); + // Root request metadata is optional too: absent for stages that made no root reads, + // and absent in caches written before root tracking existed. Kept per useGitignore + // flag since the flag changes which resources a recorded glob matches. + const rootRequests = this.#cacheManager.readTaskMetadata( + this.#project.getId(), this.#buildSignature, stageId, "root"); + const rootNoGitignoreRequests = this.#cacheManager.readTaskMetadata( + this.#project.getId(), this.#buildSignature, stageId, "root-no-gitignore"); + return BuildStageCache.fromCache({ + projectName: this.#project.getName(), + stageId, + stepBased: !!stepBased, + projectRequests, + dependencyRequests, + inputSet: inputTree, + rootRequests, + rootNoGitignoreRequests, + }); }) ); - // Ensure taskCache is filled in the order of task execution - for (const buildTaskCache of buildTaskCaches) { - this.#taskCache.set(buildTaskCache.getTaskName(), buildTaskCache); + // Ensure stageCache is filled in the order of stage execution + for (const buildStageCache of buildStageCaches) { + this.#stageCaches.set(buildStageCache.getStageId(), buildStageCache); } + // Capture the restored stage order so the result-signature functions have the single source of + // truth available before this build's setTasks runs (result-cache validation happens first). + this.#stageOrder = indexCache.tasks.map(([stageId]) => stageId); // Force mode: Fail if cache is stale (source files changed OR pending changes exist) if (this.#cacheMode === Cache.Force && @@ -1471,7 +1958,7 @@ export default class ProjectBuildCache { } this.#sourceIndex = resourceIndex; // Since all source files are part of the result, declare any detected changes as newly written resources - this.#writtenResultResourcePaths = changedPaths; + this.#setWrittenResultResourcePaths(changedPaths); // Now awaiting initialization of dependency indices this.#combinedIndexState = INDEX_STATES.RESTORING_DEPENDENCY_INDICES; } else { @@ -1517,9 +2004,7 @@ export default class ProjectBuildCache { const changedPaths = [...removed, ...added, ...updated]; // Since all source files are part of the result, declare any detected changes as newly written resources for (const resourcePath of changedPaths) { - if (!this.#writtenResultResourcePaths.includes(resourcePath)) { - this.#writtenResultResourcePaths.push(resourcePath); - } + this.#addWrittenResultResourcePath(resourcePath); } return true; } @@ -1554,9 +2039,9 @@ export default class ProjectBuildCache { const cacheWriteStart = performance.now(); // Gather all cache data before opening any transactions - const stagePrepared = await this.#prepareTaskStageCache(); + const stagePrepared = await this.#prepareStageCache(); const resultPrepared = this.#prepareResultCache(); - const taskRequestPrepared = this.#prepareTaskRequestCache(); + const stageRequestPrepared = this.#prepareStageRequestCache(); const sourceIndexPrepared = this.#prepareSourceIndex(); // Calculate CAS rows - dedupe across stages (identical integrity produced by two stages writes once) @@ -1570,7 +2055,6 @@ export default class ProjectBuildCache { } } } - this.#cacheManager.transaction(() => { for (const {integrity, compressedBuffer} of allCasRows) { this.#cacheManager.putCompressedContent(integrity, compressedBuffer); @@ -1585,9 +2069,9 @@ export default class ProjectBuildCache { this.#project.getId(), this.#buildSignature, stageId, stageSignature, metadata); } - for (const {taskName, type, metadata} of taskRequestPrepared) { + for (const {stageId, type, metadata} of stageRequestPrepared) { this.#cacheManager.writeTaskMetadata( - this.#project.getId(), this.#buildSignature, taskName, type, metadata); + this.#project.getId(), this.#buildSignature, stageId, type, metadata); } if (sourceIndexPrepared) { this.#cacheManager.writeIndexCache( @@ -1617,8 +2101,8 @@ export default class ProjectBuildCache { log.verbose(`Preparing result metadata for project ${this.#project.getName()} ` + `using result stage signature ${stageSignature}`); const stageSignatures = Object.create(null); - for (const [stageName, stageSigs] of this.#currentStageSignatures.entries()) { - stageSignatures[stageName] = stageSigs.join("-"); + for (const [stageId, stageSigs] of this.#currentStageSignatures.entries()) { + stageSignatures[stageId] = createStageSignature(stageSigs); } return { @@ -1647,7 +2131,7 @@ export default class ProjectBuildCache { * casRows: Array<{integrity: string, compressedBuffer: Buffer}> * }>>} */ - async #prepareTaskStageCache() { + async #prepareStageCache() { if (!this.#stageCache.hasPendingCacheQueue()) { return []; } @@ -1657,7 +2141,7 @@ export default class ProjectBuildCache { const payloads = []; for (const [stageId, stageSignature] of stageQueue) { - const {stage, projectTagOperations, buildTagOperations} = + const {stage, projectTagOperations, buildTagOperations, stepInvocationData} = this.#stageCache.getCacheForSignature(stageId, stageSignature); const writer = stage.getWriter(); @@ -1695,6 +2179,14 @@ export default class ProjectBuildCache { } metadata.projectTagOperations = tagOpsToObject(projectTagOperations); metadata.buildTagOperations = tagOpsToObject(buildTagOperations); + if (stepInvocationData) { + // Embed the step's per-key map in the stage's own row, keyed by this stage signature, so the + // map and the stage output can never pair with a different run's data. Persisted as + // [[keyId, entry], ...] pairs since JSON has no Map; an empty map serializes as [] so a stage + // whose key set dropped to zero overwrites (under its new signature) rather than stranding the + // previous non-empty data. A legacy stage has no map and omits the field. + metadata.stepInvocationData = [...stepInvocationData]; + } payloads.push({stageId, stageSignature, metadata, casRows: casRowsForStage}); } @@ -1794,23 +2286,33 @@ export default class ProjectBuildCache { } /** - * Builds task-request metadata payloads for all tasks with new or modified entries. + * Builds stage-request metadata payloads for all stages with new or modified entries. * - * @returns {Array<{taskName: string, type: string, metadata: object}>} + * @returns {Array<{stageId: string, type: string, metadata: object}>} */ - #prepareTaskRequestCache() { + #prepareStageRequestCache() { const out = []; - for (const [taskName, taskCache] of this.#taskCache) { - if (!taskCache.hasNewOrModifiedCacheEntries()) { + for (const [stageId, stageCache] of this.#stageCaches) { + if (!stageCache.hasNewOrModifiedCacheEntries()) { continue; } - const [projectRequests, dependencyRequests] = taskCache.toCacheObjects(); - log.verbose(`Preparing task cache metadata for task ${taskName} in project ${this.#project.getName()}`); + const [projectRequests, dependencyRequests, inputTree, rootRequests, rootNoGitignoreRequests] = + stageCache.toCacheObjects(); + log.verbose(`Preparing cache metadata for stage ${stageId} in project ${this.#project.getName()}`); if (projectRequests) { - out.push({taskName, type: "project", metadata: projectRequests}); + out.push({stageId, type: "project", metadata: projectRequests}); } if (dependencyRequests) { - out.push({taskName, type: "dependencies", metadata: dependencyRequests}); + out.push({stageId, type: "dependencies", metadata: dependencyRequests}); + } + if (inputTree) { + out.push({stageId, type: "input", metadata: inputTree}); + } + if (rootRequests) { + out.push({stageId, type: "root", metadata: rootRequests}); + } + if (rootNoGitignoreRequests) { + out.push({stageId, type: "root-no-gitignore", metadata: rootNoGitignoreRequests}); } } return out; @@ -1832,9 +2334,11 @@ export default class ProjectBuildCache { log.verbose(`Preparing resource index cache for project ${this.#project.getName()} ` + `with build signature ${this.#buildSignature}`); const sourceIndexObject = this.#sourceIndex.toCacheObject(); + // One entry per stage in execution order: a legacy task's single stage, or a step-based task's + // per-step stages. The stage id is the metadata key everything else is stored under. const tasks = []; - for (const [taskName, taskCache] of this.#taskCache) { - tasks.push([taskName, taskCache.getSupportsDifferentialBuilds() ? 1 : 0]); + for (const [stageId, stageCache] of this.#stageCaches) { + tasks.push([stageId, stageCache.getStepBased() ? 1 : 0]); } return { projectId: this.#project.getId(), @@ -1862,7 +2366,7 @@ export default class ProjectBuildCache { #createReaderForStageCache(stageId, stageSignature, resourceMetadata) { const allResourcePaths = Object.keys(resourceMetadata); return createProxy({ - name: `Cache reader for task ${stageId} in project ${this.#project.getName()}`, + name: `Cache reader for stage in project ${this.#project.getName()}`, listResourcePaths: () => { return allResourcePaths; }, @@ -1873,7 +2377,7 @@ export default class ProjectBuildCache { const {lastModified, size, integrity, inode} = resourceMetadata[virPath]; if (size === undefined || lastModified === undefined || integrity === undefined) { - throw new Error(`Incomplete metadata for resource ${virPath} of task ${stageId} ` + + throw new Error(`Incomplete metadata for resource of stage ` + `in project ${this.#project.getName()}`); } @@ -1926,40 +2430,10 @@ function cartesianProduct(arrays) { } /** - * Fast combination of two arrays into pairs - * - * Creates all possible pairs by combining each element from the first array - * with each element from the second array. - * - * @param {Array} array1 First array - * @param {Array} array2 Second array - * @returns {Array} Array of two-element pairs + * A stage signature is an explicit tuple of four independent SHA-256 hex components. The tuple format + * and its join/split primitives live in ./stageSignature.js, shared with BuildStageCache so the two + * classes compose and decompose a signature the same way. */ -function combineTwoArraysFast(array1, array2) { - const len1 = array1.length; - const len2 = array2.length; - const result = new Array(len1 * len2); - - let idx = 0; - for (let i = 0; i < len1; i++) { - for (let j = 0; j < len2; j++) { - result[idx++] = [array1[i], array2[j]]; - } - } - - return result; -} - -/** - * Creates a combined stage signature from project and dependency signatures - * - * @param {string} projectSignature Project resource signature - * @param {string} dependencySignature Dependency resource signature - * @returns {string} Combined stage signature in format "projectSignature-dependencySignature" - */ -function createStageSignature(projectSignature, dependencySignature) { - return `${projectSignature}-${dependencySignature}`; -} /** * Creates a combined signature hash from multiple stage dependency signatures diff --git a/packages/project/lib/build/cache/ResourceRequestManager.js b/packages/project/lib/build/cache/ResourceRequestManager.js index bb29c90c1bd..c9e3711193e 100644 --- a/packages/project/lib/build/cache/ResourceRequestManager.js +++ b/packages/project/lib/build/cache/ResourceRequestManager.js @@ -42,29 +42,34 @@ function serializeUnresolvedRequests(entry, unresolvedRequests) { * @class */ class ResourceRequestManager { - #taskName; + #ownerId; #projectName; #requestGraph; #treeRegistries = []; #treeUpdateDeltas = new Map(); + // Per-updateIndices scratch: each affected node's exposed (composite) signature captured before its + // unresolved requests are drained, so a recorded delta keys on the same signature the stage was + // stored under (getIndexSignatures folds unresolved requests in; the tree hash alone does not). + #deltaOriginalComposite = new Map(); #hasNewOrModifiedCacheEntries; #useDifferentialUpdate; #unusedAtLeastOnce; + #clearedExistingRequests = false; /** * Creates a new ResourceRequestManager instance * * @param {string} projectName Name of the project - * @param {string} taskName Name of the task + * @param {string} ownerId Identifier of the request owner (a stage id, or a stage's root-reads label) * @param {boolean} useDifferentialUpdate Whether to track differential updates * @param {ResourceRequestGraph} [requestGraph] Optional pre-existing request graph from cache - * @param {boolean} [unusedAtLeastOnce=false] Whether the task has been unused at least once + * @param {boolean} [unusedAtLeastOnce=false] Whether the request owner has been unused at least once */ - constructor(projectName, taskName, useDifferentialUpdate, requestGraph, unusedAtLeastOnce = false) { + constructor(projectName, ownerId, useDifferentialUpdate, requestGraph, unusedAtLeastOnce = false) { this.#projectName = projectName; - this.#taskName = taskName; + this.#ownerId = ownerId; this.#useDifferentialUpdate = useDifferentialUpdate; this.#unusedAtLeastOnce = unusedAtLeastOnce; if (requestGraph) { @@ -83,7 +88,7 @@ class ResourceRequestManager { * including both root indices and delta indices for differential updates. * * @param {string} projectName Name of the project - * @param {string} taskName Name of the task + * @param {string} ownerId Identifier of the request owner (a stage id, or a stage's root-reads label) * @param {boolean} useDifferentialUpdate Whether to track differential updates * @param {object} cacheData Cached metadata object * @param {object} cacheData.requestSetGraph Serialized request graph @@ -92,12 +97,12 @@ class ResourceRequestManager { * @param {boolean} [cacheData.unusedAtLeastOnce] Whether the task has been unused * @returns {ResourceRequestManager} Restored manager instance */ - static fromCache(projectName, taskName, useDifferentialUpdate, { + static fromCache(projectName, ownerId, useDifferentialUpdate, { requestSetGraph, rootIndices, deltaIndices, unusedAtLeastOnce }) { const requestGraph = ResourceRequestGraph.fromCache(requestSetGraph); const resourceRequestManager = new ResourceRequestManager( - projectName, taskName, useDifferentialUpdate, requestGraph, unusedAtLeastOnce); + projectName, ownerId, useDifferentialUpdate, requestGraph, unusedAtLeastOnce); const registries = new Map(); // Restore root resource indices for (const {nodeId, resourceIndex: serializedIndex, unresolvedRequests} of rootIndices) { @@ -115,7 +120,7 @@ class ResourceRequestManager { const registry = registries.get(node.getParentId()); if (!registry) { throw new Error(`Missing tree registry for parent of node ID ${nodeId} of task ` + - `'${taskName}' of project '${projectName}'`); + `'${ownerId}' of project '${projectName}'`); } const resourceIndex = parentResourceIndex.deriveTreeWithIndex(addedResourceIndex); @@ -201,7 +206,7 @@ class ResourceRequestManager { })); if (log.isLevelEnabled("perf")) { log.perf( - `refreshIndices for task '${this.#taskName}' of project '${this.#projectName}' ` + + `refreshIndices for '${this.#ownerId}' of project '${this.#projectName}' ` + `completed in ${(performance.now() - refreshStart).toFixed(2)} ms: ` + `${totalResourcesFetched} resources fetched, ${totalResourcesRemoved} resources removed`); } @@ -224,6 +229,7 @@ class ResourceRequestManager { async updateIndices(reader, changedResourcePaths) { const matchingRequestSetIds = []; const updatesByRequestSetId = new Map(); + this.#deltaOriginalComposite.clear(); if (this.#requestGraph.getSize() === 0) { // No requests recorded -> No updates necessary return false; @@ -283,7 +289,7 @@ class ResourceRequestManager { } if (log.isLevelEnabled("perf")) { log.perf( - `updateIndices for task '${this.#taskName}' of project '${this.#projectName}' ` + + `updateIndices for '${this.#ownerId}' of project '${this.#projectName}' ` + `resource fetch completed in ${(performance.now() - fetchStart).toFixed(2)} ms: ` + `${cacheHits} cache hits, ${cacheMisses} cache misses`); } @@ -295,6 +301,10 @@ class ResourceRequestManager { if (!resourceIndex) { throw new Error(`Missing resource index for request set ID ${requestSetId}`); } + // Capture the exposed signature (tree + unresolved requests) before draining below, so the + // delta records the same signature the stage was stored under (see #deltaOriginalComposite). + this.#deltaOriginalComposite.set(requestSetId, + this.#computeNodeSignature(resourceIndex, metadata.unresolvedRequests)); const resourcePathsToUpdate = updatesByRequestSetId.get(requestSetId); const resourcesToUpdate = []; @@ -425,8 +435,14 @@ class ResourceRequestManager { hasChanges = true; } for (const [tree, diff] of res.treeStats) { - const [requestSetId, originalSignature] = previousTreeSignatures.get(tree); - const newSignature = tree.getRootHash(); + const [requestSetId] = previousTreeSignatures.get(tree); + const metadata = this.#requestGraph.getMetadata(requestSetId); + // Key the delta on the exposed (composite) signatures, matching getIndexSignatures and + // the signature the stage was stored under. For a node without unresolved requests the + // composite equals the tree hash, so this is a no-op for the common case. + const originalSignature = + this.#deltaOriginalComposite.get(requestSetId) ?? previousTreeSignatures.get(tree)[1]; + const newSignature = this.#computeNodeSignature(metadata.resourceIndex, metadata.unresolvedRequests); this.#addDeltaEntry(requestSetId, originalSignature, newSignature, diff); } } @@ -447,7 +463,7 @@ class ResourceRequestManager { const results = await Promise.all(this.#treeRegistries.map((registry) => registry.flush())); if (log.isLevelEnabled("perf")) { log.perf( - `#flushTreeChanges for task '${this.#taskName}' of project '${this.#projectName}' ` + + `#flushTreeChanges for '${this.#ownerId}' of project '${this.#projectName}' ` + `completed in ${(performance.now() - flushStart).toFixed(2)} ms ` + `across ${this.#treeRegistries.length} registries`); } @@ -508,8 +524,9 @@ class ResourceRequestManager { * Gets all delta entries for differential cache updates * * Returns a map of signature transitions and their associated changed resource paths. - * Only includes deltas where no resources were removed, as removed resources prevent - * differential updates. + * A removed resource is included as a changed path: a step that read the removed + * input then re-runs (or, for a gone key, drops out), and its stale output is dropped from the + * carried-forward stage via the changed-paths merge in ProjectBuildCache.recordStageResult. * * @public * @returns {Map} Map from original signature to delta information @@ -521,11 +538,7 @@ class ResourceRequestManager { let changedPaths; if (diff) { const {added, updated, removed} = diff; - if (removed.length) { - // Cannot use differential build if a resource has been removed - continue; - } - changedPaths = Array.from(new Set([...added, ...updated])); + changedPaths = Array.from(new Set([...added, ...updated, ...removed])); } else { changedPaths = []; } @@ -578,6 +591,39 @@ class ResourceRequestManager { return "X"; // Signature for when no requests were made } + /** + * Clears all recorded resource requests, resetting the manager to an empty state. + * + * Marks the manager modified and remembers that it previously held requests, so the now-empty + * state is persisted (overwriting the stored request set) rather than skipped the way a + * never-used manager is. Used when a stage recorded reads on an earlier build but records none + * now, so its signature stops folding resources it no longer reads. + * + * @public + */ + clear() { + this.#requestGraph = new ResourceRequestGraph(); + this.#treeRegistries = []; + this.#treeUpdateDeltas = new Map(); + this.#deltaOriginalComposite = new Map(); + this.#unusedAtLeastOnce = false; + this.#hasNewOrModifiedCacheEntries = true; + this.#clearedExistingRequests = true; + } + + /** + * Whether clear() emptied a manager that previously held requests + * + * Distinguishes a manager cleared this build, whose now-empty state must be persisted to + * overwrite the stored request set, from a manager that was always empty and is not persisted. + * + * @public + * @returns {boolean} + */ + wasCleared() { + return this.#clearedExistingRequests; + } + /** * Adds a request set and creates or reuses a resource index * @@ -590,19 +636,24 @@ class ResourceRequestManager { * @returns {Promise} Object containing setId and signature of the resource index */ async #addRequestSet(requests, reader) { - this.#hasNewOrModifiedCacheEntries = true; // Try to find an existing request set that we can reuse let setId = this.#requestGraph.findExactMatch(requests); let resourceIndex; let unresolvedRequests; if (setId) { // Reuse existing resource index. - // Note: This index has already been updated before the task executed, so no update is necessary here + // Note: This index has already been updated before the task executed, so no update is necessary + // here, and nothing in the persisted request graph changed: the manager stays clean so the whole + // request graph is not needlessly re-serialized to SQLite. (A tree update that moved the index's + // signature flags the manager dirty itself in updateIndices.) Recording the same request set on + // every delta build is the common case for a step-based stage, so leaving the flag untouched here + // is what keeps a one-file-changed build from rewriting every stage's request cache. const existingMetadata = this.#requestGraph.getMetadata(setId); resourceIndex = existingMetadata.resourceIndex; unresolvedRequests = existingMetadata.unresolvedRequests; } else { // New request set, check whether we can create a delta + this.#hasNewOrModifiedCacheEntries = true; const metadata = {}; // Will populate with resourceIndex below setId = this.#requestGraph.addRequestSet(requests, metadata); @@ -614,7 +665,7 @@ class ResourceRequestManager { const addedRequests = requestSet.getAddedRequests(); const resourcesToAdd = await this.#getResourcesForRequests(addedRequests, reader); - log.verbose(`Task '${this.#taskName}' of project '${this.#projectName}' ` + + log.verbose(`Request owner '${this.#ownerId}' of project '${this.#projectName}' ` + `created derived resource index for request set ID ${setId} ` + `based on parent ID ${parentId} with ${resourcesToAdd.length} additional resources`); resourceIndex = await parentResourceIndex.deriveTree(resourcesToAdd); diff --git a/packages/project/lib/build/cache/StageCache.js b/packages/project/lib/build/cache/StageCache.js index f6680cf7111..08ec8dedc28 100644 --- a/packages/project/lib/build/cache/StageCache.js +++ b/packages/project/lib/build/cache/StageCache.js @@ -4,6 +4,8 @@ * @property {string[]} writtenResourcePaths Array of resource paths written during stage execution * @property {Map>} resourceTagOperations * Map of resource paths to their tags that were set or cleared during this stage's execution + * @property {Map} [stepInvocationData] A step-based stage's per-key invocation data, + * travelling with the stage under the same signature so the pair can never separate */ /** @@ -45,8 +47,11 @@ export default class StageCache { * @param {Map>} projectTagOperations * @param {Map>} buildTagOperations * Map of resource paths to their tags that were set or cleared during this stage's execution + * @param {Map} [stepInvocationData] A step-based stage's per-key invocation data, + * stored under this signature so a later lookup pairs the stage output with the matching per-key map */ - addSignature(stageId, signature, stageInstance, writtenResourcePaths, projectTagOperations, buildTagOperations) { + addSignature(stageId, signature, stageInstance, writtenResourcePaths, projectTagOperations, buildTagOperations, + stepInvocationData) { if (!this.#stageIdToSignatures.has(stageId)) { this.#stageIdToSignatures.set(stageId, new Map()); } @@ -57,6 +62,7 @@ export default class StageCache { writtenResourcePaths, projectTagOperations, buildTagOperations, + stepInvocationData, }); this.#cacheQueue.push([stageId, signature]); } diff --git a/packages/project/lib/build/cache/index/TaskInputSet.js b/packages/project/lib/build/cache/index/TaskInputSet.js new file mode 100644 index 00000000000..37956c43f2c --- /dev/null +++ b/packages/project/lib/build/cache/index/TaskInputSet.js @@ -0,0 +1,231 @@ +import crypto from "node:crypto"; + +// Signature of a set with zero recorded entries: the sha256 digest of no input. Every stage of a +// standard build has an empty input set, so #computeSignature returns this precomputed constant +// instead of hashing nothing on each getSignature/getSignatureWithCurrentValues call. +const EMPTY_SIGNATURE = crypto.createHash("sha256").digest("hex"); + +/** + * @typedef {object} @ui5/project/build/cache/index/TaskInputSet~InputEntry + * @property {string} type Input type, e.g. "env" for an environment variable or "project.getVersion" + * for a value read from a dependency's project interface. + * @property {string} name Input name within the type (e.g. the environment variable name or, for + * project inputs, the name of the project the value was read from). + * @property {string|undefined} value Normalized input value at recording time. May be + * undefined (e.g. an environment variable that is not set). An unset input is a + * meaningful, distinct input. + */ + +/** + * Normalizes a raw input value to the string form used for hashing and comparison. + * + * Both the recording side ([MonitoredTaskUtil]{@link @ui5/project/build/helpers/MonitoredTaskUtil}) + * and the lookup side + * ([ProjectBuildContext#resolveInputValue]{@link @ui5/project/build/helpers/ProjectBuildContext}) + * run values through this function, so a value + * recorded during one build and the current value re-derived during a later build produce the same + * string when they are semantically equal. + * + * Objects and arrays are serialized with object keys sorted, so a key-order difference does not + * register as a changed input. Arrays keep their order, which can be significant. + * + * @param {*} rawValue Raw value as returned by a TaskUtil method + * @returns {string|undefined} Normalized value, or undefined for an absent value + */ +export function normalizeInputValue(rawValue) { + if (rawValue === undefined || rawValue === null) { + return undefined; + } + const type = typeof rawValue; + if (type === "string") { + return rawValue; + } + if (type === "boolean" || type === "number" || type === "bigint") { + return String(rawValue); + } + // Objects and arrays: stable JSON with sorted object keys + return JSON.stringify(rawValue, (key, value) => { + if (value && typeof value === "object" && !Array.isArray(value)) { + return Object.keys(value).sort().reduce((sorted, k) => { + sorted[k] = value[k]; + return sorted; + }, {}); + } + return value; + }); +} + +/** + * Tracks the non-resource inputs that influenced a task's output. + * + * This is a sibling of the resource-focused + * [HashTree]{@link @ui5/project/build/cache/index/HashTree}, but deliberately not a tree: task + * inputs are few, unordered and non-hierarchical, so this holds a flat set of typed input entries + * (keyed by type + name) and hashes them into a single signature. That + * signature is folded into the task's stage signature so that a changed input invalidates the + * task's cached result exactly like a changed resource does. It shares HashTree's cache-object + * conventions (a version field, tolerant {@link #fromCache}) but none of its Merkle + * structure, structural sharing or delta detection, which flat inputs do not need. + * + * Recorded input types include env (environment variables read via + * [TaskUtil#getEnv]{@link @ui5/project/build/helpers/TaskUtil#getEnv}) and reads from the + * TaskUtil interface such as isRootProject, getDependencies and the + * project.* accessors (e.g. a dependency's version via + * getProject(name).getVersion()). See + * [MonitoredTaskUtil]{@link @ui5/project/build/helpers/MonitoredTaskUtil} for what is tracked. + * + * Only the recorded type/name pairs are persisted, never the values. A + * later build re-reads the current value for each recorded input via + * {@link #getSignatureWithCurrentValues}, so a cache lookup reflects the environment and graph of + * the build performing the lookup. + */ +export default class TaskInputSet { + // Map key: `${type}\0${name}` -> InputEntry + #entries = new Map(); + // Memoized result of getEntries(). The map is populated only in the constructor and never mutated + // afterward, so the sorted array stays valid for the instance's lifetime (no invalidation needed). + #sortedEntries = null; + + /** + * @param {@ui5/project/build/cache/index/TaskInputSet~InputEntry[]} [entries] + * Initial input entries. Values are expected to be normalized already (see + * {@link normalizeInputValue}). + */ + constructor(entries = []) { + for (const entry of entries) { + this.#entries.set(TaskInputSet.#key(entry.type, entry.name), { + type: entry.type, + name: entry.name, + value: entry.value, + }); + } + } + + static #key(type, name) { + return `${type}\0${name}`; + } + + /** + * Whether any input entries have been recorded. + * + * @returns {boolean} + */ + isEmpty() { + return this.#entries.size === 0; + } + + /** + * Returns the recorded input entries in a stable order (sorted by type, then name). + * + * @returns {@ui5/project/build/cache/index/TaskInputSet~InputEntry[]} + */ + getEntries() { + if (!this.#sortedEntries) { + // Sort by the composite `type\0name` key with a plain relational comparison. Types and names + // are ASCII identifiers, so code-point order matches the previous localeCompare order (covered + // by the sort-order equivalence test) while avoiding ICU collation on this hot signature path. + this.#sortedEntries = Array.from(this.#entries.values()).sort((a, b) => { + const keyA = TaskInputSet.#key(a.type, a.name); + const keyB = TaskInputSet.#key(b.type, b.name); + if (keyA < keyB) { + return -1; + } + return keyA > keyB ? 1 : 0; + }); + } + return this.#sortedEntries; + } + + /** + * Computes the signature over the recorded entries and their recorded values. + * + * @returns {string} Input signature + */ + getSignature() { + return this.#computeSignature((entry) => entry.value); + } + + /** + * Computes the signature over the recorded entry names, reading each value freshly via the given + * resolver instead of using the recorded value. + * + * Used on cache lookup: the recorded names identify which inputs the task consumed last time; the + * current values decide whether the cached result still applies. + * + * @param {function(string, string, (string|undefined)): (string|undefined)} [resolveValue] + * Returns the current normalized value for an input, given its type, + * name and recorded value. Defaults to reading process.env for + * env inputs and falling back to the recorded value for any other type. + * @returns {string} Input signature computed with current values + */ + getSignatureWithCurrentValues(resolveValue) { + const resolve = resolveValue ?? ((type, name, recordedValue) => { + return type === "env" ? process.env[name] : recordedValue; + }); + return this.#computeSignature((entry) => resolve(entry.type, entry.name, entry.value)); + } + + /** + * Computes the signature over the recorded entries. An empty set produces a stable, fixed digest + * (the hash of zero entries). + * + * @param {function(@ui5/project/build/cache/index/TaskInputSet~InputEntry): (string|undefined)} getValue + * @returns {string} + * @private + */ + #computeSignature(getValue) { + if (this.#entries.size === 0) { + return EMPTY_SIGNATURE; + } + const hash = crypto.createHash("sha256"); + // Entries are hashed in stable (type, name) order. Fields are NUL-separated: types are known + // identifiers, names and values are arbitrary strings, but none may contain a NUL byte, so the + // concatenation is unambiguous. An unset value is rendered as a distinct marker so it cannot + // collide with an empty-string value. + for (const entry of this.getEntries()) { + const value = getValue(entry); + hash.update(entry.type); + hash.update("\0"); + hash.update(entry.name); + hash.update("\0"); + hash.update(value === undefined ? "\0unset" : value); + hash.update("\0"); + } + return hash.digest("hex"); + } + + /** + * Serializes the recorded entry names and types for persistence. + * + * Values are deliberately not persisted: a later build re-reads the current value for each + * recorded name (see {@link #getSignatureWithCurrentValues}). + * + * @returns {object} Serialized cache object + */ + toCacheObject() { + return { + version: 1, + entries: this.getEntries().map(({type, name}) => ({type, name})), + }; + } + + /** + * Restores a TaskInputSet from its serialized form. + * + * An "input" metadata row is written only for tasks that recorded at least one input, so a + * missing row (null/undefined) yields an empty set. + * + * @param {object} [data] Serialized cache object created by {@link #toCacheObject} + * @returns {TaskInputSet} + */ + static fromCache(data) { + if (!data) { + return new TaskInputSet(); + } + if (data.version !== 1) { + throw new Error(`Unsupported TaskInputSet version: ${data.version}`); + } + // Restored entries carry no value; a lookup reads current values by name. + return new TaskInputSet(data.entries.map(({type, name}) => ({type, name, value: undefined}))); + } +} diff --git a/packages/project/lib/build/cache/stageSignature.js b/packages/project/lib/build/cache/stageSignature.js new file mode 100644 index 00000000000..284dd443cce --- /dev/null +++ b/packages/project/lib/build/cache/stageSignature.js @@ -0,0 +1,53 @@ +/** + * Stage-signature tuple format, shared by {@link @ui5/project/build/cache/ProjectBuildCache} and + * {@link @ui5/project/build/cache/BuildStageCache}. + * + * A stage signature is an explicit tuple of four independent components joined by + * {@link STAGE_SIGNATURE_SEPARATOR}, in order: project resources, dependency resources, non-resource + * inputs (env vars, tracked TaskUtil reads), and root resources. Each component is a SHA-256 hex + * digest (or the digest of an empty set), so the separator never occurs inside a component and the + * join is unambiguous and reversible via {@link splitStageSignature}. + * + * Keeping the four dimensions as separate slots, rather than folding inputs and root into the project + * component, lets a delta lookup pair a changed project or dependency signature with the current input + * and root signatures directly, with no reverse mapping back from a combined value. + * + * @module @ui5/project/build/cache/stageSignature + */ + +/** + * Separator between the components of a stage signature. A single ASCII character that cannot occur in + * a hex digest, so the join is unambiguous. + * + * @type {string} + */ +export const STAGE_SIGNATURE_SEPARATOR = "-"; + +/** + * Index of the dependency component within a stage-signature tuple. Used to read the dependency + * signature a stage was keyed on back out for the result-stage signature. + * + * @type {number} + */ +export const STAGE_SIG_DEPENDENCY_INDEX = 1; + +/** + * Joins a stage-signature tuple into its string form. + * + * @param {string[]} components The [project, dependency, input, root] signature components + * @returns {string} Combined stage signature + */ +export function createStageSignature(components) { + return components.join(STAGE_SIGNATURE_SEPARATOR); +} + +/** + * Splits a stage-signature string back into its tuple components. Each component is a hex digest, so + * the split is lossless. + * + * @param {string} signature A stage signature produced by {@link createStageSignature} + * @returns {string[]} The [project, dependency, input, root] signature components + */ +export function splitStageSignature(signature) { + return signature.split(STAGE_SIGNATURE_SEPARATOR); +} diff --git a/packages/project/lib/build/definitions/application.js b/packages/project/lib/build/definitions/application.js index 9b502502836..0fdd3359683 100644 --- a/packages/project/lib/build/definitions/application.js +++ b/packages/project/lib/build/definitions/application.js @@ -13,6 +13,7 @@ import {enhanceBundlesWithDefaults} from "../../validation/validator.js"; export default function({project, taskUtil, getTask}) { const tasks = new Map(); tasks.set("escapeNonAsciiCharacters", { + stepBased: true, options: { encoding: project.getPropertiesFileSourceEncoding(), pattern: "/**/*.properties" @@ -20,7 +21,7 @@ export default function({project, taskUtil, getTask}) { }); tasks.set("replaceCopyright", { - supportsDifferentialBuilds: true, + stepBased: true, options: { copyright: project.getCopyright(), pattern: "/**/*.{js,json}" @@ -28,7 +29,7 @@ export default function({project, taskUtil, getTask}) { }); tasks.set("replaceVersion", { - supportsDifferentialBuilds: true, + stepBased: true, options: { version: project.getVersion(), pattern: "/**/*.{js,json}" @@ -44,13 +45,13 @@ export default function({project, taskUtil, getTask}) { } } tasks.set("minify", { - supportsDifferentialBuilds: true, + stepBased: true, options: { pattern: minificationPattern } }); - tasks.set("enhanceManifest", {}); + tasks.set("enhanceManifest", {stepBased: true}); tasks.set("generateFlexChangesBundle", {}); diff --git a/packages/project/lib/build/definitions/component.js b/packages/project/lib/build/definitions/component.js index 84939958e91..2fbb396bb89 100644 --- a/packages/project/lib/build/definitions/component.js +++ b/packages/project/lib/build/definitions/component.js @@ -13,6 +13,7 @@ import {enhanceBundlesWithDefaults} from "../../validation/validator.js"; export default function({project, taskUtil, getTask}) { const tasks = new Map(); tasks.set("escapeNonAsciiCharacters", { + stepBased: true, options: { encoding: project.getPropertiesFileSourceEncoding(), pattern: "/**/*.properties" @@ -20,7 +21,7 @@ export default function({project, taskUtil, getTask}) { }); tasks.set("replaceCopyright", { - supportsDifferentialBuilds: true, + stepBased: true, options: { copyright: project.getCopyright(), pattern: "/**/*.{js,json}" @@ -28,7 +29,7 @@ export default function({project, taskUtil, getTask}) { }); tasks.set("replaceVersion", { - supportsDifferentialBuilds: true, + stepBased: true, options: { version: project.getVersion(), pattern: "/**/*.{js,json}" @@ -43,13 +44,13 @@ export default function({project, taskUtil, getTask}) { } tasks.set("minify", { - supportsDifferentialBuilds: true, + stepBased: true, options: { pattern: minificationPattern } }); - tasks.set("enhanceManifest", {}); + tasks.set("enhanceManifest", {stepBased: true}); tasks.set("generateFlexChangesBundle", {}); diff --git a/packages/project/lib/build/definitions/library.js b/packages/project/lib/build/definitions/library.js index 2229d0b9815..ec457d11517 100644 --- a/packages/project/lib/build/definitions/library.js +++ b/packages/project/lib/build/definitions/library.js @@ -14,6 +14,7 @@ import {enhanceBundlesWithDefaults} from "../../validation/validator.js"; export default function({project, taskUtil, getTask}) { const tasks = new Map(); tasks.set("escapeNonAsciiCharacters", { + stepBased: true, options: { encoding: project.getPropertiesFileSourceEncoding(), pattern: "/**/*.properties" @@ -21,7 +22,7 @@ export default function({project, taskUtil, getTask}) { }); tasks.set("replaceCopyright", { - supportsDifferentialBuilds: true, + stepBased: true, options: { copyright: project.getCopyright(), pattern: "/**/*.{js,library,css,less,theme,html}" @@ -29,7 +30,7 @@ export default function({project, taskUtil, getTask}) { }); tasks.set("replaceVersion", { - supportsDifferentialBuilds: true, + stepBased: true, options: { version: project.getVersion(), pattern: "/**/*.{js,json,library,css,less,theme,html}" @@ -37,7 +38,7 @@ export default function({project, taskUtil, getTask}) { }); tasks.set("replaceBuildtime", { - supportsDifferentialBuilds: true, + stepBased: true, options: { pattern: "/resources/sap/ui/{Global,core/Core}.js" } @@ -89,7 +90,7 @@ export default function({project, taskUtil, getTask}) { } tasks.set("minify", { - supportsDifferentialBuilds: true, + stepBased: true, options: { pattern: minificationPattern } @@ -102,7 +103,7 @@ export default function({project, taskUtil, getTask}) { tasks.set("generateLibraryManifest", {taskFunction: null}); } - tasks.set("enhanceManifest", {}); + tasks.set("enhanceManifest", {stepBased: true}); const bundles = project.getBundles(); const existingBundleDefinitionNames = @@ -159,6 +160,7 @@ export default function({project, taskUtil, getTask}) { } tasks.set("buildThemes", { + stepBased: true, requiresDependencies: true, options: { projectName: project.getName(), @@ -170,6 +172,7 @@ export default function({project, taskUtil, getTask}) { if (project.isFrameworkProject()) { tasks.set("generateThemeDesignerResources", { + stepBased: true, requiresDependencies: true, options: { version: project.getVersion() diff --git a/packages/project/lib/build/definitions/themeLibrary.js b/packages/project/lib/build/definitions/themeLibrary.js index debbaf67332..76b6eae08ff 100644 --- a/packages/project/lib/build/definitions/themeLibrary.js +++ b/packages/project/lib/build/definitions/themeLibrary.js @@ -11,7 +11,7 @@ export default function({project, taskUtil, getTask}) { const tasks = new Map(); tasks.set("replaceCopyright", { - supportsDifferentialBuilds: true, + stepBased: true, options: { copyright: project.getCopyright(), pattern: "/resources/**/*.{less,theme}" @@ -19,7 +19,7 @@ export default function({project, taskUtil, getTask}) { }); tasks.set("replaceVersion", { - supportsDifferentialBuilds: true, + stepBased: true, options: { version: project.getVersion(), pattern: "/resources/**/*.{less,theme}" @@ -27,6 +27,7 @@ export default function({project, taskUtil, getTask}) { }); tasks.set("buildThemes", { + stepBased: true, requiresDependencies: true, options: { projectName: project.getName(), @@ -38,6 +39,7 @@ export default function({project, taskUtil, getTask}) { if (project.isFrameworkProject()) { tasks.set("generateThemeDesignerResources", { + stepBased: true, requiresDependencies: true, options: { version: project.getVersion() diff --git a/packages/project/lib/build/helpers/BuildContext.js b/packages/project/lib/build/helpers/BuildContext.js index 4d398e98a1c..f4e2f7f9ed1 100644 --- a/packages/project/lib/build/helpers/BuildContext.js +++ b/packages/project/lib/build/helpers/BuildContext.js @@ -14,6 +14,7 @@ import Cache from "../cache/Cache.js"; */ class BuildContext { #cacheManager; + #buildTime; constructor(graph, taskRepository, { // buildConfig selfContained = false, @@ -85,6 +86,41 @@ class BuildContext { this._ui5DataDir = ui5DataDir; this._projectBuildContexts = new Map(); + + // One timestamp per build run, shared by every time quantization (TaskUtil#getTime on the + // record side, ProjectBuildContext#resolveInputValue on the lookup side). Initialized here as + // a safe default and refreshed at the start of each run via refreshBuildTime(); see there. + this.#buildTime = new Date(); + } + + /** + * Establishes the timestamp for a new build run. + * + * Called at the start of each run (ProjectBuilder#build / #validate), which are mutually + * exclusive, so no run observes another run's refresh. A single BuildContext is reused across + * many runs of a long-running consumer (BuildServer / `ui5 serve`), so the timestamp cannot be + * fixed at construction: `getTime("year")` would freeze at the server's start bucket and, because + * a cached time input would keep agreeing with itself, silently serve stale time-derived output + * (e.g. replaceCopyright's `${currentYear}` after New Year) without ever missing the cache. + * Refreshing per run keeps the timestamp constant within a run (all projects, and the record and + * lookup within that run, agree) while advancing across runs so a rolled-over bucket misses the + * cache and re-runs the task. + */ + refreshBuildTime() { + this.#buildTime = new Date(); + } + + /** + * Returns the timestamp of the current build run, used for all time quantization. + * + * Consumers must read this live at use time rather than snapshot it: TaskUtil and + * ProjectBuildContext instances are constructed once and reused across runs, while the timestamp + * is refreshed per run by {@link #refreshBuildTime}. + * + * @returns {Date} The current build run's timestamp + */ + getBuildTime() { + return this.#buildTime; } getRootProject() { diff --git a/packages/project/lib/build/helpers/MonitoredTaskUtil.js b/packages/project/lib/build/helpers/MonitoredTaskUtil.js new file mode 100644 index 00000000000..b1596671025 --- /dev/null +++ b/packages/project/lib/build/helpers/MonitoredTaskUtil.js @@ -0,0 +1,363 @@ +import {normalizeInputValue} from "../cache/index/TaskInputSet.js"; +import {createMonitor} from "@ui5/fs/resourceFactory"; + +// TaskUtil interfaced-project accessors whose return value is a task input. Reading one of these +// during a task makes the value part of that task's build-cache signature. Deliberately excluded: +// getRootReader/getReader (their resource reads are tracked separately as resource requests) and +// getRootPath/getSourcePath (absolute, machine-specific paths that would make cache entries +// non-portable). +const TRACKED_PROJECT_METHODS = new Set([ + "getType", "getName", "getVersion", "getNamespace", + "getCustomConfiguration", "isFrameworkProject", + "getFrameworkName", "getFrameworkVersion", "getFrameworkDependencies", +]); + +// TaskUtil methods whose return value is a task input, keyed by method name. Each records the read +// under the given input type. `getProject` and `getDependencies` are handled separately because +// they need the resolved project name (see the constructor). Deliberately absent: `getBuildTime`, +// which returns the raw per-run timestamp and is an untracked passthrough (it advances every build, +// so tracking it would miss the cache every time); contrast the quantized, tracked `getTime`. +const TRACKED_TASK_UTIL_METHODS = { + getEnv: "env", + getTime: "time", + isRootProject: "isRootProject", +}; + +// Tracked methods whose first argument is the input name (the environment variable name, the time +// granularity). Their read is recorded under that argument. Argument-less tracked methods +// (`isRootProject`) record under the empty name. +const NAME_ARG_METHODS = new Set(["getEnv", "getTime"]); + +// Root-reader subtrees excluded from tracking by default. A wide glob (e.g. "/**") over the project +// root would otherwise pull the whole dependency install and the git database into the build-cache +// signature. A recorded glob only reaches these when it targets them explicitly (see +// `augmentRootPattern`), so a bundler that wants a third-party package under `node_modules` opts in +// with `getRootReader({useGitignore: false}).byGlob("/node_modules//**")`. +const ROOT_IGNORE_PREFIXES = ["/node_modules", "/.git"]; + +// Negation globs for the ignored subtrees, applied to a recorded glob that does not opt into one +// explicitly. Derived once from ROOT_IGNORE_PREFIXES so the two stay in sync. +const ROOT_IGNORE_NEGATIONS = ROOT_IGNORE_PREFIXES.map((prefix) => `!${prefix}/**`); + +/** + * Whether a single glob pattern targets one of the default-ignored root subtrees explicitly. + * + * A leading "!" marks a negation, which never opts a subtree in. Everything else counts as explicit + * when it starts with an ignored prefix followed by "/" or the pattern end, so "/node_modules/x/**" + * opts in while "/**" and "/node_modules_stuff/**" do not. + * + * @param {string} pattern Glob pattern + * @returns {boolean} True if the pattern explicitly enters an ignored subtree + */ +function patternEntersIgnoredSubtree(pattern) { + if (typeof pattern !== "string" || pattern.startsWith("!")) { + return false; + } + return ROOT_IGNORE_PREFIXES.some((prefix) => + pattern === prefix || pattern.startsWith(`${prefix}/`)); +} + +/** + * Applies the default root-monitor ignore to a recorded glob request. + * + * A request that already targets `node_modules` or `.git` explicitly is recorded unchanged so its + * content is tracked. Any other request gains negations for those subtrees, so re-materializing the + * request set on a later build resolves the same bounded resource set the record-time read intended. + * + * @param {string|string[]} pattern Recorded glob pattern (single or array form) + * @returns {string|string[]} Pattern, augmented with ignore negations unless it opts in explicitly + */ +function augmentRootPattern(pattern) { + const patterns = Array.isArray(pattern) ? pattern : [pattern]; + if (patterns.some(patternEntersIgnoredSubtree)) { + return pattern; + } + return [...patterns, ...ROOT_IGNORE_NEGATIONS]; +} + +/** + * Records the inputs a task reads through its [TaskUtil]{@link @ui5/project/build/helpers/TaskUtil}, + * analogous to how [MonitoredReader]{@link @ui5/fs/internal/MonitoredReader} records the resources a + * task reads. + * + * The TaskRunner wraps the TaskUtil (or the spec-version interface) handed to a task in a + * MonitoredTaskUtil and passes the wrapper to the task instead. Every tracked read the task makes + * (an environment variable via getEnv, the quantized current time via + * getTime, isRootProject, getDependencies, or a + * project.* accessor on a getProject(name) result) is recorded. After the + * task finishes, the TaskRunner drains the recording via {@link #getInputRecording} and folds it into + * the task's build-cache signature so that a changed input invalidates the cached result. Reads made + * outside a task (by build orchestration code holding the raw TaskUtil) are not monitored and stay + * untracked. + * + * Wrapping is done with a Proxy so the monitor exposes exactly the same shape as the wrapped + * TaskUtil: a custom task's limited interface stays limited, and non-input members (tag mutations, + * registerCleanupTask, the resourceFactory, STANDARD_TAGS) + * pass straight through. The constructor returns the Proxy, so new MonitoredTaskUtil(taskUtil) + * yields a drop-in replacement that also answers {@link #getInputRecording}. + * + * Reads a task makes through a project reader are tracked too: a getProject(name).getReader() + * result is wrapped in a [MonitoredReader]{@link @ui5/fs/internal/MonitoredReader}, and the resources + * the task reads through it are recorded as resource requests. {@link #getResourceRequests} drains + * these, split into a project bucket (reads of the project being built) and a + * dependencies bucket (reads of any other project). The TaskRunner merges each bucket + * into the project and dependency resource requests it already collects from the workspace and + * dependencies readers. + * + * Reads through the project being built's getRootReader() are tracked in a third + * root bucket. The root reader exposes files outside the UI5 resource model (a + * tsconfig.json in the project root, third-party packages under node_modules), + * so a task reading them would otherwise bypass every tracked reader. The bucket is keyed by the + * reader's useGitignore flag, because the same glob returns a different resource set with + * the flag on versus off; a read recorded under one flag is re-materialized against a root reader + * built with the same flag. The default useGitignore: true bucket additionally tracks the + * root .gitignore as an input, so an edit that un-ignores a file invalidates the recorded + * request sets even though no filesystem change event fires for that file. node_modules + * and .git are ignored unless a request globs into them explicitly (see + * {@link augmentRootPattern}). A dependency's getRootReader() stays an unwrapped + * pass-through: root requests re-materialize against the built project's root, not a dependency's. + * + * @alias @ui5/project/build/helpers/MonitoredTaskUtil + */ +class MonitoredTaskUtil { + /** + * @param {@ui5/project/build/helpers/TaskUtil|object} taskUtil TaskUtil instance or a + * spec-version interface returned by {@link @ui5/project/build/helpers/TaskUtil#getInterface} + * @param {object} [parameters] + * @param {boolean} [parameters.recordTagOperations=false] Record every getTag, + * setTag and clearTag the wrapped task performs, drainable via + * {@link #getTagOperations}. Off for the task-level monitor (tags reach the tag collection and are + * captured through resource.getTags() like today); on for the per-step monitor the + * [StepRunner]{@link @ui5/project/build/helpers/StepRunner} wraps around this one, so a step + * restored from cache can replay its tag operations without re-running. + */ + constructor(taskUtil, {recordTagOperations = false} = {}) { + // Recorded inputs, keyed by `${type}\0${name}` so repeated reads of the same input collapse + // to a single entry (last read wins). + const recording = new Map(); + const record = (type, name, rawValue) => { + recording.set(`${type}\0${name}`, {type, name, value: normalizeInputValue(rawValue)}); + }; + + // Tag operations in call order, recorded only when recordTagOperations is set. Order is kept + // (rather than collapsed like inputs) so a replay reproduces the exact sequence a step performed, + // e.g. a setTag followed by a later clearTag of the same tag. + const tagOperations = []; + + // Monitored project readers, split by whether the read targets the project being built + // (project requests) or a dependency (dependency requests). Each entry is a MonitoredReader + // wrapping a getProject(name).getReader() result; getResourceRequests drains them. + const projectReaderMonitors = []; + const dependencyReaderMonitors = []; + + // Monitored root readers of the project being built, keyed by the useGitignore flag the read + // used. Two lists because the same glob resolves differently with the flag on versus off, so + // each recorded request must re-materialize against a matching root reader. + const rootReaderMonitors = {true: [], false: []}; + + // Name of the project being built, resolved lazily from the underlying taskUtil (getProject() + // with no argument) and cached. `null` when the wrapped interface has no getProject (spec + // version < 3.0), in which case no reader is ever wrapped. + let currentProjectName; + const getCurrentProjectName = () => { + if (currentProjectName === undefined) { + const current = typeof taskUtil.getProject === "function" ? taskUtil.getProject() : undefined; + currentProjectName = current ? current.getName() : null; + } + return currentProjectName; + }; + + // Concatenates the recorded requests of a set of MonitoredReaders into one {paths, patterns}. + const mergeResourceRequests = (monitors) => { + const paths = []; + const patterns = []; + for (const monitor of monitors) { + const requests = monitor.getResourceRequests(); + paths.push(...requests.paths); + patterns.push(...requests.patterns); + } + return {paths, patterns}; + }; + + // Drains the root reader monitors recorded under one useGitignore flag, applying the default + // node_modules/.git ignore to glob patterns. When useGitignore is on and the bucket recorded + // anything, the root .gitignore is tracked as a path input so its content joins the request + // set's signature (an edit that un-ignores a file changes what the recorded globs match). + const mergeRootRequests = (useGitignore) => { + const {paths, patterns} = mergeResourceRequests(rootReaderMonitors[useGitignore]); + const augmentedPatterns = patterns.map(augmentRootPattern); + if (useGitignore && (paths.length || augmentedPatterns.length) && !paths.includes("/.gitignore")) { + paths.push("/.gitignore"); + } + return {paths, patterns: augmentedPatterns}; + }; + + // Wraps a project (or interfaced project) returned by getProject so that reading a tracked + // accessor records the value under the project's name, and reading through getReader records + // the resources as resource requests. Methods bind to the underlying project so private fields + // keep working, and untracked methods pass straight through. + const wrapProject = (project) => { + const projectName = project.getName(); + const isCurrentProject = projectName === getCurrentProjectName(); + const readerMonitors = isCurrentProject ? + projectReaderMonitors : dependencyReaderMonitors; + return new Proxy(project, { + get(target, prop) { + const orig = target[prop]; + if (typeof orig !== "function") { + return orig; + } + if (TRACKED_PROJECT_METHODS.has(prop)) { + return function(...args) { + const result = orig.apply(target, args); + record(`project.${prop}`, projectName, result); + return result; + }; + } + if (prop === "getReader") { + return function(...args) { + const monitor = createMonitor(orig.apply(target, args)); + readerMonitors.push(monitor); + return monitor; + }; + } + if (prop === "getRootReader" && isCurrentProject) { + // Only the built project's root re-materializes correctly on lookup (against + // this.#project.getRootReader). A dependency's root reader passes through + // unwrapped, staying untracked as before. + return function({useGitignore = true} = {}) { + const monitor = createMonitor(orig.call(target, {useGitignore})); + rootReaderMonitors[!!useGitignore].push(monitor); + return monitor; + }; + } + return orig.bind(target); + }, + }); + }; + + return new Proxy(taskUtil, { + get(target, prop) { + if (prop === "getInputRecording") { + return () => Array.from(recording.values()); + } + if (prop === "getTagOperations") { + return () => tagOperations.slice(); + } + if (prop === "getResourceRequests") { + return () => ({ + project: mergeResourceRequests(projectReaderMonitors), + dependencies: mergeResourceRequests(dependencyReaderMonitors), + root: { + gitignore: mergeRootRequests(true), + noGitignore: mergeRootRequests(false), + }, + }); + } + const orig = target[prop]; + if (typeof orig !== "function") { + // STANDARD_TAGS, resourceFactory, or a member the interface does not provide + return orig; + } + if (recordTagOperations && (prop === "setTag" || prop === "clearTag" || prop === "getTag")) { + // Record the operation and delegate to the wrapped taskUtil, so a set/clear still + // reaches the project tag collection (captured by recordStageResult like a task-level + // tag) while the per-step attribution a restored step's replay needs is kept. The path + // stands in for the resource, since the tag collection keys tags by path and a restored + // step has no resource instance to hand back. + return function(resource, tag, value) { + const result = orig.call(target, resource, tag, value); + if (prop === "setTag") { + tagOperations.push({op: "set", path: resource.getPath(), tag, + value: value === undefined ? true : value}); + } else if (prop === "clearTag") { + tagOperations.push({op: "clear", path: resource.getPath(), tag}); + } else { + tagOperations.push({op: "get", path: resource.getPath(), tag}); + } + return result; + }; + } + if (Object.hasOwn(TRACKED_TASK_UTIL_METHODS, prop)) { + const type = TRACKED_TASK_UTIL_METHODS[prop]; + return function(name) { + const value = orig.call(target, name); + record(type, NAME_ARG_METHODS.has(prop) ? name : "", value); + return value; + }; + } + if (prop === "getDependencies") { + return function(projectName) { + const value = orig.call(target, projectName); + // getDependencies defaults to the project being built. Record the resolved name + // so the lookup re-derives the same input. + const resolvedName = projectName ?? target.getProject().getName(); + record("getDependencies", resolvedName, value); + return value; + }; + } + if (prop === "getProject") { + return function(nameOrResource) { + const project = orig.call(target, nameOrResource); + return project ? wrapProject(project) : project; + }; + } + return orig.bind(target); + }, + }); + } + + /** + * Returns the inputs recorded since this monitor was created. + * + * Called by the TaskRunner after the task finishes; folded into the task's build-cache signature. + * + * @returns {Array<{type: string, name: string, value: string|undefined}>} Recorded input entries + */ + getInputRecording() { + // Implemented via the constructor's Proxy trap; this declaration documents the contract. + return []; + } + + /** + * Returns the tag operations recorded since this monitor was created, in call order. Empty unless + * the monitor was constructed with recordTagOperations (the per-step monitor). + * + * The [StepRunner]{@link @ui5/project/build/helpers/StepRunner} persists these per step so a + * step restored from cache on a delta build replays its set/clear operations + * into the tag collection, reproducing tags the step would have set had it run. + * + * @returns {Array<{op: string, path: string, tag: string, value: *}>} Recorded tag operations + * (op is "set", "clear" or "get"; + * value is present only for "set") + */ + getTagOperations() { + // Implemented via the constructor's Proxy trap; this declaration documents the contract. + return []; + } + + /** + * Returns the resource requests recorded through project readers since this monitor was created. + * + * Called by the TaskRunner after the task finishes. The project and + * dependencies buckets are merged into the project and dependency resource requests + * the TaskRunner already collects from the workspace and dependencies readers. The root + * bucket carries reads through the built project's root reader, keyed by useGitignore, + * for the build cache to re-materialize against a matching root reader. + * + * @returns {{project: {paths: string[], patterns: Array}, + * dependencies: {paths: string[], patterns: Array}, + * root: {gitignore: {paths: string[], patterns: Array}, + * noGitignore: {paths: string[], patterns: Array}}}} Recorded resource requests + */ + getResourceRequests() { + // Implemented via the constructor's Proxy trap; this declaration documents the contract. + return { + project: {paths: [], patterns: []}, + dependencies: {paths: [], patterns: []}, + root: {gitignore: {paths: [], patterns: []}, noGitignore: {paths: [], patterns: []}}, + }; + } +} + +export default MonitoredTaskUtil; diff --git a/packages/project/lib/build/helpers/ProjectBuildContext.js b/packages/project/lib/build/helpers/ProjectBuildContext.js index 641163a8be2..a6a02ab2e08 100644 --- a/packages/project/lib/build/helpers/ProjectBuildContext.js +++ b/packages/project/lib/build/helpers/ProjectBuildContext.js @@ -5,6 +5,8 @@ import TaskRunner from "../TaskRunner.js"; import TaskDefinitions from "../TaskDefinitions.js"; import {getProjectSignature} from "./getBuildSignature.js"; import ProjectBuildCache from "../cache/ProjectBuildCache.js"; +import {normalizeInputValue} from "../cache/index/TaskInputSet.js"; +import {quantizeTime, TIME_GRANULARITIES} from "./quantizeTime.js"; /** * Build context of a single project. Always part of an overall @@ -82,7 +84,8 @@ class ProjectBuildContext { baseSignature, taskSignatures, project, buildContext.getGraph(), buildContext.getTaskRepository()); const cacheMode = buildContext.getBuildConfig().cache; - ctx._buildCache = new ProjectBuildCache(project, ctx._buildSignature, cacheManager, cacheMode); + ctx._buildCache = new ProjectBuildCache(project, ctx._buildSignature, cacheManager, cacheMode, + (type, name) => ctx.resolveInputValue(type, name)); return ctx; } @@ -129,6 +132,50 @@ class ProjectBuildContext { this._queues.cleanup.push(callback); } + /** + * Re-derives the current normalized value of a recorded task input. + * + * The counterpart to the recording done by + * [MonitoredTaskUtil]{@link @ui5/project/build/helpers/MonitoredTaskUtil}: on a cache lookup the + * build cache re-evaluates each input a task recorded last time against the current environment + * and project graph. A value that differs from the one baked into the cached stage signature + * misses the cache and forces the task to re-run. The recording and lookup sides run values + * through the same {@link normalizeInputValue}, so equal values compare equal. + * + * @param {string} type Input type (e.g. "env", "time", "isRootProject", "getDependencies", + * "project.*") + * @param {string} name Input name within the type + * @returns {string|undefined} Current normalized value, or undefined for an unknown + * type or an input that can no longer be resolved (e.g. a project removed from the graph) + */ + resolveInputValue(type, name) { + let rawValue; + if (type === "env") { + rawValue = process.env[name]; + } else if (type === "isRootProject") { + rawValue = this.isRootProject(); + } else if (type === "getDependencies") { + rawValue = this.getDependencies(name); + } else if (type === "time") { + // Re-derive the current bucket for the recorded granularity: the same bucket hits the + // cache, a rolled-over bucket misses and re-runs the task. A recorded granularity is always + // valid (getTime throws on an unknown one at record time); guard anyway so a corrupt cache + // row misses the cache instead of throwing during lookup. + if (!TIME_GRANULARITIES.includes(name)) { + return undefined; + } + rawValue = quantizeTime(name, this.getBuildTime()); + } else if (type.startsWith("project.")) { + const method = type.slice("project.".length); + const project = this.getProject(name); + rawValue = project && typeof project[method] === "function" ? project[method]() : undefined; + } else { + // Unknown input type: cannot re-derive a value + return undefined; + } + return normalizeInputValue(rawValue); + } + /** * Executes all registered cleanup tasks * @@ -171,6 +218,20 @@ class ProjectBuildContext { return this._buildContext.getGraph().getDependencies(projectName || this._project.getName()); } + /** + * Returns the timestamp of the current build run, used for all time quantization. + * + * Delegates to the overall [BuildContext]{@link @ui5/project/build/helpers/BuildContext}, which + * fixes one timestamp per run. Read live rather than snapshot: this context and its + * [TaskUtil]{@link @ui5/project/build/helpers/TaskUtil} outlive individual runs, while the + * timestamp is refreshed per run. + * + * @returns {Date} The current build run's timestamp + */ + getBuildTime() { + return this._buildContext.getBuildTime(); + } + /** * Gets the list of required dependencies for the current project * diff --git a/packages/project/lib/build/helpers/StepRunner.js b/packages/project/lib/build/helpers/StepRunner.js new file mode 100644 index 00000000000..72d97c9738c --- /dev/null +++ b/packages/project/lib/build/helpers/StepRunner.js @@ -0,0 +1,1173 @@ +import crypto from "node:crypto"; +import AbstractReader from "@ui5/fs/AbstractReader"; +import AbstractReaderWriter from "@ui5/fs/AbstractReaderWriter"; +import {assertDistinctWrite, flushWriteBuffer} from "@ui5/fs/internal/stepWriteBuffer"; +import {getLogger} from "@ui5/logger"; +import MonitoredTaskUtil from "./MonitoredTaskUtil.js"; + +const log = getLogger("build:helpers:StepRunner"); + +// Key identity of a scalar step's single implicit unit. A scalar step is a one-key group, so its +// invocation data has exactly one entry under this fixed id. +const SCALAR_KEY_ID = "scalar:0"; + +/** + * A step may return a resource or an array of resources. A resource is anything carrying the two + * identity accessors the key logic already relies on, so the check stays consistent with {@link #keyId}. + * + * @param {*} value Candidate return value + * @returns {boolean} true if the value is a resource + */ +function isResource(value) { + return !!value && typeof value.getPath === "function" && typeof value.getIntegrity === "function"; +} + +/** + * Names a rejected return value in an error message without assuming it is serializable. + * + * @param {*} value Rejected value + * @returns {string} A short human-readable type description + */ +export function describeValue(value) { + if (value === null) { + return "null"; + } + if (value === undefined) { + return "undefined"; + } + if (typeof value === "object") { + const name = value.constructor?.name; + return name && name !== "Object" ? `a ${name}` : "a plain object"; + } + return `a ${typeof value}`; +} + +/** + * Validates a step-based task's complete step list in one pass, before any step runs. The + * [TaskRunner]{@link @ui5/project/build/TaskRunner} calls this at discovery, before it derives the step + * names and creates one pipeline stage per step: a malformed declaration (a bad shape, a duplicate name, an + * invalid needs) must be rejected before it can create a corrupt stage (two steps sharing a + * name would create two stages with the same stage id). {@link StepRunner#runSteps} calls it again at its + * entry, so a directly or standalone instantiated runner validates the same way. + * + * Every rule is static over the step descriptors (no reader, taskUtil, options or cache state is consulted), + * so one upfront pass can replace the former per-step check that ran interleaved with execution. The + * "earlier steps" set a step's needs may reference is reconstructed by iterating in declaration + * order; because a step may reference only an earlier step, a forward or self reference throws, which is also + * what makes a needs cycle impossible. + * + * @param {*} steps The value the task factory returned + * @param {object} [parameters] + * @param {string} [parameters.taskName] The task name, named in the array- and element-shape messages when + * the caller is the TaskRunner (absent for a standalone runner) + * @throws {Error} If the step list or any step declaration is invalid + */ +export function validateSteps(steps, {taskName} = {}) { + const where = taskName ? ` for task '${taskName}'` : ""; + if (!Array.isArray(steps)) { + throw new Error( + `Step factory${where} must return an array of step objects, got ${describeValue(steps)}`); + } + const seen = new Set(); + for (let i = 0; i < steps.length; i++) { + const step = steps[i]; + if (!step || typeof step !== "object") { + throw new Error(`Step at index ${i}${where} must be an object, got ${describeValue(step)}`); + } + if (typeof step.name !== "string" || !step.name) { + throw new Error(`Step at index ${i}${where} must have a non-empty string 'name'`); + } + if (seen.has(step.name)) { + throw new Error(`Duplicate step name '${step.name}'${where}`); + } + const isScalar = typeof step.run === "function"; + const hasKeys = typeof step.keys === "function"; + const hasEach = typeof step.each === "function"; + const isMap = hasKeys && hasEach; + if (!isScalar && (hasKeys !== hasEach)) { + // A half-defined map step is the common authoring typo; name the missing half rather than the + // generic scalar-or-map message. + throw new Error( + `Map step '${step.name}' must define both 'keys' and 'each' functions`); + } + if (isScalar === isMap) { + throw new Error( + `Step '${step.name}' must be either a scalar step ({name, run}) or a ` + + `map step ({name, keys, each})`); + } + if (step.needs !== undefined) { + if (!Array.isArray(step.needs)) { + throw new Error( + `Step '${step.name}' 'needs' must be an array of earlier step names, ` + + `got ${describeValue(step.needs)}`); + } + for (let j = 0; j < step.needs.length; j++) { + const needed = step.needs[j]; + if (typeof needed !== "string") { + throw new Error( + `Step '${step.name}' 'needs' entries must be strings; ` + + `entry ${j} is ${describeValue(needed)}`); + } + if (!seen.has(needed)) { + throw new Error( + `Step '${step.name}' needs '${needed}', which is not an earlier step`); + } + } + } + seen.add(step.name); + } +} + +/** + * Collects the reads and writes of a single step (a scalar step's implicit unit, or one key of a map + * step). Project reads (workspace) and dependency reads are kept apart so they can be folded back into + * the task's project vs. dependency request graph independently: a dependency path folded into the + * project graph would resolve against the wrong reader and corrupt the signature. + */ +class StepRecorder { + projectReads = new Set(); + dependencyReads = new Set(); + // Glob patterns a unit issued, kept so a cached key's glob stays in the stage request set: a stored + // pattern is re-executed against the current reader on lookup, which is how a newly matching file moves + // the stage signature. Without it, a map-step key served from cache would drop its globs (its each body + // does not re-run), so an added match would not re-run the stage. Project and dependency patterns are + // kept apart like the paths, so each is re-executed against its own reader. + projectPatterns = new Set(); + dependencyPatterns = new Set(); + writes = new Set(); +} + +/** + * Records the reads a step makes against the dependencies reader. Every read is delegated to the + * task-level (already monitored) reader, so the task's overall requests are still captured once via + * that monitor; this wrapper additionally attributes the read to the current step. + */ +class RecordingReader extends AbstractReader { + #reader; + #recorder; + + constructor(reader, recorder) { + super(reader.getName()); + this.#reader = reader; + this.#recorder = recorder; + } + + async _byGlob(virPattern, options) { + const resources = await this.#reader.byGlob(virPattern, options); + this.#recorder.dependencyPatterns.add(virPattern); + for (const resource of resources) { + this.#recorder.dependencyReads.add(resource.getPath()); + } + return resources; + } + + async _byPath(virPath, options) { + // Record the probed path verbatim, even when it resolves to nothing: probing an absent path + // (e.g. a theme's not-yet-existing library marker) is an input, so a later creation of that + // path must re-run this step on a delta build. + this.#recorder.dependencyReads.add(virPath); + return this.#reader.byPath(virPath, options); + } +} + +/** + * Records the reads and writes a step makes against the workspace. Reads are attributed like + * {@link RecordingReader}. Writes are attributed so a delta re-run can drop outputs a step no longer + * produces, and are either persisted immediately (sequential mode) or buffered for an ordered flush + * (concurrent mode). + */ +class RecordingReaderWriter extends AbstractReaderWriter { + #workspace; + #recorder; + #writeBuffer; + #stepIndex; + + /** + * @param {@ui5/fs/AbstractReaderWriter} workspace Task-level monitored workspace + * @param {StepRecorder} recorder Recorder for this step + * @param {Map|null} writeBuffer Shared buffer for concurrent writes, or + * null to write through immediately (sequential mode) + * @param {number} stepIndex Position of this unit in the key order, used to flush buffered writes + * deterministically and to detect two units writing the same path + */ + constructor(workspace, recorder, writeBuffer, stepIndex) { + super(workspace.getName()); + this.#workspace = workspace; + this.#recorder = recorder; + this.#writeBuffer = writeBuffer; + this.#stepIndex = stepIndex; + } + + async _byGlob(virPattern, options) { + const resources = await this.#workspace.byGlob(virPattern, options); + this.#recorder.projectPatterns.add(virPattern); + for (const resource of resources) { + this.#recorder.projectReads.add(resource.getPath()); + } + return resources; + } + + async _byPath(virPath, options) { + this.#recorder.projectReads.add(virPath); + return this.#workspace.byPath(virPath, options); + } + + // Overrides _write rather than the public write (unlike @ui5/builder's runSteps BufferedWriter) because + // this writer also records every write for the build cache, and AbstractReaderWriter.write has already + // defaulted options by the time it delegates here. The buffered entry therefore stores the defaulted + // options as the single replay argument, which flushWriteBuffer replays via write(resource, ...args). + // The same-path guard, its error message and the key-order flush are shared with runSteps through + // @ui5/fs/internal/stepWriteBuffer so the two runners cannot drift. + async _write(resource, options) { + const resourcePath = resource.getPath(); + this.#recorder.writes.add(resourcePath); + if (this.#writeBuffer) { + // Concurrent mode: buffer and flush in key order once all keys finish. Concurrent keys are + // required to be independent, so two keys writing the same path is a contract violation + // rather than a last-wins race. + assertDistinctWrite(this.#writeBuffer, resourcePath, this.#stepIndex); + this.#writeBuffer.set(resourcePath, {resource, args: [options], index: this.#stepIndex}); + return; + } + // Sequential mode: persist immediately so a later key reads what this key wrote. + return this.#workspace.write(resource, options); + } +} + +/** + * Per-task driver behind the step-factory build API. A step-based task default-exports a factory + * build(options) => Step[]; the [TaskRunner]{@link @ui5/project/build/TaskRunner} calls the + * factory, hands the resulting ordered step list to this driver, and folds the driver's outcome into the + * task's build-cache entry. + * + * Two step shapes are supported: + *
    + *
  • scalar: {name, needs?, run} where + * run: async ({needs, workspace, dependencies, taskUtil, options}) => value? runs once
  • + *
  • map: {name, needs?, sequential?, keys, each} where keys enumerates the + * key set and each runs once per key
  • + *
+ * + * Write contract: every write a unit makes must go through its own workspace (the recording + * reader/writer passed in the unit's context). A write through any other handle (a stage workspace captured + * in a closure, a writer reached some other way) is not attributed to the unit, so a delta build cannot drop + * that output when the unit stops producing it: {@link #computeStaleOutputs} derives stale outputs from the + * unit's recorded writes, so an unrecorded write leaves a stale file behind. The step API hands + * a unit no writable handle other than its workspace (the factory receives options + * only, and taskUtil.getProject().getReader() is read-only), so honoring this is the default; a + * custom step that reaches a writer another way breaks the stale-output guarantee. + * + * Steps run in array order via {@link #runSteps}. A scalar step is a one-key group; a map step is a + * multi-key group. Each unit runs against per-step recording readers and a per-step + * [MonitoredTaskUtil]{@link @ui5/project/build/helpers/MonitoredTaskUtil} that record what it reads, + * writes, returns, reads as a non-resource input, and tags. The recording lets a delta build re-run only + * the units whose observed inputs changed and drop the outputs of units that no longer produce them, + * without any delta bookkeeping in the task itself. A unit whose recorded non-resource input (an env var, + * a dependency version) no longer resolves to its stored value re-runs; a unit whose consumed + * needs return changed re-runs; a unit served from cache replays its recorded tag operations + * so its tags reappear this build. + * + * A step lists earlier step names in needs, and those steps' returns arrive as + * needs.<name>. A step may return resources (stored in the CAS by integrity, rebuilt on a + * cache hit) or a JSON-serializable value (persisted inline with the unit's invocation data). Either is + * injected into a consumer via needs, and the return's signature folds into the consumer's + * per-unit selection so a changed producer return re-runs the consumer. + * + * A key is identified by content and identity: a resource key by its path and a content discriminator (the + * path distinguishes resources that share content but produce different output, the discriminator makes a + * content change a new key that cannot yield a stale hit), a string key by its value. The discriminator is + * tiered like isResourceUnchanged (lastModified + size when + * statically available, SSRI integrity otherwise); see {@link #keyId}. A compound key is the caller's + * responsibility to express as a stable string. + * + * @private + */ +export default class StepRunner { + #steps; + #options; + #prepareStage; + #reopenStage; + #recordStage; + #createStageContext; + #getPreviousInvocationData; + #returnValueStore; + #resolveInputValue; + #applyTagOperations; + #notifyStepExecution; + #signal; + + // Each step's return value and return signature, filled as steps run so a later step's needs can pull + // them. Only populated on the factory (runSteps) path. + #returns = new Map(); + #returnSignatures = new Map(); + + /** + * The StepRunner drives one pipeline stage per step: before a step it calls + * prepareStage(step) (which switches the project to the step's own stage and returns the + * stage's cache verdict), runs or restores the step against a fresh per-stage context, then calls + * recordStage(step, ...) to record that stage. A map step's single stage still carries an + * internal per-key delta; a scalar step is a one-key stage. There is no cross-step fold: each step's + * stage records only its own reads, inputs, and stale outputs. + * + * @param {object} parameters + * @param {object[]} [parameters.steps] Ordered step list from the task factory (factory path) + * @param {object} [parameters.options] Task options, passed through to each step's context + * @param {function(string): Promise<(object|boolean)>} [parameters.prepareStage] Switches the project to + * the named step's stage and returns its cache verdict: true (fully cached, do not run), + * an object (delta cacheInfo for the map step's internal key-delta), or a falsy value (run every unit). + * Absent for standalone use (no cache): every unit runs. This verdict is read three ways in + * {@link #runSteps}: compared against true for the full-hit short-circuit, passed to + * markStageExecuting as a "was this a delta" flag, and treated as "run everything if + * falsy" by {@link #selectStepsToRun}. An explicit verdict shape would read more clearly, but it is + * deferred: the shape is shared with the stage-signature code and reshaping it belongs with that work. + * @param {function(string): Promise<(object|boolean)>} [parameters.reopenStage] Reopens the named step's + * stage with a fresh live writer after a full cache hit that must be re-run (a consumed + * needs return changed), and returns the cache verdict to run it under (deliberately a + * falsy value for a full re-run, not a delta; see {@link #runSteps} for why pruning is not attempted + * here). The full-hit restore had installed a read-only cached stage; re-running needs a writable one. + * Absent for standalone use, where a full hit never occurs. + * @param {function(string, object): Promise} [parameters.recordStage] Records the named step's + * stage from the run outcome {projectRequests, dependencyRequests, inputRecording, + * rootRequests, cacheInfo, invocationData, staleOutputs}. Absent for standalone use. + * @param {function(): {workspace, dependencies, taskUtil, monitoredTaskUtil, getResourceRequests, + * getInputRecording}} parameters.createStageContext Returns a fresh per-stage context bound to the + * stage the last prepareStage switched to: the monitored workspace/dependencies readers, + * the pass-through taskUtil the step units wrap per key, and drains for the monitored + * task-level requests and inputs. Called once per step that runs. + * @param {function(string): (Map|undefined)} [parameters.getPreviousInvocationData] + * Returns the named step's stage's previous per-key invocation data, or undefined on a first build. + * @param {object} [parameters.returnValueStore] CAS-backed store for resource return values, with + * store(resources) (persist content, return path-aligned descriptors) and + * restore(descriptor) (rebuild a resource from a descriptor). Absent for standalone + * use without a build cache: resource returns are then handed back for the current build but not + * persisted, so a later delta build cannot restore a unit that is served from cache. Serializable + * returns persist inline and need no store. + * @param {function(string, string): (string|undefined)} [parameters.resolveInputValue] Re-derives the + * current normalized value of a recorded non-resource input (env var, time bucket, dependency + * version, ...), the same resolver the task-level input lookup uses. A cached unit whose recorded + * input no longer resolves to its stored value is re-run. Absent for standalone use, where no input + * can be re-resolved and a unit is selected on its resource reads alone. + * @param {function(Array): void} [parameters.applyTagOperations] Replays a restored unit's + * recorded tag operations into the project tag collection, so a unit served from cache contributes + * the same tags it would have set had it run. Absent for standalone use, where tags are not persisted. + * @param {function(boolean): void} [parameters.notifyStepExecution] Called once, before the first step + * that actually executes runs anything, with whether that step's stage carries a delta verdict. The + * TaskRunner reports the task started from it, so task-start precedes the work it + * announces. Not called when every step is served from cache (the task is reported skipped instead). + * @param {AbortSignal} [parameters.signal] Build abort signal, checked between units + */ + constructor({ + steps, options, prepareStage, recordStage, createStageContext, getPreviousInvocationData, + returnValueStore, resolveInputValue, applyTagOperations, notifyStepExecution, signal, reopenStage + }) { + this.#steps = steps; + this.#options = options; + this.#prepareStage = prepareStage; + this.#reopenStage = reopenStage; + this.#recordStage = recordStage; + this.#createStageContext = createStageContext; + this.#getPreviousInvocationData = getPreviousInvocationData; + this.#returnValueStore = returnValueStore; + this.#resolveInputValue = resolveInputValue; + this.#applyTagOperations = applyTagOperations; + this.#notifyStepExecution = notifyStepExecution; + this.#signal = signal; + } + + /** + * Runs the step list in array order, caching each unit's result. + * + * A scalar step runs its run once; a map step enumerates keys via keys then + * runs each per key. Before a step runs, a needs object is assembled from the + * returns of the earlier steps it names and passed in the step context. Each step's return is recorded + * so later steps can consume it. + * + * @returns {Promise} + */ + async runSteps() { + validateSteps(this.#steps); + let anyStepExecuted = false; + const writtenResourcePaths = []; + + // Marks the task as executing. Called from the one place a stage stops being a pure cache hit, so + // the "task started" report and the anyStepExecuted verdict cannot drift apart: the TaskRunner + // emits task-start from the notification, before the stage does any work. Only the first executing + // stage notifies, since the later stages of one task are not separate executions to report. + const markStageExecuting = (cacheInfo) => { + if (!anyStepExecuted) { + this.#notifyStepExecution?.(!!cacheInfo); + } + anyStepExecuted = true; + }; + + // A step-based task with no steps (e.g. replaceCopyright with no copyright configured) still has a + // single stage that must participate in caching: prepare it, and if it is not a cache hit, record an + // empty result so the empty stage caches and a later build reports it as skipped. Its stage id is + // task/{taskName} (stepName undefined), matching the single stage setTasks created for it. + if (this.#steps.length === 0) { + if (!this.#prepareStage) { + markStageExecuting(false); + return {anyStepExecuted: true, writtenResourcePaths}; + } + const cacheInfo = await this.#prepareStage(undefined); + if (cacheInfo === true) { + return {anyStepExecuted: false, writtenResourcePaths}; + } + markStageExecuting(cacheInfo); + const ctx = this.#createStageContext(); + if (this.#recordStage) { + await this.#recordStage(undefined, {ctx, cacheInfo, invocationData: new Map(), staleOutputs: []}); + } + return {anyStepExecuted: true, writtenResourcePaths}; + } + + for (const step of this.#steps) { + this.#signal?.throwIfAborted(); + + // Switch the project to this step's own stage and get its cache verdict. Standalone use (no + // prepareStage) always runs every unit. + let cacheInfo = this.#prepareStage ? await this.#prepareStage(step.name) : false; + const previous = this.#getPreviousInvocationData ? + this.#getPreviousInvocationData(step.name) : undefined; + + const needs = this.#buildNeeds(step.needs); + const needsSignatures = this.#collectNeedsSignatures(step.needs); + + const isScalar = typeof step.run === "function"; + + if (cacheInfo === true) { + // Full stage-cache hit: the step's own stage signature matched. A consumed needs return is + // deliberately excluded from the stage signature (see #foldStageKeys), so a full hit can + // occur even though a producer this step needs re-ran this build with a changed return. The + // delta path catches that via needsInputs in #selectStepsToRun, but a full hit never runs + // #selectStepsToRun. Check it here: when a consumed return changed, reopen the stage with a + // live writer and re-run it rather than serving stale cached output. + if (this.#canRestoreCachedStage(step, previous, isScalar) && + !this.#needsReturnChanged(previous, needsSignatures)) { + // Fully cached stage: the step does not run. Rebuild its return from the persisted + // per-key invocation data (in key order) so later steps' needs still resolve, and replay + // each key's tag operations so its tags reappear this build (the stage writer was already + // restored). + const {results, invocationData, entries} = + this.#restoreCachedStage(previous, isScalar); + this.#returns.set(step.name, isScalar ? results[0] : results); + this.#returnSignatures.set(step.name, + this.#computeStepReturnSignature(invocationData, entries, isScalar)); + continue; + } + log.verbose( + `step '${step.name}': re-running despite a full stage-cache hit ` + + `(a consumed needs return changed, or the stage's invocation data cannot be restored)`); + // Reopen the stage (fresh writer) and re-run it as a full execution: the hook returns a + // falsy verdict, so #selectStepsToRun runs every unit rather than pruning. Reopening installs + // a fresh EMPTY writable stage (unlike a delta verdict, whose stage was pre-seeded with the + // cached outputs at import), so the units cannot be pruned without first re-seeding that + // stage; re-running the whole stage keeps the output complete without that machinery. No + // shipped multi-step task is penalized: generateThemeDesignerResources is the only task + // wiring needs, and a changed scan return means its themes must regenerate anyway. The cost + // of a full re-run here is a large map consumer gated behind a trivially-changed producer + // return, which no shipped task has. Standalone use has no hook and no full hit. + cacheInfo = this.#reopenStage ? await this.#reopenStage(step.name) : false; + } + + // A step past the fully-cached short-circuit executes its stage (fresh recording), even if it + // enumerates zero units this build (an empty map step). This is "the task ran" for reporting, + // distinct from a stage served entirely from cache. + markStageExecuting(cacheInfo); + + // Fresh per-stage context bound to the stage prepareStage just switched to. Created before key + // enumeration so the keys() enumerator's reads are captured by the stage's monitored readers. + const ctx = this.#createStageContext(); + + let entries; + let callback; + let options; + if (isScalar) { + // A scalar step is a single implicit unit; writes persist immediately (sequential) so the + // step reads back its own writes and later steps see them. + entries = [{key: undefined, index: 0, keyId: SCALAR_KEY_ID}]; + callback = (key, unitCtx) => step.run(unitCtx); + options = {sequential: true}; + } else { + entries = await this.#enumerateKeys(step, needs, ctx); + callback = (key, unitCtx) => step.each(key, unitCtx); + options = step.sequential ? {sequential: true} : undefined; + } + + const {results, invocationData, freshInvocationData} = await this.#runGroup( + step.name, entries, options, callback, + {needs, needsSignatures, cacheInfo, previous, ctx, isScalar}); + + this.#returns.set(step.name, isScalar ? results[0] : results); + this.#returnSignatures.set(step.name, + this.#computeStepReturnSignature(invocationData, entries, isScalar)); + + // Record this step's stage. There is no cross-step fold, but the stage still folds its + // own keys' reads and inputs — including keys restored from cache on a delta build, whose reads + // and inputs the stage-level monitor never saw — so the stage re-keys on its complete input set. + if (this.#recordStage) { + const staleOutputs = this.#computeStaleOutputs( + previous, invocationData, entries, freshInvocationData); + const {reads, inputs} = this.#foldStageKeys(invocationData); + const stageWritten = await this.#recordStage(step.name, { + ctx, cacheInfo, invocationData, staleOutputs, foldedReads: reads, foldedInputs: inputs, + }); + if (stageWritten) { + writtenResourcePaths.push(...stageWritten); + } + } + } + return {anyStepExecuted, writtenResourcePaths}; + } + + /** + * Whether a full stage-cache hit can be faithfully reconstructed from its persisted per-key invocation + * data. A full hit serves the stage from {@link #restoreCachedStage} without running it, so the per-key + * sidecar must describe the stage. It can fail to, because the stage result and its sidecar are two + * independent rows with independent write conditions (a missing sidecar after discardIncrementalState, + * or a drop-to-zero map). When the sidecar does not describe the stage, re-run it rather than serving a + * reconstruction that is wrong: + * + * - previous === undefined: no sidecar at all. A scalar producer would restore an + * undefined return that crashes a consumer dereferencing it through needs. + * - A scalar step whose previous.size !== 1: a scalar step records exactly its one implicit + * unit, so any other count means the sidecar does not match. + * - A needs-declaring step whose previous.size === 0: {@link #needsReturnChanged} + * has no recorded needsInputs to compare against, so it cannot observe that a consumed + * producer return changed (the zero-key map step gated behind a needs flag). Re-running + * re-enumerates keys() against the current needs. + * + * An empty map step that declares no needs stays restorable, so its cache is preserved. + * + * @param {object} step The step descriptor + * @param {Map|undefined} previous The stage's previous per-key invocation data + * @param {boolean} isScalar Whether the step is scalar + * @returns {boolean} true if the stage can be served from cache + */ + #canRestoreCachedStage(step, previous, isScalar) { + if (previous === undefined) { + return false; + } + if (isScalar) { + return previous.size === 1; + } + if (step.needs?.length && previous.size === 0) { + return false; + } + return true; + } + + /** + * Whether any of a fully-cached stage's keys consumed a needs return whose current + * signature differs from the value it recorded on its previous run. Mirrors the needs check in + * {@link #selectStepsToRun}, applied to the full-hit path where that selection never runs. A stage with + * no previous data, no needs, or unchanged returns reports false, so the + * full-hit fast path is preserved for the common case. + * + * @param {Map|undefined} previous The stage's previous per-key invocation data + * @param {Map} [needsSignatures] Current producer return signatures by producer name + * @returns {boolean} true if a consumed return changed + */ + #needsReturnChanged(previous, needsSignatures) { + if (!previous || !needsSignatures || needsSignatures.size === 0) { + return false; + } + for (const data of previous.values()) { + if (data.needsInputs?.some( + (needed) => needsSignatures.get(needed.name) !== needed.value)) { + return true; + } + } + return false; + } + + /** + * Rebuilds a fully-cached stage's per-key results without running the step: each key's return is + * restored from its persisted descriptor (in key order), so later steps' needs resolve. + * + * Tags are not replayed here. On a full hit the stage was installed by + * ProjectBuildCache.prepareStageExecutionAndValidateCache via + * ProjectResources.setStage, carrying the stage's complete cached tag operations (the full + * set captured at record time, including the keys enumerator's tags, which belong to no + * key). Those reach the live tag collection through #applyCachedResourceTags when a later + * stage or the result stage reads over this one. A per-key replay would re-apply a strict subset of the + * same operations for nothing. The delta path (#runGroup) does replay, because there the + * stage re-runs and is re-recorded, so a restored key's tags must land in the monitored collection to be + * captured and persisted again. + * + * @param {Map|undefined} previous The stage's previous per-key invocation data + * @param {boolean} isScalar Whether the step is scalar + * @returns {{results: Array, invocationData: Map, entries: Array<{keyId: string}>}} + */ + #restoreCachedStage(previous, isScalar) { + const invocationData = previous ?? new Map(); + const entries = [...invocationData.keys()].map((keyId, index) => ({keyId, index})); + const results = new Array(entries.length); + for (const {keyId, index} of entries) { + const prev = invocationData.get(keyId); + results[index] = this.#restoreReturn(prev?.returns); + } + return {results, invocationData, entries}; + } + + /** + * Output paths a stage produced on a previous build but no longer produces (a re-run unit that writes + * fewer paths, or a key gone this build), so they can be dropped from the carried-forward stage. Scoped + * to this one stage: a stage owns its outputs, so a path it stops producing is stale for it. + * + * A path is only dropped when nothing in the stage claims it this build, so the writes of every key + * present this build are subtracted, including those of a key served from cache (which still owns the + * paths it wrote on an earlier build). The re-run comparison, in contrast, runs over the units that + * actually executed: a cached unit's entry is carried over from previous unchanged, so + * comparing it against itself could never report a dropped path anyway. + * + * This relies on each unit's recorded writes being a complete record of what it produced, + * which holds only while units honor the write contract (see the class description): a write that went + * through a handle other than the unit's workspace is absent from writes, so + * this derivation cannot drop it and the stale output survives. + * + * @param {Map|undefined} previous The stage's previous per-key invocation data + * @param {Map} invocationData The stage's complete per-key invocation data this build + * (re-run units merged over the carried-over cached ones) + * @param {Array<{keyId: string}>} entries The stage's key entries this build + * @param {Map} freshInvocationData The per-key data of the units that ran this build + * @returns {string[]} Paths to drop + */ + #computeStaleOutputs(previous, invocationData, entries, freshInvocationData) { + if (!previous) { + return []; + } + const currentWrites = new Set(); + for (const data of invocationData.values()) { + data.writes.forEach((path) => currentWrites.add(path)); + } + const currentKeyIds = new Set(entries.map((entry) => entry.keyId)); + const stale = new Set(); + for (const [keyId, prev] of previous) { + if (!currentKeyIds.has(keyId)) { + // Key gone this build: every path it owned is stale. + prev.writes.forEach((path) => stale.add(path)); + continue; + } + const reRun = freshInvocationData.get(keyId); + if (reRun) { + // Re-run unit: any path it owned but did not re-write is stale. + prev.writes.forEach((path) => { + if (!reRun.writes.includes(path)) { + stale.add(path); + } + }); + } + } + for (const path of currentWrites) { + stale.delete(path); + } + return [...stale]; + } + + /** + * Folds a stage's every key's reads and non-resource inputs into one request set and one input set, + * from the stage's complete per-key invocation data. This includes keys served from cache on a delta + * build (whose reads and inputs the stage-level monitor never observed), so the stage re-keys on its + * full input set and a first-seen or cached-key input stays tracked. This is the map step's internal + * key-delta fold (kept for the map step); it does NOT fold across steps. + * + * The needs returns a key consumes are deliberately excluded (tracked separately in + * needsInputs for per-key selection only): they are re-derived from producer reads/inputs + * that are themselves tracked, so folding one into the stage signature would permanently miss the cache. + * + * Because a consumer stage's signature omits the producer return, a cached consumer's correctness rests + * on two checks instead of the signature: {@link #needsReturnChanged} on the full-hit path and the + * needsInputs comparison in {@link #selectStepsToRun} on the delta path. These are the only + * two paths that reach a cached stage through the StepRunner. A whole project restored from the + * project-level result cache never runs the StepRunner at all (and ProjectBuildCache.#importStages + * installs those stages blindly by signature), yet that is safe without a needs check: the result + * signature aggregates every stage's inputs, the producer stage included (project source, dependency + * reads, non-resource inputs and root reads across all stages), so any change that could alter a + * producer's return perturbs the result signature, misses the result cache, and defers to the StepRunner + * where these two checks run. The gap the needs checks close is specific to a single consumer stage whose + * own signature omits the producer; aggregation closes it at the project level. + * + * The recorder stores resolved paths and the glob patterns a unit issued, and the same path or pattern is + * commonly read by more than one key (a shared marker probe, a dependency a map step's keys each resolve, + * a shared glob), so the raw concatenation held one entry per read. The duplicates collapse downstream (the + * request graph keys on a Set), but carrying them inflates the recording the TaskRunner folds onto the + * stage monitor and the request-key set the request graph's exact-match lookup rebuilds, so the fold dedups + * each bucket into a Set here (and the TaskRunner's foldReadsInto dedups again against the + * monitored requests). Deduplication does not move the resulting signature. A cached key's patterns are + * folded back too, so a glob a map-step key issued on a previous run keeps moving the stage signature when + * a newly matching file appears, even on a build where that key did not re-run. + * + * @param {Map} invocationData The stage's complete per-key invocation data + * @returns {{reads: {project: {paths: string[], patterns: string[]}, + * dependencies: {paths: string[], patterns: string[]}}, + * inputs: Array<{type: string, name: string, value: string|undefined}>}} Folded reads and inputs + */ + #foldStageKeys(invocationData) { + const projectPaths = new Set(); + const dependencyPaths = new Set(); + const projectPatterns = new Set(); + const dependencyPatterns = new Set(); + const mergedInputs = new Map(); + for (const data of invocationData.values()) { + for (const path of data.reads ?? []) { + projectPaths.add(path); + } + for (const path of data.dependencyReads ?? []) { + dependencyPaths.add(path); + } + for (const pattern of data.patterns ?? []) { + projectPatterns.add(pattern); + } + for (const pattern of data.dependencyPatterns ?? []) { + dependencyPatterns.add(pattern); + } + for (const input of data.inputs ?? []) { + mergedInputs.set(`${input.type}\0${input.name}`, input); + } + } + return { + reads: { + project: {paths: [...projectPaths], patterns: [...projectPatterns]}, + dependencies: {paths: [...dependencyPaths], patterns: [...dependencyPatterns]}, + }, + inputs: [...mergedInputs.values()], + }; + } + + /** + * Assembles the needs object passed to a step from the recorded returns of the steps it + * names. + * + * One object is shared by the step's keys enumerator and all of its units, so it is + * frozen: a unit assigning to needs.<producer> would otherwise leak into its + * siblings and into the recorded needsInputs of whichever unit ran next, making a delta + * build's per-unit selection depend on execution order. The freeze is shallow, since a producer may + * return resources whose own state must stay writable; a step that needs a mutable copy makes one. + * + * @param {string[]} [names] Names of earlier steps this step needs + * @returns {object} Frozen {[name]: return} + */ + #buildNeeds(names) { + const needs = {}; + if (names) { + for (const name of names) { + needs[name] = this.#returns.get(name); + } + } + return Object.freeze(needs); + } + + /** + * The current return signatures of the steps a step needs, for per-unit delta selection. + * + * @param {string[]} [names] Names of earlier steps this step needs + * @returns {Map} Producer step name to its current return signature + */ + #collectNeedsSignatures(names) { + const signatures = new Map(); + if (names) { + for (const name of names) { + signatures.set(name, this.#returnSignatures.get(name)); + } + } + return signatures; + } + + /** + * Runs a map step's keys enumerator against the stage's own context. + * + * The enumerator owns no key, so there is no per-key invocation entry to attribute its activity to, + * and none is needed: it runs whenever the stage runs, so its reads and non-resource inputs are + * captured by the stage's monitored readers and MonitoredTaskUtil and fold into the stage signature, + * and the tags it sets reach the project tag collection and are recorded as the stage's tag + * operations, which a fully cached stage restores along with its writer. + * + * @param {object} step The map step + * @param {object} needs The step's needs object + * @param {object} ctx The per-stage context ({workspace, dependencies, taskUtil}) + * @returns {Promise>} Resolved key entries + */ + async #enumerateKeys(step, needs, ctx) { + const keys = await step.keys({ + needs, + workspace: ctx.workspace, + dependencies: ctx.dependencies, + taskUtil: ctx.taskUtil, + options: this.#options, + }); + if (!keys) { + return []; + } + return this.#resolveEntries(keys); + } + + /** + * The combined return signature of a step, used by a consumer's per-unit selection to detect a changed + * producer return. A scalar step's signature derives from its single unit's return; a map step's from the + * order-independent set of its units' returns. + * + * The map case hashes a keyId -> signature map sorted by keyId, not a + * positional array in entries order. entries order is keys() + * order, which for the shipped tasks is workspace.byGlob(...) order, so a reordering that + * changes nothing semantically (an adapter change, filesystem ordering, a reader-collection reshuffle) + * would otherwise move the signature and re-run every consumer for nothing. + * + * Both cases return a fixed-size SHA-256 hex digest rather than the raw serialization, so the value is + * consistent with every other signature in the cache system and the per-unit needsInputs + * each consumer records stays O(1) in size. The raw map serialization would be O(producer key count), + * copied into every consumer unit, so a map producer feeding a map consumer would persist an + * O(producers x consumers) sidecar. + * + * @param {Map} invocationData The step's per-key invocation data this build + * @param {Array<{keyId: string}>} entries The step's key entries this build + * @param {boolean} isScalar Whether the step is scalar + * @returns {string} The step's return signature, a SHA-256 hex digest + */ + #computeStepReturnSignature(invocationData, entries, isScalar) { + if (isScalar) { + // A scalar step is a single implicit unit, so there is no key order to normalize. + const first = entries[0]; + const raw = first ? + this.#returnDescriptorSignature(invocationData?.get(first.keyId)?.returns) : "none"; + return this.#hashSignature(raw); + } + const pairs = entries + .map(({keyId}) => [keyId, this.#returnDescriptorSignature(invocationData?.get(keyId)?.returns)]) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return this.#hashSignature(JSON.stringify(pairs)); + } + + /** + * Hashes a step's raw return serialization into a fixed-size SHA-256 hex digest. + * + * @param {string} data The raw serialization to hash + * @returns {string} SHA-256 hex digest + */ + #hashSignature(data) { + return crypto.createHash("sha256").update(data).digest("hex"); + } + + /** + * A stable string signature of a recorded return descriptor. + * + * @param {object|null} [returns] Recorded return descriptor + * @returns {string} Signature (content integrities for resources, serialized value for a value) + */ + #returnDescriptorSignature(returns) { + if (!returns) { + return "none"; + } + if (returns.kind === "value") { + return `v:${JSON.stringify(returns.value)}`; + } + return `r:${JSON.stringify({isArray: returns.isArray, integrities: returns.items.map((x) => x.integrity)})}`; + } + + async #resolveEntries(keys) { + return Promise.all([...keys].map(async (key, index) => ({ + key, + index, + keyId: await this.#keyId(key), + }))); + } + + async #keyId(key) { + if (key && typeof key.getIntegrity === "function") { + // The path distinguishes resources that share content but produce different output (e.g. two + // libraries' identical library.source.less); the trailing component makes a content change a + // new key, so the unit re-runs and its previous output is dropped rather than served stale. + // + // That trailing component is tiered like isResourceUnchanged (utils.js), cheapest first, to + // avoid hashing every key's content on every build (a stale-cache sap.m build spent ~420 ms + // here, see performance-investigation.md §12): lastModified + size when both are statically + // available (no content read), falling back to the SSRI integrity when either is missing (a + // memory- or generated resource with no lastModified, or one whose size is not statically + // known). A resource restored from a stage cache carries its integrity, so getIntegrity() + // resolves without reading content and the fallback stays cheap for that path too. + // + // Residual staleness risk, identical to isResourceUnchanged's and accepted for the same + // reason: a content change that preserves BOTH lastModified and size keeps the same key, so + // the unit is not re-run. A real edit moves mtime; the gap is for mtime-preserving replacements + // (cp -p, tar -x, atomic rename) that also hold size constant. The changed-path delta does not + // cover this case either, since it is derived through the same tiered comparison. + const lastModified = key.getLastModified?.(); + if (typeof lastModified === "number" && typeof key.hasSize === "function" && key.hasSize()) { + return `resource:${key.getPath()}\0m${lastModified}\0s${await key.getSize()}`; + } + return `resource:${key.getPath()}\0i${await key.getIntegrity()}`; + } + if (typeof key === "string") { + return `string:${key}`; + } + throw new Error( + "Map-step keys must be resources or strings. " + + "Express a compound key as a stable string."); + } + + /** + * Validates a unit's return value and, for a resource return, persists its content in the CAS. A + * serializable value is persisted inline in the invocation entry; a resource return is stored in the + * CAS by integrity; anything else throws. + * + * @param {*} returnValue The value the unit returned + * @returns {Promise} The persisted return descriptor: + * {kind: "resources", isArray, items: [{path, integrity, ...}]}, + * {kind: "value", value}, or null when the unit returned nothing (or a + * resource return has no store to persist against) + */ + async #recordReturn(returnValue) { + const normalized = this.#normalizeReturn(returnValue); + if (!normalized) { + return null; + } + if (normalized.kind === "value") { + // Serializable value: persisted inline, no CAS involved. + return {kind: "value", value: normalized.value}; + } + if (!this.#returnValueStore) { + // Standalone use: the fresh resource(s) are handed back for this build via the results array, + // but with no CAS to persist against there is nothing for a later build to restore. + return null; + } + const items = await this.#returnValueStore.store(normalized.resources); + return {kind: "resources", isArray: normalized.isArray, items}; + } + + /** + * Rebuilds a cached unit's return from the descriptor recorded on its previous run. + * + * @param {object|null} [returns] The recorded return descriptor, or falsy when the unit returned nothing + * @returns {*} The value, the single resource, the array of resources, or undefined + */ + #restoreReturn(returns) { + if (!returns) { + return undefined; + } + if (returns.kind === "value") { + return returns.value; + } + if (!this.#returnValueStore) { + throw new Error( + "Cannot restore a cached step's returned resources without a return value store"); + } + const items = returns.items.map((descriptor) => this.#returnValueStore.restore(descriptor)); + return returns.isArray ? items : items[0]; + } + + /** + * Classifies a return value as nothing, resource(s), or a serializable value, throwing on anything + * else (a returned array mixing resources and non-resources, or a value that is not JSON-serializable). + * + * @param {*} value The value the unit returned + * @returns {{kind: string, isArray: boolean|undefined, resources: object[]|undefined, + * value: *}|null} The classified + * return, or null when the unit returned nothing + */ + #normalizeReturn(value) { + if (value === undefined || value === null) { + return null; + } + if (Array.isArray(value) && value.every(isResource)) { + return {kind: "resources", isArray: true, resources: value}; + } + if (isResource(value)) { + return {kind: "resources", isArray: false, resources: [value]}; + } + if (Array.isArray(value) && value.some(isResource)) { + const i = value.findIndex((entry) => !isResource(entry)); + throw new Error( + `A returned array must contain only resources; array entry ${i} is ${describeValue(value[i])}`); + } + // Serializable value: reject anything JSON cannot represent (a function, a symbol, ...). + let serializable = true; + try { + serializable = JSON.stringify(value) !== undefined; + } catch { + serializable = false; + } + if (!serializable) { + throw new Error( + `A step may return resources or a JSON-serializable value; got ${describeValue(value)}`); + } + return {kind: "value", value}; + } + + #selectStepsToRun(entries, previous, needsSignatures, cacheInfo, isScalar) { + if (!cacheInfo || !previous || isScalar) { + // Full build, or a step with no previous data: run every unit. A scalar step always re-runs + // on a delta verdict too: it is a single implicit unit, so the per-unit reads delta cannot + // prune it, and that delta cannot catch a file newly matching a glob the step evaluated. The + // recorder stores resolved paths, not patterns, so a file that did not exist on the previous + // build appears in no recorded read; the owning stage signature does change (the stage-level + // monitor recorded the glob), so prepareStage returns a delta verdict here rather than a full + // hit. A full stage-cache hit (cacheInfo === true) never reaches this method, so an unchanged + // scalar step stays cached. + return entries; + } + const changedProject = new Set(cacheInfo.changedProjectResourcePaths ?? []); + const changedDependency = new Set(cacheInfo.changedDependencyResourcePaths ?? []); + return entries.filter(({keyId}) => { + const prev = previous.get(keyId); + if (!prev) { + // New key (a new string key, or a resource whose changed content yields a new integrity). + return true; + } + // A unit whose recorded reads intersect the changed paths must re-run: this is the reverse + // mapping that re-runs the owner of a changed cross-resource input (a .js whose .js.map + // changed, a theme whose gating marker was added or removed). + if (prev.reads.some((path) => changedProject.has(path)) || + prev.dependencyReads.some((path) => changedDependency.has(path))) { + return true; + } + // A unit whose recorded non-resource input no longer resolves to its stored value must re-run, + // so only the unit that read a changed env var, rolled-over time bucket or bumped dependency + // version re-runs. Without a resolver (standalone use) an input cannot be re-derived, so the + // unit is selected on its resource reads alone. + if (this.#resolveInputValue && prev.inputs?.some( + (input) => this.#resolveInputValue(input.type, input.name) !== input.value)) { + return true; + } + // A unit whose consumed needs return changed must re-run: the producer ran (or was restored) + // earlier this build, so its current return signature is known. A producer that re-ran with a + // changed return advances its signature; a restored producer keeps its previous one. + if (needsSignatures && prev.needsInputs?.some( + (needed) => needsSignatures.get(needed.name) !== needed.value)) { + return true; + } + return false; + }); + } + + /** + * Replays a restored unit's recorded tag operations into the project tag collection, so a unit served + * from cache contributes the same tags it would have set had it run. get operations carry + * no persistent effect and are skipped by the applier. A no-op without an applier (standalone use) or + * when the unit recorded no tag operations. + * + * @param {Array} [tagOperations] The unit's recorded tag operations + */ + #replayTagOperations(tagOperations) { + if (this.#applyTagOperations && tagOperations?.length) { + this.#applyTagOperations(tagOperations); + } + } + + #mergeInvocationData(current, entries, previous, cacheInfo) { + if (!cacheInfo || !previous) { + return current; + } + // A delta build re-runs only some units, so the persisted map must stay the complete set: keep a + // previous entry whose key is still present but was not re-run, drop keys no longer present, and + // let a re-run entry supersede its predecessor. + const currentKeyIds = new Set(entries.map((entry) => entry.keyId)); + const merged = new Map(); + for (const [keyId, data] of previous) { + if (!current.has(keyId) && currentKeyIds.has(keyId)) { + merged.set(keyId, data); + } + } + for (const [keyId, data] of current) { + merged.set(keyId, data); + } + return merged; + } + + /** + * Runs one step's stage (a scalar step's implicit unit, or a map step's keys): selects the units to + * run, restores the rest from cache, records each unit's reads/writes/inputs/tags/return, and merges + * the result into the stage's persisted invocation data. + * + * @param {string} group Step name + * @param {Array<{key: *, index: number, keyId: string}>} entries Resolved key entries + * @param {object} [options] Optional settings ({sequential}) + * @param {Function} callback async (key, unitCtx) => value? + * @param {object} context + * @param {object} [context.needs] The step's needs object, injected into each unit's context + * @param {Map} [context.needsSignatures] Current producer return signatures, recorded + * with each unit for the next build's selection + * @param {object|boolean} [context.cacheInfo] The stage's delta cache verdict (map step internal key-delta) + * @param {Map} [context.previous] The stage's previous per-key invocation data + * @param {object} context.ctx The per-stage context ({workspace, dependencies, taskUtil}) + * @param {boolean} [context.isScalar] Whether this stage is a scalar step (a single implicit unit), + * which is selected to run on any delta verdict (see {@link #selectStepsToRun}) + * @returns {Promise<{results: Array, invocationData: Map, + * freshInvocationData: Map}>} Per-key results aligned to entries order, + * the stage's complete per-key invocation data, and the subset of it recorded by the units that + * ran this build + */ + async #runGroup(group, entries, options, callback, + {needs, needsSignatures, cacheInfo, previous, ctx, isScalar}) { + const sequential = options?.sequential ?? false; + const concurrent = !sequential; + const toRun = this.#selectStepsToRun(entries, previous, needsSignatures, cacheInfo, isScalar); + const toRunIndices = new Set(toRun.map((entry) => entry.index)); + + const currentInvocationData = new Map(); + const results = new Array(entries.length); + const writeBuffer = concurrent ? new Map() : null; + const recordedNeeds = needsSignatures ? + [...needsSignatures].map(([name, value]) => ({name, value})) : []; + + // A unit served from cache did not run, so its return is rebuilt from the previous run's recorded + // descriptor, and its recorded tag operations are replayed so its tags reappear this build. Its + // slots are disjoint from the re-run units below, so this can happen before or after they run. + for (const {keyId, index} of entries) { + if (toRunIndices.has(index)) { + continue; + } + const prev = previous?.get(keyId); + results[index] = this.#restoreReturn(prev?.returns); + this.#replayTagOperations(prev?.tagOperations); + } + + const runStep = async ({key, keyId, index}) => { + this.#signal?.throwIfAborted(); + const recorder = new StepRecorder(); + const workspace = new RecordingReaderWriter(ctx.workspace, recorder, writeBuffer, index); + const dependencies = ctx.dependencies ? + new RecordingReader(ctx.dependencies, recorder) : undefined; + // A per-unit MonitoredTaskUtil wrapping the stage's taskUtil: it attributes the unit's + // non-resource inputs and tag operations to the unit for per-unit selection and restore, while + // reads still delegate through the stage's monitored readers. + const taskUtil = new MonitoredTaskUtil(ctx.taskUtil, {recordTagOperations: true}); + + const returnValue = await callback(key, {workspace, dependencies, taskUtil, needs, options: this.#options}); + results[index] = returnValue; + currentInvocationData.set(keyId, { + reads: [...recorder.projectReads], + dependencyReads: [...recorder.dependencyReads], + patterns: [...recorder.projectPatterns], + dependencyPatterns: [...recorder.dependencyPatterns], + writes: [...recorder.writes], + inputs: taskUtil.getInputRecording(), + needsInputs: recordedNeeds, + tagOperations: taskUtil.getTagOperations(), + returns: await this.#recordReturn(returnValue), + }); + }; + + if (concurrent) { + await Promise.all(toRun.map(runStep)); + await flushWriteBuffer(writeBuffer, ctx.workspace); + } else { + for (const entry of toRun) { + await runStep(entry); + } + } + + // The return value store buffers each unit's returned content; flush this step's buffer in one + // transaction now that every unit has recorded its return. A no-op for a step that returned + // nothing, and for standalone use with no store. + this.#returnValueStore?.flush?.(); + + const invocationData = this.#mergeInvocationData(currentInvocationData, entries, previous, cacheInfo); + + if (log.isLevelEnabled("verbose")) { + log.verbose(`step '${group}': ran ${toRun.length} of ${entries.length} unit(s)`); + } + return {results, invocationData, freshInvocationData: currentInvocationData}; + } +} + + diff --git a/packages/project/lib/build/helpers/TaskUtil.js b/packages/project/lib/build/helpers/TaskUtil.js index 0856e832fb1..ea55bb50fc3 100644 --- a/packages/project/lib/build/helpers/TaskUtil.js +++ b/packages/project/lib/build/helpers/TaskUtil.js @@ -6,6 +6,7 @@ import { createLinkReader, createFlatReader } from "@ui5/fs/resourceFactory"; +import {quantizeTime} from "./quantizeTime.js"; /** * Convenience functions for UI5 tasks. @@ -137,6 +138,91 @@ class TaskUtil { return collection.clearTag(resource, tag); } + /** + * Reads an environment variable. + * + * Tasks whose output depends on an environment variable must read it through this method rather + * than accessing process.env directly. During a build the task receives a + * [MonitoredTaskUtil]{@link @ui5/project/build/helpers/MonitoredTaskUtil} that records the read + * and folds it into the task's build-cache signature, so that changing the variable between + * builds invalidates the task's cached result (the same way a changed resource does). Reading + * process.env directly is not tracked and can lead to a stale cached result being + * served. + * + *

+ * This method is only available to custom task extensions defining + * Specification Version 5.0 and above. + * + * @param {string} name Environment variable name + * @returns {string|undefined} The environment variable value, or undefined if unset + * @public + */ + getEnv(name) { + return process.env[name]; + } + + /** + * Reads the build's time quantized to a fixed granularity. + * + * The returned value comes from a single timestamp fixed once per build run (shared by every + * project and task in the run), not a fresh Date per call, so all time reads within + * a run agree. + * + * Tasks whose output depends on the current time (for example + * [replaceCopyright]{@link @ui5/builder/tasks/replaceCopyright}, which expands + * ${currentYear}) must read it through this method rather than calling + * new Date() directly. During a build the task receives a + * [MonitoredTaskUtil]{@link @ui5/project/build/helpers/MonitoredTaskUtil} that records the read + * and folds it into the task's build-cache signature, so a cached result re-runs once the time + * bucket rolls over (a "year"-granularity result is re-run at the next calendar year). + * A direct Date read is not tracked and can serve a stale result. + * + * The granularity is the contract: it names the bucket at which the output is stable. A + * millisecond-precision read would change on every build and miss the cache every time, so this + * method never returns a raw timestamp. Reading at "year" means "re-run only when the + * year changes". + * + *

+ * This method is only available to custom task extensions defining + * Specification Version 5.0 and above. + * + * @param {string} granularity Time bucket, one of "year", "month", + * "day", "hour" + * @returns {string} Current time quantized to the granularity, e.g. "2026" for + * "year" or "2026-09-25T14" for "hour" + * @throws {Error} If the granularity is not one of the supported buckets + * @public + */ + getTime(granularity) { + return quantizeTime(granularity, this._projectBuildContext.getBuildTime()); + } + + /** + * Returns the build run's shared timestamp as a raw Date. + * + * The value is the single timestamp fixed once per build run (shared by every project and task in + * the run), the same source [getTime]{@link @ui5/project/build/helpers/TaskUtil#getTime} quantizes. + * + * Unlike getTime, this read is deliberately not tracked as a build-cache input: + * the [MonitoredTaskUtil]{@link @ui5/project/build/helpers/MonitoredTaskUtil} passes it through + * untracked, so its value never folds into the task's cache signature. A cached result therefore + * keeps the timestamp it embedded on the build that produced it, rather than re-running because the + * timestamp advanced. That staleness is intended: a build timestamp changes on every build, so + * tracking it would force a cache miss every time and defeat caching for any task that reads it. + * Read through getTime instead when the output should re-run once a time bucket rolls + * over (for example a copyright year). + * + *

+ * This method is only available to custom task extensions defining + * Specification Version 5.0 and above. + * + * @returns {Date} The current build run's timestamp + * @public + */ + getBuildTime() { + return this._projectBuildContext.getBuildTime(); + } + /** * Check whether the project currently being built is the root project. * @@ -346,6 +432,10 @@ class TaskUtil { baseInterface.resourceFactory[factoryFunction] = this.resourceFactory[factoryFunction]; }); } + + if (specVersion.gte("5.0")) { + bindFunctions(this, baseInterface, ["getEnv", "getTime", "getBuildTime"]); + } return baseInterface; } @@ -390,6 +480,10 @@ class TaskUtil { baseInterface.resourceFactory[factoryFunction] = this.resourceFactory[factoryFunction]; }); } + + if (specVersion.gte("5.0")) { + bindFunctions(this, baseInterface, ["getEnv", "getTime", "getBuildTime"]); + } return baseInterface; } } diff --git a/packages/project/lib/build/helpers/quantizeTime.js b/packages/project/lib/build/helpers/quantizeTime.js new file mode 100644 index 00000000000..98469894b24 --- /dev/null +++ b/packages/project/lib/build/helpers/quantizeTime.js @@ -0,0 +1,58 @@ +/** + * Fixed set of time granularities [TaskUtil#getTime]{@link @ui5/project/build/helpers/TaskUtil#getTime} + * accepts. Each names the bucket at which a time-derived task output is stable: reading at + * "year" means "re-run only when the calendar year changes". Sub-hour buckets are + * deliberately absent, since they rarely name a stable output and would miss the cache on almost + * every build. + * + * @type {string[]} + */ +export const TIME_GRANULARITIES = ["year", "month", "day", "hour"]; + +function pad(value) { + return String(value).padStart(2, "0"); +} + +/** + * Quantizes a point in time to a stable string for the given granularity. + * + * The counterpart accessors [TaskUtil#getTime]{@link @ui5/project/build/helpers/TaskUtil#getTime} + * (record side) and + * [ProjectBuildContext#resolveInputValue]{@link @ui5/project/build/helpers/ProjectBuildContext} + * (lookup side) both quantize through this function, + * so the value recorded during one build and the value re-derived during a later build are equal + * whenever they fall in the same bucket. Two builds within the same bucket produce the same string + * and hit the cache; a rolled-over bucket produces a different string and re-runs the task. + * + * Local time throughout, matching the new Date().getFullYear() reads these accessors + * replace. Record and lookup run on the same machine, so the local-time bucket agrees across builds; + * a cache shared across machines in different timezones could disagree at a bucket boundary, which + * mirrors the existing local-machine assumptions of the dev cache. + * + * The date is mandatory: both callers pass the build run's shared timestamp (from + * [BuildContext#getBuildTime]{@link @ui5/project/build/helpers/BuildContext}) so every quantization + * in a run resolves against the same instant. Requiring it stops a caller from silently falling back + * to a fresh new Date(), which would reintroduce intra-run divergence. + * + * @param {string} granularity One of {@link TIME_GRANULARITIES} + * @param {Date} date Point in time to quantize (typically the build run's shared timestamp) + * @returns {string} Stable bucket string, e.g. "2026" for "year" or + * "2026-09-25T14" for "hour" + * @throws {Error} If the granularity is not one of {@link TIME_GRANULARITIES}, or if + * date is not a Date + */ +export function quantizeTime(granularity, date) { + if (!TIME_GRANULARITIES.includes(granularity)) { + throw new Error( + `Invalid time granularity "${granularity}". Expected one of: ${TIME_GRANULARITIES.join(", ")}`); + } + if (!(date instanceof Date)) { + throw new Error(`Missing or invalid 'date' argument: expected a Date instance`); + } + // Buckets nest, so each coarser bucket is a prefix of the next finer one. + const year = String(date.getFullYear()); + const month = `${year}-${pad(date.getMonth() + 1)}`; + const day = `${month}-${pad(date.getDate())}`; + const hour = `${day}T${pad(date.getHours())}`; + return {year, month, day, hour}[granularity]; +} diff --git a/packages/project/lib/resources/ProjectResources.js b/packages/project/lib/resources/ProjectResources.js index cf918464a5e..4f63ae1a4f5 100644 --- a/packages/project/lib/resources/ProjectResources.js +++ b/packages/project/lib/resources/ProjectResources.js @@ -388,6 +388,39 @@ class ProjectResources { return true; // Indicate that the stored stage has changed } + /** + * Reopen a stage with a fresh, empty live writer, discarding any cached read-only stage a prior + * {@link #setStage} installed for it. + * + * Used when a step-based task's stage was a full cache hit that must be re-run after all (a consumed + * producer needs return changed): the fast-path restore had swapped in a read-only cached + * stage via {@link #setStage}, so the stage has no writer, and re-running its units needs a writable + * one. The reopened stage reads all previous stages exactly as {@link #useStage} does; the re-run's + * output goes into the fresh writer. + * + * @public + * @param {string} stageId The ID of the stage to reopen + * @throws {Error} If the stage does not exist + */ + reopenStage(stageId) { + const stageIdx = this.#stages.findIndex((s) => s.getId() === stageId); + if (stageIdx === -1) { + throw new Error(`Stage '${stageId}' does not exist in project ${this.#getName()}`); + } + const newStage = new Stage(stageId, this.#createWriter(stageId)); + this.#stages[stageIdx] = newStage; + this.#currentStage = newStage; + this.#currentStageId = stageId; + this.#currentStageReadIndex = stageIdx - 1; // Read from all previous stages + + // Unset "current" reader/writer caches. They will be recreated on demand + this.#currentStageReaders = new Map(); + this.#currentStageWorkspace = null; + + this.#monitoredProjectResourceTagCollection = null; + this.#monitoredBuildResourceTagCollection = null; + } + buildFinished() { // Clear build resource tag collections. They must not be provided to dependent projects this.#buildResourceTagCollection = null; @@ -406,6 +439,21 @@ class ProjectResources { * @throws {Error} If no collection accepts the given tag */ getResourceTagCollection(resource, tag) { + return this.#getMonitoredTagCollection(tag, resource); + } + + /** + * Returns the monitored tag collection that accepts tag, creating it on first use. The + * routing is by tag alone (project-level tags such as ui5:IsDebugVariant vs. build-level + * tags such as ui5:OmitFromBuildResult); resource is used only to name the + * offending resource when no collection accepts the tag. + * + * @param {string} tag Tag to route + * @param {@ui5/fs/Resource} [resource] Resource the tag is for, for the error message only + * @returns {@ui5/fs/internal/MonitoredResourceTagCollection} The monitored collection + * @throws {Error} If no collection accepts the given tag + */ + #getMonitoredTagCollection(tag, resource) { this.#applyCachedResourceTags(); const projectCollection = this.#getProjectResourceTagCollection(); if (!tag || projectCollection.acceptsTag(tag)) { @@ -421,7 +469,32 @@ class ProjectResources { } return this.#monitoredBuildResourceTagCollection; } - throw new Error(`Could not find collection for resource ${resource.getPath()} and tag ${tag}`); + throw new Error( + `Could not find collection for resource ${resource ? resource.getPath() : "(tag replay)"} and tag ${tag}`); + } + + /** + * Replays a set of tag operations recorded by a + * [StepRunner]{@link @ui5/project/build/helpers/StepRunner} step into the monitored tag collections, + * routing each by tag and applying it by path. Used when a step is restored from cache on a delta + * build: the step did not run, so its set/clear operations are replayed here + * so its tags reappear in this build's tag operations (captured by {@link #getResourceTagOperations} + * like a step that ran). get operations carry no persistent effect and are skipped. + * + * @param {Array<{op: string, path: string, tag: string, value: *}>} tagOperations Recorded operations + */ + replayTagOperations(tagOperations) { + for (const {op, path, tag, value} of tagOperations) { + if (op !== "set" && op !== "clear") { + continue; + } + const collection = this.#getMonitoredTagCollection(tag); + if (op === "clear") { + collection.clearTag(path, tag); + } else { + collection.setTag(path, tag, value); + } + } } getResourceTagOperations() { diff --git a/packages/project/lib/specifications/extensions/Task.js b/packages/project/lib/specifications/extensions/Task.js index c622af965b1..ac9ce69f00a 100644 --- a/packages/project/lib/specifications/extensions/Task.js +++ b/packages/project/lib/specifications/extensions/Task.js @@ -39,10 +39,14 @@ class Task extends Extension { } /** - * @public - */ - async getSupportsDifferentialBuildsCallback() { - return (await this._getImplementation()).supportsDifferentialBuilds; + * Whether the task opts into the step-factory build API, declared as a static stepBased + * export (a plain boolean value, not a callback). Honored from Specification Version 5.0. + * + * @public + * @returns {Promise} The task module's stepBased export + */ + async getStepBased() { + return (await this._getImplementation()).stepBased; } /** diff --git a/packages/project/test/fixtures/application.a/task.root-conditional.js b/packages/project/test/fixtures/application.a/task.root-conditional.js new file mode 100644 index 00000000000..fa15cac1527 --- /dev/null +++ b/packages/project/test/fixtures/application.a/task.root-conditional.js @@ -0,0 +1,19 @@ +// Custom task whose root read is conditional: it reads /tsconfig.json through the project root reader +// only while /toggle.js exists in the workspace. Exercises a stage that stops reading a root resource. +// Once /toggle.js is gone the task records no root read, so the previously recorded root request must +// be cleared (and the emptied request set re-persisted), and a later change to /tsconfig.json must then +// no longer invalidate this stage. +module.exports = async function ({taskUtil, workspace, options: {projectNamespace}}) { + const {createResource} = taskUtil.resourceFactory; + const toggle = await workspace.byPath(`/resources/${projectNamespace}/toggle.js`); + let content = "root-not-read"; + if (toggle) { + const rootReader = taskUtil.getProject().getRootReader(); + const tsconfig = await rootReader.byPath("/tsconfig.json"); + content = tsconfig ? await tsconfig.getString() : "no-tsconfig"; + } + await workspace.write(createResource({ + path: "/rootConditionalDigest.js", + string: `export const content = ${JSON.stringify(content)};\n`, + })); +}; diff --git a/packages/project/test/fixtures/application.a/task.root-config.js b/packages/project/test/fixtures/application.a/task.root-config.js new file mode 100644 index 00000000000..20fe7b00a15 --- /dev/null +++ b/packages/project/test/fixtures/application.a/task.root-config.js @@ -0,0 +1,13 @@ +// Custom task that reads a configuration file from the project root (outside the UI5 resource model) +// through taskUtil.getProject().getRootReader() and derives its output from that file's content. +// Exercises root-resource tracking: a change to the root config must invalidate this task's cache. +module.exports = async function ({taskUtil, workspace}) { + const {createResource} = taskUtil.resourceFactory; + const rootReader = taskUtil.getProject().getRootReader(); + const tsconfig = await rootReader.byPath("/tsconfig.json"); + const content = tsconfig ? await tsconfig.getString() : "no-tsconfig"; + await workspace.write(createResource({ + path: "/tsconfigDigest.js", + string: `export const tsconfig = ${JSON.stringify(content)};\n`, + })); +}; diff --git a/packages/project/test/fixtures/application.a/task.root-glob.js b/packages/project/test/fixtures/application.a/task.root-glob.js new file mode 100644 index 00000000000..931eb65bb73 --- /dev/null +++ b/packages/project/test/fixtures/application.a/task.root-glob.js @@ -0,0 +1,14 @@ +// Custom task that reads root configuration files through a glob with the gitignore filter disabled, so +// the read is recorded against the useGitignore:false root manager. Exercises glob-based root tracking: +// adding or removing a file matching the glob must invalidate the task's cache (root indices refresh by +// re-globbing, so a newly matching file is detected). +module.exports = async function ({taskUtil, workspace}) { + const {createResource} = taskUtil.resourceFactory; + const rootReader = taskUtil.getProject().getRootReader({useGitignore: false}); + const configs = await rootReader.byGlob("/rootcfg/**/*.json"); + const names = configs.map((r) => r.getPath()).sort(); + await workspace.write(createResource({ + path: "/rootGlobDigest.js", + string: `export const configs = ${JSON.stringify(names)};\n`, + })); +}; diff --git a/packages/project/test/fixtures/application.a/task.step-based.js b/packages/project/test/fixtures/application.a/task.step-based.js new file mode 100644 index 00000000000..1021fb396e3 --- /dev/null +++ b/packages/project/test/fixtures/application.a/task.step-based.js @@ -0,0 +1,40 @@ +const Logger = require("@ui5/logger"); +const log = Logger.getLogger("builder:tasks:stepBasedTask"); + +// Custom step-based task: one cached unit per `.src` resource under /procEach, expressed as a single map +// step. The step's keys enumerator lists the `.src` resources; each unit reads its sibling `.dep` file as +// a cross-resource input through the step workspace and writes a combined `.out`. Because the `.dep` read +// goes through the step workspace, it is recorded as that unit's input, so on a delta build changing only +// a `.dep` re-runs its owning unit and leaves the others served from cache. This is the same reverse +// mapping the built-in minify relies on for its `.js.map` -> `.js` relation, expressed by a custom task. +module.exports = function build() { + return [{ + name: "procEach", + keys: async ({workspace, taskUtil, options: {projectNamespace}}) => { + // Tag from the enumerator, not from a unit: the enumerator runs whenever the stage runs, but + // it owns no key, so its tags have no per-key invocation entry to be replayed from. They + // survive a fully cached stage through the stage's own recorded tag operations. + const omittedResources = await workspace.byGlob(`/resources/${projectNamespace}/procEach/*.omitme`); + for (const omittedResource of omittedResources) { + taskUtil.setTag(omittedResource, taskUtil.STANDARD_TAGS.OmitFromBuildResult); + } + const srcResources = await workspace.byGlob(`/resources/${projectNamespace}/procEach/*.src`); + log.verbose(`step-based-task processing ${srcResources.length} source(s)`); + return srcResources; + }, + each: async (srcResource, {workspace, taskUtil}) => { + const srcPath = srcResource.getPath(); + const depPath = srcPath.replace(/\.src$/, ".dep"); + // Read the cross-resource input through the step workspace so it is tracked as this unit's input. + const depResource = await workspace.byPath(depPath); + const depContent = depResource ? await depResource.getString() : ""; + const srcContent = await srcResource.getString(); + const outResource = taskUtil.resourceFactory.createResource({ + path: srcPath.replace(/\.src$/, ".out"), + string: `${srcContent}\n// dep: ${depContent}\n`, + }); + await workspace.write(outResource); + }, + }]; +}; +module.exports.stepBased = true; diff --git a/packages/project/test/fixtures/application.a/ui5-customTask-root-conditional.yaml b/packages/project/test/fixtures/application.a/ui5-customTask-root-conditional.yaml new file mode 100644 index 00000000000..55eb1306750 --- /dev/null +++ b/packages/project/test/fixtures/application.a/ui5-customTask-root-conditional.yaml @@ -0,0 +1,17 @@ +--- +specVersion: "5.0" +type: application +metadata: + name: application.a +builder: + customTasks: + - name: root-conditional + afterTask: minify +--- +specVersion: "5.0" +kind: extension +type: task +metadata: + name: root-conditional +task: + path: task.root-conditional.js diff --git a/packages/project/test/fixtures/application.a/ui5-customTask-root-config.yaml b/packages/project/test/fixtures/application.a/ui5-customTask-root-config.yaml new file mode 100644 index 00000000000..f89d06b0008 --- /dev/null +++ b/packages/project/test/fixtures/application.a/ui5-customTask-root-config.yaml @@ -0,0 +1,17 @@ +--- +specVersion: "5.0" +type: application +metadata: + name: application.a +builder: + customTasks: + - name: root-config + afterTask: minify +--- +specVersion: "5.0" +kind: extension +type: task +metadata: + name: root-config +task: + path: task.root-config.js diff --git a/packages/project/test/fixtures/application.a/ui5-customTask-root-multi.yaml b/packages/project/test/fixtures/application.a/ui5-customTask-root-multi.yaml new file mode 100644 index 00000000000..782d5be40dd --- /dev/null +++ b/packages/project/test/fixtures/application.a/ui5-customTask-root-multi.yaml @@ -0,0 +1,27 @@ +--- +specVersion: "5.0" +type: application +metadata: + name: application.a +builder: + customTasks: + - name: root-config + afterTask: minify + - name: root-glob + afterTask: root-config +--- +specVersion: "5.0" +kind: extension +type: task +metadata: + name: root-config +task: + path: task.root-config.js +--- +specVersion: "5.0" +kind: extension +type: task +metadata: + name: root-glob +task: + path: task.root-glob.js diff --git a/packages/project/test/fixtures/application.a/ui5-customTask-stepBased.yaml b/packages/project/test/fixtures/application.a/ui5-customTask-stepBased.yaml new file mode 100644 index 00000000000..0ef7266b983 --- /dev/null +++ b/packages/project/test/fixtures/application.a/ui5-customTask-stepBased.yaml @@ -0,0 +1,17 @@ +--- +specVersion: "5.0" +type: application +metadata: + name: application.a +builder: + customTasks: + - name: step-based-task + afterTask: minify +--- +specVersion: "5.0" +kind: extension +type: task +metadata: + name: step-based-task +task: + path: task.step-based.js diff --git a/packages/project/test/fixtures/library.framework/main/src/library/framework/.library b/packages/project/test/fixtures/library.framework/main/src/library/framework/.library new file mode 100644 index 00000000000..2fb203252b9 --- /dev/null +++ b/packages/project/test/fixtures/library.framework/main/src/library/framework/.library @@ -0,0 +1,11 @@ + + + + library.framework + SAP SE + ${copyright} + ${version} + + Framework Library + + diff --git a/packages/project/test/fixtures/library.framework/main/src/library/framework/some.js b/packages/project/test/fixtures/library.framework/main/src/library/framework/some.js new file mode 100644 index 00000000000..719155d1e6d --- /dev/null +++ b/packages/project/test/fixtures/library.framework/main/src/library/framework/some.js @@ -0,0 +1,4 @@ +/*! + * ${copyright} + */ +console.log('HelloWorld'); diff --git a/packages/project/test/fixtures/library.framework/package.json b/packages/project/test/fixtures/library.framework/package.json new file mode 100644 index 00000000000..dce49829178 --- /dev/null +++ b/packages/project/test/fixtures/library.framework/package.json @@ -0,0 +1,9 @@ +{ + "name": "@openui5/library.framework", + "version": "1.0.0", + "description": "Framework-named library used to exercise generateThemeDesignerResources", + "dependencies": {}, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + } +} diff --git a/packages/project/test/fixtures/library.framework/ui5.yaml b/packages/project/test/fixtures/library.framework/ui5.yaml new file mode 100644 index 00000000000..287f5283c42 --- /dev/null +++ b/packages/project/test/fixtures/library.framework/ui5.yaml @@ -0,0 +1,11 @@ +--- +specVersion: "2.3" +type: library +metadata: + name: library.framework + copyright: Some fancy copyright +resources: + configuration: + paths: + src: main/src + test: main/test diff --git a/packages/project/test/lib/build/BuildServer.abortRetry.integration.js b/packages/project/test/lib/build/BuildServer.abortRetry.integration.js index 9571d402a5e..70a9b9d2484 100644 --- a/packages/project/test/lib/build/BuildServer.abortRetry.integration.js +++ b/packages/project/test/lib/build/BuildServer.abortRetry.integration.js @@ -23,9 +23,9 @@ const FixtureTester = createFixtureTesterFactory("abortRetry", {graphFromPackage registerBuildHooks(test, {watcherMock}); // ProjectBuildCache's StageCache must be cleared correctly when a build is aborted. -// A task that completed during an aborted attempt has already called recordTaskResult, +// A task that completed during an aborted attempt has already called recordStageResult, // which adds its stage to the in-memory StageCache. On retry, -// prepareTaskExecutionAndValidateCache might finds those entries via #findStageCache if not cleaned up. +// prepareStageExecutionAndValidateCache might finds those entries via #findStageCache if not cleaned up. // It will then emit task-skip events for tasks that the retry should have actually re-executed. test.serial("Aborted initial build must not leak in-memory StageCache to retry", async (t) => { const fixtureTester = t.context.fixtureTester = await FixtureTester.create(t, "library.d"); @@ -37,7 +37,7 @@ test.serial("Aborted initial build must not leak in-memory StageCache to retry", // One-shot trigger: when `replaceBuildtime` (the 4th task for this fixture) ends in the // initial build, simulate a watcher event by calling _projectResourceChanged directly. // This invalidates library.d, aborts the running build at the next signal check, and - // re-enqueues it. By that point, tasks 1-4 have completed recordTaskResult and live in + // re-enqueues it. By that point, tasks 1-4 have completed recordStageResult and live in // the in-memory StageCache. Tasks 5+ never started. let aborted = false; const abortHandler = (event) => { @@ -62,7 +62,7 @@ test.serial("Aborted initial build must not leak in-memory StageCache to retry", t.true(aborted, "Test setup precondition: abort trigger should have fired"); // On a fresh fixture the persistent cache is empty. After the fix, the retry's - // prepareTaskExecutionAndValidateCache should find no cached stages (in-memory cache + // prepareStageExecutionAndValidateCache should find no cached stages (in-memory cache // from the aborted build is discarded) and execute every task. No task-skip events // should be emitted for library.d. const skippedTasks = t.context.projectBuildStatusEventStub.args diff --git a/packages/project/test/lib/build/BuildServer.buildSignature.integration.js b/packages/project/test/lib/build/BuildServer.buildSignature.integration.js index 9fce676dbc2..70f94f11b39 100644 --- a/packages/project/test/lib/build/BuildServer.buildSignature.integration.js +++ b/packages/project/test/lib/build/BuildServer.buildSignature.integration.js @@ -80,7 +80,7 @@ test.serial.failing( // This asserts the desired behavior (the changed input map is reflected in the served debug map without // a server restart) and is marked test.failing because the delta path does not yet achieve it. See the // minify FIXME for why a fix needs the `.map` -> `.js` relation, not a local pattern tweak. -test.serial.failing( +test.serial( "Serve application.a, changing only an input source map read via fs by minify invalidates the debug source map", async (t) => { const fixtureTester = t.context.fixtureTester = await FixtureTester.create(t, "application.a"); diff --git a/packages/project/test/lib/build/ProjectBuilder.bundling.integration.js b/packages/project/test/lib/build/ProjectBuilder.bundling.integration.js index 1110560cd3a..ce618b43788 100644 --- a/packages/project/test/lib/build/ProjectBuilder.bundling.integration.js +++ b/packages/project/test/lib/build/ProjectBuilder.bundling.integration.js @@ -598,25 +598,20 @@ test.serial("Build component.a (Custom Component preload configuration)", async // section excludes it ("Do not include manifest.json in UI5 2.x and higher ..."). The sap.ui.core // version is therefore a genuine input of the preload. // -// But the project build signature (getBuildSignature.js `getProjectSignature`) folds in only the -// project's OWN id/config, task signatures, and the @ui5/builder/@ui5/fs/@ui5/project versions — -// never a DEPENDENCY project's version. So changing only the sap.ui.core dependency version between -// builds, while the library's own sources are untouched, does not change the signature: the result -// cache hits and the previously built (stale) preload is served. +// The project build signature (getBuildSignature.js `getProjectSignature`) folds in only the +// project's OWN id/config, task signatures, and the @ui5/builder/@ui5/fs/@ui5/project versions, +// never a DEPENDENCY project's version. generateLibraryPreload also reads ONLY the current project's +// own workspace (`nonDbgWorkspace.byGlob(...)`), never the `dependencies` reader, so the built +// resource content of the sap.ui.core dependency is not an input either. The version reaches the +// output solely through `taskUtil.getProject("sap.ui.core").getVersion()`. // -// generateLibraryPreload reads ONLY the current project's own workspace (`nonDbgWorkspace.byGlob(...)`), -// never the `dependencies` reader. So the built RESOURCE content of the sap.ui.core dependency is never -// an input to library.d's preload — a version bump reaches the output solely through -// `taskUtil.getProject("sap.ui.core").getVersion()`, which is precisely the input the signature does -// not observe. -// -// This asserts the desired behavior: after bumping the sap.ui.core dependency from 1.x to 2.x, the -// rebuilt preload reflects the >=2 form (manifest.json no longer bundled). It is marked test.failing -// because the signature does not yet observe the dependency version, so the v1 preload is served -// stale. AVA reports a failing-marked test as a pass while it throws and as a hard error once it -// starts passing; committing it keeps CI green and flips to a signal the moment the gap is fixed -// (at which point drop the `.failing`). -test.serial.failing( +// The task receives its taskUtil wrapped in a MonitoredTaskUtil, so that getVersion read is recorded +// as a task input ("project.getVersion" for sap.ui.core) and folded into the task's build-cache +// signature. On a later build the recorded input is re-evaluated against the current graph +// (ProjectBuildContext#resolveInputValue), so a version bump misses the cache and rebuilds the +// preload. This asserts that: after bumping sap.ui.core from 1.x to 2.x, the rebuilt preload reflects +// the >=2 form (manifest.json no longer bundled). +test.serial( "Build library.d (changing the sap.ui.core dependency version invalidates the library preload)", async (t) => { const fixtureTester = new FixtureTester(t, "library.d"); @@ -673,3 +668,75 @@ test.serial.failing( t.false(preloadV2.includes("library/d/manifest.json"), "Preload rebuilt against sap.ui.core@2.x no longer bundles the library's manifest.json"); }); + +// generateLibraryPreload reads process.env.UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD +// (generateLibraryPreload.js: `const createBundleInfoPreload = !!process.env.UI5_CLI_EXPERIMENTAL_...`). +// The env var switches the task between two output shapes: without it a single +// `${namespace}/library-preload.js` is produced; with it the experimental bundle-info path additionally +// emits a `${namespace}/_library-content.js` bundle (getBundleInfoPreloadDefinition / +// getContentBundleDefinition). The env var is therefore a genuine input of the task's output. +// +// But the incremental build tracks only RESOURCE inputs: a task's stage signature is derived from the +// content hashes of the project/dependency resources it reads (BuildStageCache.recordRequests -> +// ResourceRequestManager), and the project build signature (getBuildSignature.js `getProjectSignature`) +// folds in only build config, task option signatures, project id/config and tool versions. No +// process.env value feeds either. So flipping UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD between builds, +// while the library's sources are untouched, does not change any signature: the result cache hits and +// the previously built (non-bundle-info) preload is served, and `_library-content.js` is never produced. +// +// This asserts the desired behavior: after enabling the env var, the rebuilt output includes the +// experimental `_library-content.js` bundle. generateLibraryPreload reads the flag through +// taskUtil.getEnv, so the incremental build tracks it as a task input and invalidates the cached +// result when it changes. +test.serial( + "Build library.d (toggling UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD invalidates the library preload)", + async (t) => { + const fixtureTester = new FixtureTester(t, "library.d"); + const destPath = fixtureTester.destPath; + const contentBundlePath = `${destPath}/resources/library/d/_library-content.js`; + const preloadPath = `${destPath}/resources/library/d/library-preload.js`; + + // Always restore the env var so neighbouring tests observe the ambient value. + const previousFlag = process.env.UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD; + t.teardown(() => { + if (previousFlag === undefined) { + delete process.env.UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD; + } else { + process.env.UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD = previousFlag; + } + }); + + await fixtureTester._initialize(); + + // The experimental bundle-info path resolves `${namespace}/library.js` and a + // `${namespace}/manifest.json`; ship a manifest.json so both the preload and the content bundle + // have something to bundle. + await fs.writeFile(`${fixtureTester.fixturePath}/main/src/library/d/manifest.json`, + JSON.stringify({"sap.app": {"id": "library.d", "type": "library"}}, null, "\t")); + + // #1 build (no cache) with the experimental flag DISABLED: regular preload, no content bundle. + delete process.env.UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD; + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5.yaml"}, + config: {destPath, cleanDest: true}, + }); + + await t.notThrowsAsync(fs.readFile(preloadPath, {encoding: "utf8"}), + "Regular build produces the library-preload.js bundle"); + await t.throwsAsync(fs.readFile(contentBundlePath, {encoding: "utf8"}), undefined, + "Regular build does not produce the experimental _library-content.js bundle"); + + // Enable the experimental flag. No library source resource changes. + process.env.UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD = "true"; + + // #2 build (with cache): generateLibraryPreload must re-run under the experimental path and + // produce the additional _library-content.js bundle. + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5.yaml"}, + config: {destPath, cleanDest: true}, + }); + + await t.notThrowsAsync(fs.readFile(contentBundlePath, {encoding: "utf8"}), + "Rebuild with UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD enabled produces the " + + "experimental _library-content.js bundle"); + }); diff --git a/packages/project/test/lib/build/ProjectBuilder.caching.integration.js b/packages/project/test/lib/build/ProjectBuilder.caching.integration.js index eb80ef1f5c4..fdc4f08cd3d 100644 --- a/packages/project/test/lib/build/ProjectBuilder.caching.integration.js +++ b/packages/project/test/lib/build/ProjectBuilder.caching.integration.js @@ -20,15 +20,14 @@ function themeOutputs(namespace) { // available via workspace+dependencies (its `librariesPattern` filter, active when the theme-library // is built as a DEPENDENCY). The `themelib.multi` fixture ships `library.source.less` for two library // namespaces (`lib/one`, `lib/two`), each gated by its own `library.js` marker. Adding/removing a -// marker changes which single theme should be (re)built — the others must stay served from cache. +// marker changes which single theme should be (re)built, and the others must stay served from cache. // -// Both tests are marked test.serial.failing: buildThemes does NOT set `supportsDifferentialBuilds`, -// so ANY tracked-input change re-runs the whole task and rewrites EVERY matched theme. There is no -// per-theme delta and no preservation of unaffected theme output. The new task system -// (CPOUI5FOUNDATION-1363) is expected to make this correct by design; dropping `.failing` once that -// work lands will show the gap is closed. The assertions below state the DESIRED behavior. +// buildThemes builds each theme as a map-step unit (CPOUI5FOUNDATION-1363), so adding a marker now +// rebuilds only the newly enabled theme, and removing a marker rebuilds nothing: the removed input +// yields a delta (ResourceRequestManager.getDeltas includes removed paths), the owning unit drops out, +// and its stale output is dropped from the carried-forward stage while the other theme stays cached. -test.serial.failing( +test.serial( "buildThemes: adding a library rebuilds only the newly enabled theme, others stay cached", async (t) => { const fixtureTester = new FixtureTester(t, "application.a"); @@ -96,7 +95,7 @@ test.serial.failing( } }); -test.serial.failing( +test.serial( "buildThemes: removing a library removes only its theme, others stay cached", async (t) => { const fixtureTester = new FixtureTester(t, "application.a"); @@ -137,9 +136,8 @@ test.serial.failing( // removed. lib/one's theme is unaffected and should be reused from cache (not rewritten). await fixtureTester.setMultiLibraryThemeLibTwoMarker(false); - // #2 build (with cache, with changes): DESIRED — buildThemes does NOT rewrite lib/one's theme + // #2 build (with cache, with changes): buildThemes does NOT rewrite lib/one's theme // (empty written set for buildThemes; the survivor is carried forward from cache). - // Fails today: the whole task re-runs and rewrites lib/one's theme (3 files instead of 0). // (Only themelib.multi is rebuilt here; application.a is fully served from cache.) await fixtureTester.buildProject({ config: {destPath, cleanDest: true, dependencyIncludes: {includeAllDependencies: true}}, @@ -372,7 +370,7 @@ test.serial("Build application.a project multiple times", async (t) => { // marked test.failing because the delta path does not yet achieve it. See BuildServer.integration.js for // the same scenario over the served build, and the minify FIXME for why a fix needs the `.map` -> `.js` // relation, not a local pattern tweak. -test.serial.failing( +test.serial( "Build application.a, changing only an input source map read via fs by minify invalidates the debug source map", async (t) => { const fixtureTester = new FixtureTester(t, "application.a"); @@ -418,7 +416,9 @@ test.serial.failing( "generateFlexChangesBundle", "generateVersionInfo", // replaceCopyright is skipped because no copyright is configured in the project - "replaceCopyright" + "replaceCopyright", + // replaceVersion has no work for the changed .js.map and is skipped + "replaceVersion" // "minify" is NOT skipped: it re-runs in differential mode for the changed .js.map ] } @@ -987,3 +987,337 @@ resources: }, }); }); + +test.serial("Build application.a (custom task reads a root config file, tracked as a root input)", async (t) => { + const fixtureTester = new FixtureTester(t, "application.a"); + const destPath = fixtureTester.destPath; + await fixtureTester._initialize(); + + // A tsconfig.json in the project root: outside the UI5 resource model, so it is reachable only + // through getRootReader() and bypasses the source and dependency readers. The root-config custom + // task embeds its content into an output resource, so a change to it must invalidate the task's + // cache even though no source or dependency resource changed. + const tsconfigPath = `${fixtureTester.fixturePath}/tsconfig.json`; + const digestPath = `${destPath}/tsconfigDigest.js`; + await fs.writeFile(tsconfigPath, `{"compilerOptions":{"target":"es2022"}}`); + + // #1 build (no cache): the full graph builds and the task reads the initial tsconfig. + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5-customTask-root-config.yaml"}, + config: {destPath, cleanDest: true}, + assertions: { + projects: { + "library.d": {}, + "library.a": {}, + "library.b": {}, + "library.c": {}, + "application.a": {}, + }, + }, + }); + t.true((await fs.readFile(digestPath, {encoding: "utf8"})).includes("es2022"), + "Output embeds the initial tsconfig content"); + + // #2 build (with cache, no changes): the whole project is served from cache, nothing is built. + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5-customTask-root-config.yaml"}, + config: {destPath, cleanDest: true}, + assertions: {projects: {}}, + }); + + // Change only the root config. No source or dependency resource changes. + await fs.writeFile(tsconfigPath, `{"compilerOptions":{"target":"es2015"}}`); + + // #3 build (with cache, root config changed): the root change invalidates application.a's result + // cache, so it is rebuilt and the root-config task re-runs with the new content. Without root + // tracking this build would serve the stale cached result and build nothing. + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5-customTask-root-config.yaml"}, + config: {destPath, cleanDest: true}, + assertions: { + projects: { + "application.a": { + // Source is unchanged, so every source-driven task is served from cache. Only + // root-config re-runs, because its stage signature folds in the root resources. + skippedTasks: [ + "enhanceManifest", + "escapeNonAsciiCharacters", + "generateComponentPreload", + "generateFlexChangesBundle", + "generateVersionInfo", + "minify", + "replaceCopyright", + "replaceVersion", + ], + writtenResources: { + "root-config": ["/tsconfigDigest.js"], + }, + }, + }, + }, + }); + t.true((await fs.readFile(digestPath, {encoding: "utf8"})).includes("es2015"), + "Output embeds the changed tsconfig content after the root change"); + + // #4 build (with cache, no changes): fresh again, nothing is built. + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5-customTask-root-config.yaml"}, + config: {destPath, cleanDest: true}, + assertions: {projects: {}}, + }); +}); + +// generateThemeDesignerResources opens with a scalar "scan" step whose body globs +// `library.source.less` to decide whether the library has any themes, and writes that verdict into the +// library `.theming` as the bIgnore flag (bIgnore true == no themes, so the SAP Theme Designer skips the +// library). A scalar step is a single implicit unit, so the per-unit reads delta cannot prune or select +// it, and that delta cannot see a file that newly matches the glob: the recorder stores resolved paths, +// not patterns, so a file absent on the previous build appears in no recorded read. Re-running the scalar +// step on any delta verdict (the owning stage signature does change, because the stage-level monitor +// recorded the glob) is what flips the verdict. The task is gated on isFrameworkProject(), so the +// `library.framework` fixture carries an `@openui5/` package id; it declares no framework version and no +// framework libraries, so graph enrichment resolves no framework and the build stays hermetic. The task +// is off by default (composeTaskList), so each build opts in through includedTasks. +// +// On a cleanDest rebuild a generateThemeDesignerResources served from cache would restore the previous +// build's `.theming`, so a flipped bIgnore flag is proof the scalar step re-ran this build. +const generateThemeDesignerResourcesTask = "generateThemeDesignerResources"; + +// Per-task build status of one project from the recorded project-build-status events, so a test can +// assert a single task re-ran (task-start) rather than being served from cache (task-skip). +function taskStatusOf(t, projectName) { + const started = new Set(); + const skipped = new Set(); + for (const [event] of t.context.projectBuildStatusEventStub.args) { + if (event.projectName !== projectName) { + continue; + } + if (event.status === "task-start") { + started.add(event.taskName); + } else if (event.status === "task-skip") { + skipped.add(event.taskName); + } + } + return {started, skipped}; +} + +async function readTheming(destPath) { + return JSON.parse(await fs.readFile( + `${destPath}/resources/library/framework/.theming`, {encoding: "utf8"})); +} + +const SELF_CONTAINED_LESS = `@mycolor: blue;\n.sapUiBody {\n\tbackground-color: @mycolor;\n}\n`; + +test.serial( + "generateThemeDesignerResources: adding the first theme re-runs the scalar scan on a delta build", + async (t) => { + const fixtureTester = new FixtureTester(t, "library.framework"); + const destPath = fixtureTester.destPath; + const includedTasks = [generateThemeDesignerResourcesTask]; + const themeSourcePath = + `${fixtureTester.fixturePath}/main/src/library/framework/themes/my_theme/library.source.less`; + + // #1 build (fills the cache): the library has no themes, so scan reports none and the library + // `.theming` carries bIgnore. + await fixtureTester.buildProject({ + config: {destPath, cleanDest: false, includedTasks}, + assertions: {projects: {"library.framework": {}}}, + }); + t.is((await readTheming(destPath)).bIgnore, true, + "Initial library .theming reports the library has no themes"); + + // Add the first theme. Its `library.source.less` newly matches scan's glob, whose result was empty + // on build #1. + await fs.mkdir(`${fixtureTester.fixturePath}/main/src/library/framework/themes/my_theme`, + {recursive: true}); + await fs.writeFile(themeSourcePath, SELF_CONTAINED_LESS); + + // #2 build (with cache, with changes): a delta build where unaffected tasks stay cached, yet the + // scalar scan step re-runs and flips hasThemes. + await fixtureTester.buildProject({ + config: {destPath, cleanDest: true, includedTasks}, + }); + const status = taskStatusOf(t, "library.framework"); + t.true(status.skipped.has("minify"), + "Delta build: a source-unaffected task is served from cache"); + t.true(status.started.has(generateThemeDesignerResourcesTask), + "generateThemeDesignerResources re-ran as a step-based task"); + t.false(status.skipped.has(generateThemeDesignerResourcesTask), + "generateThemeDesignerResources was not served from cache"); + t.is((await readTheming(destPath)).bIgnore, undefined, + "After adding the first theme the library .theming reports the library HAS themes"); + // buildThemes generated the newly added theme's CSS on the same delta build. + await t.notThrowsAsync( + fs.readFile(`${destPath}/resources/library/framework/themes/my_theme/library.css`, + {encoding: "utf8"}), + "The newly added theme was built"); + }); + +test.serial( + "generateThemeDesignerResources: removing the last theme re-runs the scalar scan on a delta build", + async (t) => { + const fixtureTester = new FixtureTester(t, "library.framework"); + const destPath = fixtureTester.destPath; + const includedTasks = [generateThemeDesignerResourcesTask]; + const themeSourcePath = + `${fixtureTester.fixturePath}/main/src/library/framework/themes/my_theme/library.source.less`; + + // Ship the fixture with one theme present before the first build fills the cache. + await fixtureTester._initialize(); + await fs.mkdir(`${fixtureTester.fixturePath}/main/src/library/framework/themes/my_theme`, + {recursive: true}); + await fs.writeFile(themeSourcePath, SELF_CONTAINED_LESS); + + // #1 build (fills the cache): the library has a theme, so scan reports themes and the library + // `.theming` carries no bIgnore. + await fixtureTester.buildProject({ + config: {destPath, cleanDest: false, includedTasks}, + }); + t.is((await readTheming(destPath)).bIgnore, undefined, + "Initial library .theming reports the library HAS themes"); + + // Remove the only theme. Its `library.source.less` yields a delta (a removed path). + await fs.rm(`${fixtureTester.fixturePath}/main/src/library/framework/themes`, + {recursive: true, force: true}); + + // #2 build (with cache, with changes): the scalar scan step re-runs and flips hasThemes back. + await fixtureTester.buildProject({ + config: {destPath, cleanDest: true, includedTasks}, + }); + const status = taskStatusOf(t, "library.framework"); + t.true(status.started.has(generateThemeDesignerResourcesTask), + "generateThemeDesignerResources re-ran as a step-based task"); + t.false(status.skipped.has(generateThemeDesignerResourcesTask), + "generateThemeDesignerResources was not served from cache"); + t.is((await readTheming(destPath)).bIgnore, true, + "After removing the last theme the library .theming reports the library has no themes"); + // The removed theme's CSS is gone from the built output. + await t.throwsAsync( + fs.readFile(`${destPath}/resources/library/framework/themes/my_theme/library.css`, + {encoding: "utf8"}), + undefined, "The removed theme is no longer built"); + }); + +// task.root-conditional reads /tsconfig.json through the project root reader only while /toggle.js +// exists in the workspace. Removing /toggle.js is a source change that re-runs the stage; on that +// re-run the task reads no root resource, so its previously recorded root request is cleared and the +// emptied request set is re-persisted. A later change to /tsconfig.json must then NOT invalidate the +// task, because the stage no longer reads that file. Without clearing, the stale root request survives +// in the cache, keeps folding /tsconfig.json into the stage signature, and every edit to it rebuilds +// application.a. +test.serial( + "Build application.a (a stage that stops reading a root file stops being invalidated by it)", + async (t) => { + const fixtureTester = new FixtureTester(t, "application.a"); + const destPath = fixtureTester.destPath; + await fixtureTester._initialize(); + + const tsconfigPath = `${fixtureTester.fixturePath}/tsconfig.json`; + const togglePath = `${fixtureTester.fixturePath}/webapp/toggle.js`; + const digestPath = `${destPath}/rootConditionalDigest.js`; + const graphConfig = {rootConfigPath: "ui5-customTask-root-conditional.yaml"}; + await fs.writeFile(tsconfigPath, `{"compilerOptions":{"target":"es2022"}}`); + await fs.writeFile(togglePath, `sap.ui.define([], () => {});\n`); + + // #1 build (no cache): /toggle.js is present, so the task reads and records /tsconfig.json. + await fixtureTester.buildProject({graphConfig, config: {destPath, cleanDest: true}}); + t.true((await fs.readFile(digestPath, {encoding: "utf8"})).includes("es2022"), + "Output embeds the tsconfig content while the root read is active"); + + // Remove /toggle.js. Its deletion re-runs the root-conditional stage, and on that re-run the + // task reads no root resource. + await fs.rm(togglePath); + + // #2 build (cache, toggle removed): the stage re-runs, records no root read, so its stale root + // request is cleared and the emptied set is persisted. + await fixtureTester.buildProject({graphConfig, config: {destPath, cleanDest: true}}); + t.is(await fs.readFile(digestPath, {encoding: "utf8"}), + `export const content = "root-not-read";\n`, + "Output no longer embeds the tsconfig content once the root read stopped"); + + // Change only /tsconfig.json. The task no longer reads it. + await fs.writeFile(tsconfigPath, `{"compilerOptions":{"target":"es2015"}}`); + + // #3 build (cache, tsconfig changed, toggle still absent): application.a is a full result-cache + // hit and nothing rebuilds. Without the fix the stale root request would still fold tsconfig.json + // into the stage signature, invalidating application.a and rebuilding it. + await fixtureTester.buildProject({ + graphConfig, config: {destPath, cleanDest: true}, + assertions: {projects: {}}, + }); + }); + +// Two custom tasks read the project root: task.root-config reads /tsconfig.json by path with the default +// gitignore filter (useGitignore:true), task.root-glob globs /rootcfg/**/*.json with the filter disabled +// (useGitignore:false, recorded against the second root manager). Each task's root reads fold into its own +// stage's root signature, and the per-stage root signatures aggregate at the result-cache level, so a +// change to one task's root input re-runs only that task. Adding or removing a file matching root-glob's +// glob invalidates root-glob (root indices refresh by re-globbing), while root-config stays cached. +const ROOT_MULTI_SKIPPED_SOURCE_TASKS = [ + "enhanceManifest", "escapeNonAsciiCharacters", "generateComponentPreload", + "generateFlexChangesBundle", "generateVersionInfo", "minify", "replaceCopyright", "replaceVersion", +]; +test.serial( + "Build application.a (root reads across two stages invalidate independently; glob + useGitignore:false)", + async (t) => { + const fixtureTester = new FixtureTester(t, "application.a"); + const destPath = fixtureTester.destPath; + await fixtureTester._initialize(); + + const tsconfigPath = `${fixtureTester.fixturePath}/tsconfig.json`; + const cfgDir = `${fixtureTester.fixturePath}/rootcfg`; + const globDigestPath = `${destPath}/rootGlobDigest.js`; + const graphConfig = {rootConfigPath: "ui5-customTask-root-multi.yaml"}; + await fs.writeFile(tsconfigPath, `{"compilerOptions":{"target":"es2022"}}`); + await fs.mkdir(cfgDir, {recursive: true}); + await fs.writeFile(`${cfgDir}/a.json`, `{"a":1}`); + await fs.writeFile(`${cfgDir}/b.json`, `{"b":2}`); + + // #1 build (no cache): both tasks read and record their root reads. + await fixtureTester.buildProject({graphConfig, config: {destPath, cleanDest: true}}); + let globDigest = await fs.readFile(globDigestPath, {encoding: "utf8"}); + t.true(globDigest.includes("a.json") && globDigest.includes("b.json"), + "root-glob output lists both matching root files"); + + // #2 build (cache, no changes): full result-cache hit. + await fixtureTester.buildProject({graphConfig, config: {destPath, cleanDest: true}, + assertions: {projects: {}}}); + + // Delete a matching root file (a novel state, never cached): root-glob must re-run, root-config + // stays cached. + await fs.rm(`${cfgDir}/b.json`); + await fixtureTester.buildProject({ + graphConfig, config: {destPath, cleanDest: true}, + assertions: {projects: {"application.a": { + skippedTasks: [...ROOT_MULTI_SKIPPED_SOURCE_TASKS, "root-config"], + writtenResources: {"root-glob": ["/rootGlobDigest.js"]}, + }}}, + }); + globDigest = await fs.readFile(globDigestPath, {encoding: "utf8"}); + t.false(globDigest.includes("b.json"), "root-glob output drops the removed root file"); + + // Add a matching root file (again a novel state): root-glob must re-run, root-config stays cached. + await fs.writeFile(`${cfgDir}/c.json`, `{"c":3}`); + await fixtureTester.buildProject({ + graphConfig, config: {destPath, cleanDest: true}, + assertions: {projects: {"application.a": { + skippedTasks: [...ROOT_MULTI_SKIPPED_SOURCE_TASKS, "root-config"], + writtenResources: {"root-glob": ["/rootGlobDigest.js"]}, + }}}, + }); + globDigest = await fs.readFile(globDigestPath, {encoding: "utf8"}); + t.true(globDigest.includes("a.json") && globDigest.includes("c.json"), + "root-glob output lists the newly added root file"); + + // Change only /tsconfig.json: now root-config must re-run, root-glob stays cached. This also shows + // the per-stage root signatures aggregate independently, so one task's root change does not re-run + // the other. + await fs.writeFile(tsconfigPath, `{"compilerOptions":{"target":"es2015"}}`); + await fixtureTester.buildProject({ + graphConfig, config: {destPath, cleanDest: true}, + assertions: {projects: {"application.a": { + skippedTasks: [...ROOT_MULTI_SKIPPED_SOURCE_TASKS, "root-glob"], + writtenResources: {"root-config": ["/tsconfigDigest.js"]}, + }}}, + }); + }); diff --git a/packages/project/test/lib/build/ProjectBuilder.customTasks.integration.js b/packages/project/test/lib/build/ProjectBuilder.customTasks.integration.js index 5cbb1705f21..10f2ffafce7 100644 --- a/packages/project/test/lib/build/ProjectBuilder.customTasks.integration.js +++ b/packages/project/test/lib/build/ProjectBuilder.customTasks.integration.js @@ -504,3 +504,136 @@ builder: } }); }); + +// The `.out` a unit of the step-based custom task fixture writes per `.src` key (see task.step-based.js). +// procEachOut is the virtual path recorded in writtenResources; procEachDist is the on-disk location, +// where an application build has dropped the `/resources/id1/` namespace prefix. +const procEachOut = (name) => `/resources/id1/procEach/${name}.out`; +const procEachDist = (destPath, name) => `${destPath}/procEach/${name}.out`; + +test.serial("Build application.a (step-based custom task with per-step delta caching)", async (t) => { + const fixtureTester = new FixtureTester(t, "application.a"); + const destPath = fixtureTester.destPath; + await fixtureTester._initialize(); + + // The custom task "step-based-task" (Specification Version 5.0, a step-based factory) runs one + // cached map-step unit per `.src` file. Each unit reads its sibling `.dep` through the step workspace, + // so that `.dep` is a tracked input of the owning unit alone. Changing only `a.dep` must re-run only + // a's unit and leave b's unit served from cache, proving per-step delta caching for a custom task. + const procEachDir = `${fixtureTester.fixturePath}/webapp/procEach`; + await fs.mkdir(procEachDir, {recursive: true}); + await fs.writeFile(`${procEachDir}/a.src`, "source-a"); + await fs.writeFile(`${procEachDir}/b.src`, "source-b"); + await fs.writeFile(`${procEachDir}/a.dep`, "dep-a-v1"); + await fs.writeFile(`${procEachDir}/b.dep`, "dep-b-v1"); + + // #1 build (no cache): both steps run, so the task writes both `.out` files. + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5-customTask-stepBased.yaml"}, + config: {destPath, cleanDest: true}, + assertions: { + projects: { + "library.d": {}, + "library.a": {}, + "library.b": {}, + "library.c": {}, + "application.a": { + writtenResources: { + "step-based-task": [procEachOut("a"), procEachOut("b")], + }, + }, + }, + }, + }); + + // Both outputs reflect their v1 dep content. + t.is(await fs.readFile(procEachDist(destPath, "a"), {encoding: "utf8"}), "source-a\n// dep: dep-a-v1\n"); + t.is(await fs.readFile(procEachDist(destPath, "b"), {encoding: "utf8"}), "source-b\n// dep: dep-b-v1\n"); + + // Change ONLY a's cross-resource input. a.src is untouched, so a's step re-runs solely because its + // recorded read of a.dep changed. b's step reads b.dep (unchanged) and stays cached. + await fs.writeFile(`${procEachDir}/a.dep`, "dep-a-v2"); + + // #2 build (with cache, with changes): the task re-runs in delta mode and writes ONLY a's `.out`. + // Only a.dep changed and only the step-based-task reads `.dep` files, so every standard task is + // served from cache and only application.a is rebuilt. + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5-customTask-stepBased.yaml"}, + config: {destPath, cleanDest: true}, + assertions: { + projects: { + "application.a": { + skippedTasks: [ + "escapeNonAsciiCharacters", + "replaceCopyright", + "enhanceManifest", + "generateFlexChangesBundle", + "generateVersionInfo", + "minify", + "replaceVersion", + "generateComponentPreload", + ], + writtenResources: { + "step-based-task": [procEachOut("a")], + }, + }, + }, + }, + }); + + // a's output reflects the new dep; b's output is carried forward from cache unchanged. + t.is(await fs.readFile(procEachDist(destPath, "a"), {encoding: "utf8"}), "source-a\n// dep: dep-a-v2\n"); + t.is(await fs.readFile(procEachDist(destPath, "b"), {encoding: "utf8"}), "source-b\n// dep: dep-b-v1\n"); + + // #3 build (with cache, no changes): everything is served from cache, including the custom task. + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5-customTask-stepBased.yaml"}, + config: {destPath, cleanDest: true}, + assertions: { + projects: {} + } + }); +}); + + +// A map step's keys() enumerator owns no key of its own, so anything it does is attributed to the stage +// rather than to a unit. For resource tags that matters across a fully cached stage: the enumerator does +// not run at all, and the tag has to come back from the stage's recorded tag operations. +test.serial("Build application.a (step-based custom task: a tag set in keys() survives a cached stage)", + async (t) => { + const fixtureTester = new FixtureTester(t, "application.a"); + const destPath = fixtureTester.destPath; + await fixtureTester._initialize(); + + // The fixture's keys() enumerator tags every `.omitme` file with OmitFromBuildResult while it + // globs for `.src` keys, so `keep.omitme` must never reach the build result. + const procEachDir = `${fixtureTester.fixturePath}/webapp/procEach`; + await fs.mkdir(procEachDir, {recursive: true}); + await fs.writeFile(`${procEachDir}/a.src`, "source-a"); + await fs.writeFile(`${procEachDir}/a.dep`, "dep-a-v1"); + await fs.writeFile(`${procEachDir}/keep.omitme`, "should not reach the build result"); + + // #1 build (no cache): the enumerator runs and sets the tag live. + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5-customTask-stepBased.yaml"}, + config: {destPath, cleanDest: true}, + }); + await t.throwsAsync(fs.readFile(`${destPath}/procEach/keep.omitme`, {encoding: "utf8"}), + undefined, "#1 build: the tagged resource was omitted from the build result"); + + // #2 build: an unrelated source file changed, so application.a rebuilds, but nothing the step read + // changed. Its stage is a full cache hit, which means keys() never runs and the tag can only come + // from the restored stage. + await fs.writeFile(`${fixtureTester.fixturePath}/webapp/unrelated.js`, "console.log(\"unrelated\");"); + await fixtureTester.buildProject({ + graphConfig: {rootConfigPath: "ui5-customTask-stepBased.yaml"}, + config: {destPath, cleanDest: true}, + }); + const skippedTasks = t.context.projectBuildStatusEventStub.args.map(([event]) => event) + .filter((event) => event.projectName === "application.a" && event.status === "task-skip") + .map((event) => event.taskName); + t.true(skippedTasks.includes("step-based-task"), + `#2 build: the step's stage was served from cache (skipped: ${skippedTasks})`); + await t.throwsAsync(fs.readFile(`${destPath}/procEach/keep.omitme`, {encoding: "utf8"}), + undefined, "#2 build (stage served from cache): the tagged resource is still omitted"); + }); diff --git a/packages/project/test/lib/build/ProjectBuilder.dependencies.integration.js b/packages/project/test/lib/build/ProjectBuilder.dependencies.integration.js index 2f697d054e1..0a5823bd844 100644 --- a/packages/project/test/lib/build/ProjectBuilder.dependencies.integration.js +++ b/packages/project/test/lib/build/ProjectBuilder.dependencies.integration.js @@ -222,14 +222,52 @@ test.serial("Build application.a (including only some dependencies)", async (t) await fs.writeFile(`${fixtureTester.fixturePath}/package.json`, JSON.stringify(packageJsonContent, null, 2)); // #4 build - // Build application.a again with "includeAllDependencies" - // and check with assertion "allProjects" that "library.d" isn't even seen: + // Build application.a again with "includeAllDependencies" and check with assertion "allProjects" + // that "library.d" isn't even seen. + // + // library.a, library.b and library.c each declare library.d as a dependency in their .library, so + // generateLibraryManifest embeds library.d's version as the dependency minVersion in their + // manifest.json (manifestCreator resolves it via taskUtil.getProject("library.d").getVersion()). + // Because that read is tracked as a task input, removing library.d changes the input and + // re-runs generateLibraryManifest (and the downstream generateLibraryManifest-dependent tasks) + // for library.a/b/c; their unaffected tasks stay cached. application.a rebuilds because its + // dependency set changed. await fixtureTester.buildProject({ config: {destPath, cleanDest: true, dependencyIncludes: {includeAllDependencies: true}}, assertions: { allProjects: ["library.a", "library.b", "library.c", "application.a"], projects: { + "library.a": { + skippedTasks: [ + "buildThemes", + "escapeNonAsciiCharacters", + "minify", + "replaceBuildtime", + "replaceCopyright", + "replaceVersion", + ] + }, + "library.b": { + skippedTasks: [ + "buildThemes", + "escapeNonAsciiCharacters", + "minify", + "replaceBuildtime", + "replaceCopyright", + "replaceVersion", + ] + }, + "library.c": { + skippedTasks: [ + "buildThemes", + "escapeNonAsciiCharacters", + "minify", + "replaceBuildtime", + "replaceCopyright", + "replaceVersion", + ] + }, "application.a": { skippedTasks: [ "enhanceManifest", diff --git a/packages/project/test/lib/build/TaskRunner.js b/packages/project/test/lib/build/TaskRunner.js index c06859aa01a..14c38a4a179 100644 --- a/packages/project/test/lib/build/TaskRunner.js +++ b/packages/project/test/lib/build/TaskRunner.js @@ -9,6 +9,19 @@ function emptyarray() { return []; } +// A task receives its taskUtil wrapped in a MonitoredTaskUtil, which records the inputs the task +// reads. The wrapper exposes getInputRecording() and delegates every other member to the underlying +// taskUtil, so a wrapped member reads back the underlying value. +function assertMonitoredTaskUtil(t, actual, {delegates} = {}) { + t.is(typeof actual.getInputRecording, "function", "task received a MonitoredTaskUtil"); + t.deepEqual(actual.getInputRecording(), [], "no task inputs recorded"); + if (delegates) { + for (const [key, value] of Object.entries(delegates)) { + t.is(actual[key], value, `MonitoredTaskUtil delegates '${key}' to the underlying taskUtil`); + } + } +} + const buildConfig = { selfContained: false, jsdoc: false, @@ -99,7 +112,7 @@ test.beforeEach(async (t) => { }; }, getRequiredDependenciesCallback: t.context.getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false), + getStepBased: async () => false, }; t.context.graph = { @@ -129,10 +142,15 @@ test.beforeEach(async (t) => { t.context.buildCache = { setTasks: sinon.stub(), - prepareTaskExecutionAndValidateCache: sinon.stub().resolves(false), - recordTaskResult: sinon.stub().resolves(), + prepareStageExecutionAndValidateCache: sinon.stub().resolves(false), + recordStageResult: sinon.stub().resolves(), allTasksCompleted: sinon.stub().resolves([]), - prefetchStageCache: sinon.stub(), + getStepInvocationData: sinon.stub().returns(undefined), + getStepReturnValueStore: sinon.stub().returns(undefined), + getResolveInputValue: sinon.stub().returns(undefined), + setStepInvocationData: sinon.stub(), + getStageId: sinon.stub().callsFake((taskName, stepName) => + stepName === undefined ? `task/${taskName}` : `task/${taskName}::step/${stepName}`), }; t.context.resourceFactory = { @@ -145,7 +163,7 @@ test.beforeEach(async (t) => { return { constructor: {name: "MonitoredReader"}, getName: () => name, - getResourceRequests: sinon.stub().returns([]) + getResourceRequests: sinon.stub().returns({paths: [], patterns: []}) }; } return resource; @@ -666,9 +684,9 @@ test("Custom task is called correctly", async (t) => { getTask: () => taskStub, getSpecVersion: () => mockSpecVersion, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); - t.context.taskUtil.getInterface.returns("taskUtil interface"); + t.context.taskUtil.getInterface.returns({isTaskUtilInterface: true}); const project = getMockProject("module"); project.getCustomTasks = () => [ {name: "myTask", configuration: "configuration"} @@ -685,12 +703,12 @@ test("Custom task is called correctly", async (t) => { await taskRunner._tasks["myTask"].task(); t.is(specVersionGteStub.callCount, 3, "SpecificationVersion#gte got called three times"); - t.is(specVersionGteStub.getCall(2).args[0], "3.0", - "SpecificationVersion#gte got called with correct arguments on third call (task execution)"); t.is(specVersionGteStub.getCall(0).args[0], "3.0", "SpecificationVersion#gte got called with correct arguments on first call"); t.is(specVersionGteStub.getCall(1).args[0], "5.0", - "SpecificationVersion#gte got called with correct arguments on second call (differential updates check)"); + "SpecificationVersion#gte got called with correct arguments on second call (step-based opt-in)"); + t.is(specVersionGteStub.getCall(2).args[0], "3.0", + "SpecificationVersion#gte got called with correct arguments on third call (task execution)"); t.is(createDependencyReaderStub.callCount, 1, "getDependenciesReader got called once"); t.deepEqual(createDependencyReaderStub.getCall(0).args[0], @@ -702,7 +720,7 @@ test("Custom task is called correctly", async (t) => { const taskArgs = taskStub.getCall(0).args[0]; t.is(taskArgs.workspace.constructor.name, "MonitoredReader", "workspace is MonitoredReader"); t.is(taskArgs.dependencies.constructor.name, "MonitoredReader", "dependencies is MonitoredReader"); - t.is(taskArgs.taskUtil, "taskUtil interface", "taskUtil is correct"); + assertMonitoredTaskUtil(t, taskArgs.taskUtil, {delegates: {isTaskUtilInterface: true}}); t.is(taskArgs.options.projectName, "project.b", "projectName is correct"); t.is(taskArgs.options.projectNamespace, "project/b", "projectNamespace is correct"); t.is(taskArgs.options.configuration, "configuration", "configuration is correct"); @@ -725,7 +743,7 @@ test("Custom task with legacy spec version", async (t) => { getTask: () => taskStub, getSpecVersion: () => mockSpecVersion, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); t.context.taskUtil.getInterface.returns(undefined); // simulating no taskUtil for old specVersion const project = getMockProject("module"); @@ -745,12 +763,12 @@ test("Custom task with legacy spec version", async (t) => { await taskRunner._tasks["myTask"].task(); t.is(specVersionGteStub.callCount, 3, "SpecificationVersion#gte got called three times"); - t.is(specVersionGteStub.getCall(2).args[0], "3.0", - "SpecificationVersion#gte got called with correct arguments on third call (task execution)"); t.is(specVersionGteStub.getCall(0).args[0], "3.0", "SpecificationVersion#gte got called with correct arguments on first call"); t.is(specVersionGteStub.getCall(1).args[0], "5.0", - "SpecificationVersion#gte got called with correct arguments on second call (differential updates check)"); + "SpecificationVersion#gte got called with correct arguments on second call (step-based opt-in)"); + t.is(specVersionGteStub.getCall(2).args[0], "3.0", + "SpecificationVersion#gte got called with correct arguments on third call (task execution)"); t.is(createDependencyReaderStub.callCount, 1, "getDependenciesReader got called once"); t.deepEqual(createDependencyReaderStub.getCall(0).args[0], @@ -785,7 +803,7 @@ test("Custom task with legacy spec version and requiredDependenciesCallback", as getTask: () => taskStub, getSpecVersion: () => mockSpecVersion, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); t.context.taskUtil.getInterface.returns(undefined); // simulating no taskUtil for old specVersion const project = getMockProject("module"); @@ -816,12 +834,12 @@ test("Custom task with legacy spec version and requiredDependenciesCallback", as await taskRunner._tasks["myTask"].task(); t.is(specVersionGteStub.callCount, 3, "SpecificationVersion#gte got called three times"); - t.is(specVersionGteStub.getCall(2).args[0], "3.0", - "SpecificationVersion#gte got called with correct arguments on third call (task execution)"); t.is(specVersionGteStub.getCall(0).args[0], "3.0", "SpecificationVersion#gte got called with correct arguments on first call"); t.is(specVersionGteStub.getCall(1).args[0], "5.0", - "SpecificationVersion#gte got called with correct arguments on second call (differential updates check)"); + "SpecificationVersion#gte got called with correct arguments on second call (step-based opt-in)"); + t.is(specVersionGteStub.getCall(2).args[0], "3.0", + "SpecificationVersion#gte got called with correct arguments on third call (task execution)"); t.is(createDependencyReaderStub.callCount, 1, "getDependenciesReader got called once"); t.deepEqual(createDependencyReaderStub.getCall(0).args[0], @@ -859,7 +877,7 @@ test("Custom task with specVersion 3.0", async (t) => { getTask: () => taskStub, getSpecVersion: () => mockSpecVersion, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); const project = getMockProject("module"); @@ -907,12 +925,12 @@ test("Custom task with specVersion 3.0", async (t) => { await taskRunner._tasks["myTask"].task(); t.is(specVersionGteStub.callCount, 3, "SpecificationVersion#gte got called three times"); - t.is(specVersionGteStub.getCall(2).args[0], "3.0", - "SpecificationVersion#gte got called with correct arguments on third call (task execution)"); t.is(specVersionGteStub.getCall(0).args[0], "3.0", "SpecificationVersion#gte got called with correct arguments on first call"); t.is(specVersionGteStub.getCall(1).args[0], "5.0", - "SpecificationVersion#gte got called with correct arguments on second call (differential updates check)"); + "SpecificationVersion#gte got called with correct arguments on second call (step-based opt-in)"); + t.is(specVersionGteStub.getCall(2).args[0], "3.0", + "SpecificationVersion#gte got called with correct arguments on third call (task execution)"); t.is(taskUtil.getInterface.callCount, 2, "taskUtil#getInterface got called twice"); t.is(taskUtil.getInterface.getCall(0).args[0], mockSpecVersion, @@ -931,7 +949,7 @@ test("Custom task with specVersion 3.0", async (t) => { t.is(taskArgs.workspace.constructor.name, "MonitoredReader", "workspace is MonitoredReader"); t.is(taskArgs.dependencies.constructor.name, "MonitoredReader", "dependencies is MonitoredReader"); t.is(taskArgs.log, "group logger", "log is correct"); - t.deepEqual(taskArgs.taskUtil, taskUtil, "taskUtil is correct"); + assertMonitoredTaskUtil(t, taskArgs.taskUtil); t.is(taskArgs.options.projectName, "project.b", "projectName is correct"); t.is(taskArgs.options.projectNamespace, "project/b", "projectNamespace is correct"); t.is(taskArgs.options.taskName, "myTask", "taskName is correct"); @@ -954,7 +972,7 @@ test("Custom task with specVersion 3.0 and no requiredDependenciesCallback", asy getTask: () => taskStub, getSpecVersion: () => mockSpecVersion, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); const project = getMockProject("module"); @@ -973,12 +991,12 @@ test("Custom task with specVersion 3.0 and no requiredDependenciesCallback", asy await taskRunner._tasks["myTask"].task(); t.is(specVersionGteStub.callCount, 3, "SpecificationVersion#gte got called three times"); - t.is(specVersionGteStub.getCall(2).args[0], "3.0", - "SpecificationVersion#gte got called with correct arguments on third call (task execution)"); t.is(specVersionGteStub.getCall(0).args[0], "3.0", "SpecificationVersion#gte got called with correct arguments on first call"); t.is(specVersionGteStub.getCall(1).args[0], "5.0", - "SpecificationVersion#gte got called with correct arguments on second call (differential updates check)"); + "SpecificationVersion#gte got called with correct arguments on second call (step-based opt-in)"); + t.is(specVersionGteStub.getCall(2).args[0], "3.0", + "SpecificationVersion#gte got called with correct arguments on third call (task execution)"); t.is(taskUtil.getInterface.callCount, 1, "taskUtil#getInterface got called once"); t.is(taskUtil.getInterface.getCall(0).args[0], mockSpecVersion, @@ -991,7 +1009,7 @@ test("Custom task with specVersion 3.0 and no requiredDependenciesCallback", asy const taskArgs = taskStub.getCall(0).args[0]; t.is(taskArgs.workspace.constructor.name, "MonitoredReader", "workspace is MonitoredReader"); t.is(taskArgs.log, "group logger", "log is correct"); - t.deepEqual(taskArgs.taskUtil, taskUtil, "taskUtil is correct"); + assertMonitoredTaskUtil(t, taskArgs.taskUtil); t.is(taskArgs.options.projectName, "project.b", "projectName is correct"); t.is(taskArgs.options.projectNamespace, "project/b", "projectNamespace is correct"); t.is(taskArgs.options.taskName, "myTask", "taskName is correct"); @@ -1032,28 +1050,28 @@ test("Multiple custom tasks with same name are called correctly", async (t) => { getTask: () => taskStubA, getSpecVersion: () => mockSpecVersionA, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); graph.getExtension.onSecondCall().returns({ getName: () => "Task Name B", getTask: () => taskStubB, getSpecVersion: () => mockSpecVersionB, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); graph.getExtension.onThirdCall().returns({ getName: () => "Task Name C", getTask: () => taskStubC, getSpecVersion: () => mockSpecVersionC, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); graph.getExtension.onCall(3).returns({ getName: () => "Task Name D", getTask: () => taskStubD, getSpecVersion: () => mockSpecVersionD, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); const project = getMockProject("module"); project.getCustomTasks = () => [ @@ -1173,7 +1191,7 @@ test("Multiple custom tasks with same name are called correctly", async (t) => { const taskCArgs = taskStubC.getCall(0).args[0]; t.is(taskCArgs.workspace.constructor.name, "MonitoredReader", "workspace is MonitoredReader"); t.is(taskCArgs.log, "group logger", "log is correct"); - t.deepEqual(taskCArgs.taskUtil, taskUtil, "taskUtil is correct"); + assertMonitoredTaskUtil(t, taskCArgs.taskUtil); t.is(taskCArgs.options.projectName, "project.b", "projectName is correct"); t.is(taskCArgs.options.projectNamespace, "project/b", "projectNamespace is correct"); t.is(taskCArgs.options.taskName, "myTask--3", "taskName is correct"); @@ -1185,7 +1203,7 @@ test("Multiple custom tasks with same name are called correctly", async (t) => { t.is(taskDArgs.workspace.constructor.name, "MonitoredReader", "workspace is MonitoredReader"); t.is(taskDArgs.dependencies.constructor.name, "MonitoredReader", "dependencies is MonitoredReader"); t.is(taskDArgs.log, "group logger", "log is correct"); - t.deepEqual(taskDArgs.taskUtil, taskUtil, "taskUtil is correct"); + assertMonitoredTaskUtil(t, taskDArgs.taskUtil); t.is(taskDArgs.options.projectName, "project.b", "projectName is correct"); t.is(taskDArgs.options.projectNamespace, "project/b", "projectNamespace is correct"); t.is(taskDArgs.options.taskName, "myTask--4", "taskName is correct"); @@ -1210,7 +1228,7 @@ test("Custom task: requiredDependenciesCallback returns unknown dependency", asy getTask: () => taskStub, getSpecVersion: () => mockSpecVersion, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); const project = getMockProject("module"); @@ -1246,7 +1264,7 @@ test("Custom task: requiredDependenciesCallback returns Array instead of Set", a getTask: () => taskStub, getSpecVersion: () => mockSpecVersion, getRequiredDependenciesCallback: getRequiredDependenciesCallbackStub, - getSupportsDifferentialBuildsCallback: sinon.stub().returns(() => false) + getStepBased: async () => false, }); const project = getMockProject("module"); @@ -1272,7 +1290,9 @@ test("Custom task attached to a disabled task", async (t) => { {name: "myTask", afterTask: "generateBundle", configuration: "dog"} ]; - taskRepository.getTask = sinon.stub().returns({task: sinon.stub()}); + // Standard tasks are step-based factories; the stub returns an empty step list so the step runner is + // a no-op and this test only exercises task ordering and the custom task's execution. + taskRepository.getTask = sinon.stub().returns({task: sinon.stub().returns([])}); customTask.getTask = () => customTaskFnStub; const taskRunner = createTaskRunner(t, project); @@ -1302,7 +1322,7 @@ test("Custom task attached to a disabled task", async (t) => { }); test.serial("_addTask", async (t) => { - const {sinon, taskUtil, taskRepository} = t.context; + const {sinon, taskRepository} = t.context; const taskStub = sinon.stub(); taskRepository.getTask.withArgs("standardTask").resolves({ @@ -1334,11 +1354,11 @@ test.serial("_addTask", async (t) => { t.is(taskCallArgs.workspace.constructor.name, "MonitoredReader", "workspace is MonitoredReader"); t.is(taskCallArgs.options.projectName, "project.b", "projectName is correct"); t.is(taskCallArgs.options.projectNamespace, "project/b", "projectNamespace is correct"); - t.is(taskCallArgs.taskUtil, taskUtil, "taskUtil is correct"); + assertMonitoredTaskUtil(t, taskCallArgs.taskUtil); }); test.serial("_addTask with options", async (t) => { - const {sinon, taskUtil, taskRepository} = t.context; + const {sinon, taskRepository} = t.context; const taskStub = sinon.stub(); const project = getMockProject("module"); @@ -1376,7 +1396,60 @@ test.serial("_addTask with options", async (t) => { t.is(taskCallArgs.options.projectName, "project.b", "projectName is correct"); t.is(taskCallArgs.options.projectNamespace, "project/b", "projectNamespace is correct"); t.is(taskCallArgs.options.myTaskOption, "cat", "myTaskOption is correct"); - t.is(taskCallArgs.taskUtil, taskUtil, "taskUtil is correct"); + assertMonitoredTaskUtil(t, taskCallArgs.taskUtil); +}); + +// A fake AbstractReader-like project reader. Answers byPath/byGlob for the pass-through path and +// exposes the _byPath/_byGlob hooks a real MonitoredReader delegates to once the reader is wrapped. +function fakeProjectReader(name) { + return { + getName: () => name, + byPath: async (virPath) => ({getPath: () => virPath}), + byGlob: async () => [], + _byPath: async (virPath) => ({getPath: () => virPath}), + _byGlob: async () => [], + }; +} + +test.serial("Folds taskUtil project-reader reads into the recorded resource requests", async (t) => { + const {sinon, taskUtil, buildCache} = t.context; + const project = getMockProject("module"); + + // getProject() (no arg) / "project.b" is the project being built; "dep.a" is a dependency. + taskUtil.getProject.callsFake((name) => { + if (name === undefined || name === "project.b") { + return {getName: () => "project.b", getReader: () => fakeProjectReader("project.b reader")}; + } + if (name === "dep.a") { + return {getName: () => "dep.a", getReader: () => fakeProjectReader("dep.a reader")}; + } + return undefined; + }); + + const taskStub = sinon.stub().callsFake(async (params) => { + await params.taskUtil.getProject().getReader().byPath("/resources/project/b/own.js"); + await params.taskUtil.getProject("dep.a").getReader().byGlob("/resources/dep/a/**"); + }); + + const taskRunner = createTaskRunner(t, project); + await taskRunner._initTasks(); + taskRunner._addTask("standardTask", {requiresDependencies: true, taskFunction: taskStub}); + + // Warm the cached dependencies reader (normally done by runTasks) + await taskRunner.getDependenciesReader(new Set(["dep.a", "dep.b"]), true); + await taskRunner._tasks["standardTask"].task(); + + t.is(taskStub.callCount, 1, "task executed"); + const {projectResourceRequests, dependencyResourceRequests} = + buildCache.recordStageResult.getCall(0).args[0]; + t.deepEqual(projectResourceRequests, { + paths: ["/resources/project/b/own.js"], + patterns: [], + }, "reads of the project being built are folded into the project resource requests"); + t.deepEqual(dependencyResourceRequests, { + paths: [], + patterns: ["/resources/dep/a/**"], + }, "reads of a dependency's reader are folded into the dependency resource requests"); }); test("_addTask: Duplicate task", async (t) => { @@ -1564,3 +1637,800 @@ test("getDependenciesReader: No dependencies required", async (t) => { t.is(res.getName(), "custom reader collection", "Shared (all-)dependency reader returned"); }); + +// Integration: a step-based standard task built on the real MonitoredTaskUtil + StepRunner. A per-step +// non-resource input change (an env var one step reads) must re-run only that step, and a step served from +// cache must replay its recorded tag operations into the project tag collection. +test("Step-based task: a per-step input change re-runs only that step; a restored step replays its tags", + async (t) => { + const {sinon, projectBuildLogger} = t.context; + + // A mutable environment the step reads per key; the resolver re-derives the current value on the + // delta build the same way the real ProjectBuildContext does. + const env = {a: "1", b: "1"}; + const resolveInputValue = (type, name) => (type === "env" ? env[name] : undefined); + + // The taskUtil the per-step MonitoredTaskUtil wraps: getEnv is a tracked input, setTag passes + // through to reach the tag collection when a step actually runs. + const setTag = sinon.stub(); + const taskUtil = { + isRootProject: sinon.stub().returns(true), + getDependencies: sinon.stub().returns([]), + getInterface: sinon.stub(), + getEnv: (name) => env[name], + setTag, + }; + taskUtil.getInterface.returns(taskUtil); + + const ran = []; + // A step factory: one map step keyed by "a"/"b" whose each reads an env var and tags its output. + const build = () => [{ + name: "stepGroup", + keys: async () => ["a", "b"], + each: async (key, {taskUtil}) => { + ran.push(key); + taskUtil.getEnv(key); + taskUtil.setTag({getPath: () => `/out/${key}`}, "ui5:IsBundle", true); + }, + }]; + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map([ + ["stepTask", + {requiresDependencies: false, stepBased: true, options: {}, taskFunction: build}], + ]), + customTasks: new Map(), + }), + }; + + let capturedInvocationData; + let deltaMode = false; + const buildCache = { + setTasks: sinon.stub(), + recordStageResult: sinon.stub().resolves(), + allTasksCompleted: sinon.stub().resolves([]), + getStageId: (taskName, stepName) => + stepName === undefined ? `task/${taskName}` : `task/${taskName}::step/${stepName}`, + prepareStageExecutionAndValidateCache: sinon.stub().callsFake(async () => + (deltaMode ? {changedProjectResourcePaths: [], changedDependencyResourcePaths: []} : false)), + getStepInvocationData: sinon.stub().callsFake(() => capturedInvocationData), + getStepReturnValueStore: sinon.stub().returns(undefined), + getResolveInputValue: sinon.stub().returns(resolveInputValue), + setStepInvocationData: sinon.stub().callsFake((name, data) => { + capturedInvocationData = data; + }), + }; + + const replayTagOperations = sinon.stub(); + const project = getMockProject("module"); + project.getProjectResources = () => ({replayTagOperations}); + + const taskRunner = createTaskRunner(t, project, {taskUtil, buildCache, taskDefinitions}); + await taskRunner._initTasks(); + + // Build 1 (full): both steps run and record their env input and tag operation. + await taskRunner._tasks["stepTask"].task(projectBuildLogger); + t.deepEqual(ran, ["a", "b"], "The full build ran every step"); + t.is(replayTagOperations.callCount, 0, "A full build restores no step, so nothing is replayed"); + + // Build 2 (delta): only env var "a" changed, so step "a" re-runs and step "b" is restored. + ran.length = 0; + setTag.resetHistory(); + deltaMode = true; + env.a = "2"; + await taskRunner._tasks["stepTask"].task(projectBuildLogger); + + t.deepEqual(ran, ["a"], "Only the step whose env input changed re-ran on the delta build"); + t.is(setTag.callCount, 1, "Only the re-run step set its tag live"); + t.is(setTag.getCall(0).args[0].getPath(), "/out/a", "The re-run step's live setTag targeted its own output"); + t.is(replayTagOperations.callCount, 1, "The restored step replayed its tag operations"); + t.deepEqual(replayTagOperations.getCall(0).args[0], + [{op: "set", path: "/out/b", tag: "ui5:IsBundle", value: true}], + "The restored step's recorded tag operation was replayed, so its tag survives"); + }); + +// Builds a custom task extension stub whose spec version is driven by the given gte(version) result and +// whose step-based opt-in is the given flag. +function createCustomTaskExtension(sinon, {taskFunction, gte, stepBased = false}) { + return { + getName: () => "myCustom", + getSpecVersion: () => ({gte}), + getTask: async () => taskFunction, + getRequiredDependenciesCallback: sinon.stub().resolves(undefined), + getStepBased: async () => stepBased, + }; +} + +test("Step-based task: a full stage-cache hit re-runs a read-free consumer when its producer's return " + + "changed", async (t) => { + const {sinon, projectBuildLogger} = t.context; + + // A mutable env the producer reads; the resolver re-derives its current value on the delta build. + const env = {x: "1"}; + const resolveInputValue = (type, name) => (type === "env" ? env[name] : undefined); + const taskUtil = { + isRootProject: sinon.stub().returns(true), + getDependencies: sinon.stub().returns([]), + getInterface: sinon.stub(), + getEnv: (name) => env[name], + }; + taskUtil.getInterface.returns(taskUtil); + + const ran = []; + // A scalar producer 'scan' reads env x and returns it; a read-free scalar consumer 'use' consumes the + // producer return via needs and writes nothing observable to a reader. 'use' has a constant stage + // signature, so its verdict is a full hit even when 'scan' re-ran with a changed return. + const build = () => [ + {name: "scan", run: async ({taskUtil}) => ({v: taskUtil.getEnv("x")})}, + {name: "use", needs: ["scan"], run: async ({needs}) => { + ran.push(`use:${needs.scan.v}`); + }}, + ]; + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map([ + ["stepTask", {requiresDependencies: false, stepBased: true, options: {}, taskFunction: build}], + ]), + customTasks: new Map(), + }), + }; + + // Per-stage invocation data, keyed by stage id so 'scan' and 'use' carry their own data forward. + const invocationByStage = new Map(); + const getStageId = (taskName, stepName) => + stepName === undefined ? `task/${taskName}` : `task/${taskName}::step/${stepName}`; + // Build-2 verdicts per step: 'scan' is a delta (so its recorded env input re-selects it), 'use' is a + // full stage-cache hit. + let verdicts = {}; + const buildCache = { + setTasks: sinon.stub(), + recordStageResult: sinon.stub().resolves(), + allTasksCompleted: sinon.stub().resolves([]), + getStageId, + prepareStageExecutionAndValidateCache: sinon.stub().callsFake(async (taskName, stepName) => + (stepName in verdicts ? verdicts[stepName] : false)), + getStepInvocationData: sinon.stub().callsFake((stageId) => invocationByStage.get(stageId)), + getStepReturnValueStore: sinon.stub().returns(undefined), + getResolveInputValue: sinon.stub().returns(resolveInputValue), + setStepInvocationData: sinon.stub().callsFake((stageId, data) => { + invocationByStage.set(stageId, data); + }), + reopenStageForRerun: sinon.stub(), + }; + + const project = getMockProject("module"); + project.getProjectResources = () => ({replayTagOperations: sinon.stub()}); + + const taskRunner = createTaskRunner(t, project, {taskUtil, buildCache, taskDefinitions}); + await taskRunner._initTasks(); + + // Build 1 (full): both steps run; scan returns {v:"1"}, use records scan's return signature. + await taskRunner._tasks["stepTask"].task(projectBuildLogger); + t.deepEqual(ran, ["use:1"], "The full build ran the consumer with the producer's initial return"); + + // Build 2: env x changed, so scan re-runs via delta and its return advances to {v:"2"}. use is a full + // stage-cache hit, but its consumed needs return changed, so it must re-run. + ran.length = 0; + env.x = "2"; + verdicts = { + scan: {changedProjectResourcePaths: [], changedDependencyResourcePaths: []}, + use: true, + }; + await taskRunner._tasks["stepTask"].task(projectBuildLogger); + + t.deepEqual(ran, ["use:2"], + "The read-free consumer re-ran despite a full stage-cache hit, with the producer's fresh return"); + t.is(buildCache.reopenStageForRerun.callCount, 1, "The consumer's stage was reopened for the re-run"); + t.deepEqual(buildCache.reopenStageForRerun.getCall(0).args, ["stepTask", "use"], + "reopenStageForRerun targeted the consumer step's stage"); +}); + +// runTasks calls a step-based task's factory once, at discovery, to enumerate its step names for setTasks, +// and keeps the returned step array on the task so execution reuses it instead of calling the factory again. +// A factory may branch on options.projectNamespace (generateThemeDesignerResources emits its libraryTheming +// step only for a namespace), so discovery has to see a complete options object. Otherwise it misses a stage +// that the step runner then asks for, and the build fails on the missing stage. +test("Step-based task: the factory runs once per build and its steps are reused for execution", async (t) => { + const {sinon, taskUtil} = t.context; + + const factoryOptions = []; + // Mirrors the generateThemeDesignerResources shape: a step that only exists for a namespace. + const build = (options) => { + factoryOptions.push({...options}); + const steps = [{name: "scan", run: async () => undefined}]; + if (options.projectNamespace) { + steps.push({name: "namespaced", run: async () => undefined}); + } + return steps; + }; + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map([ + ["stepTask", + {requiresDependencies: false, stepBased: true, options: {}, taskFunction: build}], + ]), + customTasks: new Map(), + }), + }; + + // Only stages that setTasks created can be prepared, as in ProjectResources#useStage. + const createdStages = new Set(); + const buildCache = { + ...t.context.buildCache, + setTasks: sinon.stub().callsFake((tasks) => { + for (const {taskName, stepNames} of tasks) { + for (const stepName of stepNames ?? [undefined]) { + createdStages.add(buildCache.getStageId(taskName, stepName)); + } + } + }), + prepareStageExecutionAndValidateCache: sinon.stub().callsFake(async (taskName, stepName) => { + const stageId = buildCache.getStageId(taskName, stepName); + if (!createdStages.has(stageId)) { + throw new Error(`Stage '${stageId}' does not exist`); + } + return false; + }), + }; + + const project = getMockProject("module"); + project.getProjectResources = () => ({replayTagOperations: sinon.stub()}); + + const taskRunner = createTaskRunner(t, project, {taskUtil, buildCache, taskDefinitions}); + sinon.stub(taskRunner, "getDependenciesReader").resolves({getName: () => "dependencies"}); + + await t.notThrowsAsync(taskRunner.runTasks(), + "The step the factory emits for a namespace has a stage, so preparing it succeeds"); + + t.is(factoryOptions.length, 1, + "The factory was called once at discovery; execution reused the discovered step array"); + t.is(factoryOptions[0].projectNamespace, "project/b", + "Step discovery already saw the project namespace"); + t.deepEqual(buildCache.setTasks.firstCall.firstArg, + [{taskName: "stepTask", stepNames: ["scan", "namespaced"]}], + "A stage was created for every step the factory emits"); +}); + +// A step factory that forgets its return yields undefined, the single most likely authoring mistake. +// Discovery validates the whole step list (validateSteps) before it derives the step names and creates +// stages, so the container shape is validated here, naming the task and the value the factory returned. +test("Step-based task: a factory returning a non-array throws a named error at discovery", async (t) => { + const {sinon, taskUtil} = t.context; + + const build = () => undefined; // missing return + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map([ + ["stepTask", + {requiresDependencies: false, stepBased: true, options: {}, taskFunction: build}], + ]), + customTasks: new Map(), + }), + }; + + const project = getMockProject("module"); + const taskRunner = createTaskRunner(t, project, {taskUtil, taskDefinitions}); + sinon.stub(taskRunner, "getDependenciesReader").resolves({getName: () => "dependencies"}); + + const err = await t.throwsAsync(taskRunner.runTasks()); + t.is(err.message, + "Step factory for task 'stepTask' must return an array of step objects, got undefined", + "The error names the task, the expected shape, and the actual returned value"); +}); + +// A duplicate step name would create two stages with the same stage id (setTasks derives one id per step +// name), so discovery must reject it before setTasks runs — not later when the step runs and the corrupt +// stage already exists. The spied setTasks proves validation precedes stage creation. +test("Step-based task: a duplicate step name throws at discovery, before setTasks", async (t) => { + const {sinon, taskUtil, buildCache} = t.context; + + const build = () => [ + {name: "dup", run: async () => {}}, + {name: "dup", run: async () => {}}, + ]; + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map([ + ["stepTask", + {requiresDependencies: false, stepBased: true, options: {}, taskFunction: build}], + ]), + customTasks: new Map(), + }), + }; + + const project = getMockProject("module"); + const taskRunner = createTaskRunner(t, project, {taskUtil, taskDefinitions}); + sinon.stub(taskRunner, "getDependenciesReader").resolves({getName: () => "dependencies"}); + + const err = await t.throwsAsync(taskRunner.runTasks()); + t.is(err.message, "Duplicate step name 'dup' for task 'stepTask'", + "The error names the duplicate step and the task"); + t.false(buildCache.setTasks.called, "No stage was created: validation ran before setTasks"); +}); + +// The factory is called once per build, so discovery and execution share one step array by construction. +// There is no second call for an impure factory (an untracked environment or clock read in its body) to +// diverge on: the discovered steps are the ones that run. This records the deliberate removal of the former +// discovery-versus-execution check, which only ever caught a divergence between two calls. +test("Step-based task: the factory is called once, so discovery and execution cannot diverge", async (t) => { + const {sinon, projectBuildLogger, taskUtil} = t.context; + + let callCount = 0; + // An impure factory that would add a step on a second call. With one call per build, the second output + // never happens, so the step runner only ever sees the discovered ["scan"]. + const build = () => { + callCount++; + const steps = [{name: "scan", run: async () => undefined}]; + if (callCount > 1) { + steps.push({name: "sneaked", run: async () => undefined}); + } + return steps; + }; + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map([ + ["stepTask", + {requiresDependencies: false, stepBased: true, options: {}, taskFunction: build}], + ]), + customTasks: new Map(), + }), + }; + + const createdStages = new Set(); + const buildCache = { + ...t.context.buildCache, + setTasks: sinon.stub().callsFake((tasks) => { + for (const {taskName, stepNames} of tasks) { + for (const stepName of stepNames ?? [undefined]) { + createdStages.add(buildCache.getStageId(taskName, stepName)); + } + } + }), + prepareStageExecutionAndValidateCache: sinon.stub().callsFake(async (taskName, stepName) => { + const stageId = buildCache.getStageId(taskName, stepName); + if (!createdStages.has(stageId)) { + throw new Error(`Stage '${stageId}' does not exist`); + } + return false; + }), + }; + + const project = getMockProject("module"); + project.getProjectResources = () => ({replayTagOperations: sinon.stub()}); + + const taskRunner = createTaskRunner(t, project, {taskUtil, buildCache, taskDefinitions}); + sinon.stub(taskRunner, "getDependenciesReader").resolves({getName: () => "dependencies"}); + + await t.notThrowsAsync(taskRunner.runTasks(), + "No stage is ever asked for that discovery did not create, because the factory runs once"); + t.is(callCount, 1, "The factory was called exactly once, so no second call could return a different set"); + t.deepEqual(buildCache.setTasks.firstCall.firstArg, + [{taskName: "stepTask", stepNames: ["scan"]}], + "Only the discovered step produced a stage; the would-be second-call step never appeared"); + t.is(projectBuildLogger.skipTask.callCount, 0, "The discovered step ran rather than being skipped"); +}); + +// The discovered step array is kept on the task, frozen so a custom task cannot mutate the shared value, and +// re-derived on the next build so a surviving TaskRunner (reused across ui5 serve rebuilds) never serves a +// stale array. Options are fixed for a TaskRunner's lifetime (a changed ui5.yaml rebuilds the whole stack), +// so re-deriving per build keeps the steps in step with the options without any explicit invalidation. +test("Step-based task: the kept step array is frozen and re-derived on each build", async (t) => { + const {sinon, taskUtil} = t.context; + + let callCount = 0; + const build = () => { + callCount++; + return [{name: "scan", run: async () => undefined}]; + }; + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map([ + ["stepTask", + {requiresDependencies: false, stepBased: true, options: {}, taskFunction: build}], + ]), + customTasks: new Map(), + }), + }; + + const createdStages = new Set(); + const buildCache = { + ...t.context.buildCache, + setTasks: sinon.stub().callsFake((tasks) => { + for (const {taskName, stepNames} of tasks) { + for (const stepName of stepNames ?? [undefined]) { + createdStages.add(buildCache.getStageId(taskName, stepName)); + } + } + }), + prepareStageExecutionAndValidateCache: sinon.stub().callsFake(async (taskName, stepName) => { + const stageId = buildCache.getStageId(taskName, stepName); + if (!createdStages.has(stageId)) { + throw new Error(`Stage '${stageId}' does not exist`); + } + return false; + }), + }; + + const project = getMockProject("module"); + project.getProjectResources = () => ({replayTagOperations: sinon.stub()}); + + const taskRunner = createTaskRunner(t, project, {taskUtil, buildCache, taskDefinitions}); + sinon.stub(taskRunner, "getDependenciesReader").resolves({getName: () => "dependencies"}); + + // First build: one factory call, the kept array is frozen. + await taskRunner.runTasks(); + t.is(callCount, 1, "The factory ran once for the first build"); + const firstSteps = taskRunner._tasks["stepTask"].steps; + t.true(Object.isFrozen(firstSteps), "The kept step array is frozen"); + t.throws(() => firstSteps.push({name: "injected"}), undefined, + "A custom task cannot mutate the frozen step array"); + + // Second build through the same TaskRunner: the factory runs again and the kept array is a fresh one. + await taskRunner.runTasks(); + t.is(callCount, 2, "The factory ran once more for the second build (re-derived, not reused across builds)"); + const secondSteps = taskRunner._tasks["stepTask"].steps; + t.not(secondSteps, firstSteps, "The second build keeps a fresh step array, not the previous one"); + t.true(Object.isFrozen(secondSteps), "The re-derived array is frozen too"); +}); + +// Integration: the custom-task path drives the same real MonitoredTaskUtil + StepRunner as the standard-task +// path, gated at Specification Version 5.0 via the static stepBased export. A per-step input change (an env +// var one step reads) re-runs only that step, a restored step replays its tags, and the runner's outcome is +// folded into recordStageResult (step-based flag set, invocation data persisted). +test("Step-based custom task: bound at Specification Version 5.0, folds the runner outcome", + async (t) => { + const {sinon, projectBuildLogger} = t.context; + + const env = {a: "1", b: "1"}; + const resolveInputValue = (type, name) => (type === "env" ? env[name] : undefined); + + const setTag = sinon.stub(); + const taskUtil = { + isRootProject: sinon.stub().returns(true), + getDependencies: sinon.stub().returns([]), + getInterface: sinon.stub(), + getEnv: (name) => env[name], + setTag, + }; + taskUtil.getInterface.returns(taskUtil); + + const ran = []; + const build = () => [{ + name: "stepGroup", + keys: async () => ["a", "b"], + each: async (key, {taskUtil}) => { + ran.push(key); + taskUtil.getEnv(key); + taskUtil.setTag({getPath: () => `/out/${key}`}, "ui5:IsBundle", true); + }, + }]; + + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map(), + customTasks: new Map([ + ["myCustom", { + taskDef: {name: "myCustom"}, + task: createCustomTaskExtension(sinon, {taskFunction: build, gte: () => true, stepBased: true}), + }], + ]), + }), + }; + + let capturedInvocationData; + let deltaMode = false; + const buildCache = { + setTasks: sinon.stub(), + recordStageResult: sinon.stub().resolves(), + allTasksCompleted: sinon.stub().resolves([]), + getStageId: (taskName, stepName) => + stepName === undefined ? `task/${taskName}` : `task/${taskName}::step/${stepName}`, + prepareStageExecutionAndValidateCache: sinon.stub().callsFake(async () => + (deltaMode ? {changedProjectResourcePaths: [], changedDependencyResourcePaths: []} : false)), + getStepInvocationData: sinon.stub().callsFake(() => capturedInvocationData), + getStepReturnValueStore: sinon.stub().returns(undefined), + getResolveInputValue: sinon.stub().returns(resolveInputValue), + setStepInvocationData: sinon.stub().callsFake((name, data) => { + capturedInvocationData = data; + }), + }; + + const replayTagOperations = sinon.stub(); + const project = getMockProject("module"); + project.getProjectResources = () => ({replayTagOperations}); + + const taskRunner = createTaskRunner(t, project, {taskUtil, buildCache, taskDefinitions}); + await taskRunner._initTasks(); + + // Build 1 (full): both steps run; the runner's outcome is folded into recordStageResult. + await taskRunner._tasks["myCustom"].task(projectBuildLogger); + t.deepEqual(ran, ["a", "b"], "The full build ran every step"); + t.is(buildCache.setStepInvocationData.callCount, 1, "The invocation data was persisted"); + t.is(buildCache.recordStageResult.getCall(0).args[0].stepBased, true, + "recordStageResult was told the task ran the step runner"); + + // Build 2 (delta): only env var "a" changed, so step "a" re-runs and step "b" is restored. + ran.length = 0; + setTag.resetHistory(); + deltaMode = true; + env.a = "2"; + await taskRunner._tasks["myCustom"].task(projectBuildLogger); + + t.deepEqual(ran, ["a"], "Only the step whose env input changed re-ran on the delta build"); + t.is(setTag.callCount, 1, "Only the re-run step set its tag live"); + t.is(setTag.getCall(0).args[0].getPath(), "/out/a", "The re-run step's live setTag targeted its own output"); + t.is(replayTagOperations.callCount, 1, "The restored step replayed its tag operations"); + t.deepEqual(replayTagOperations.getCall(0).args[0], + [{op: "set", path: "/out/b", tag: "ui5:IsBundle", value: true}], + "The restored step's recorded tag operation was replayed, so its tag survives"); + }); + +// The TaskRunner's recordStage hook must not mutate the stage's delta verdict: the StepRunner still +// holds it and #selectStepsToRun already read its changed paths before recordStage runs. The extended +// changed-path list (the verdict's own paths plus the stage's stale outputs) is handed to +// recordStageResult as an explicit field instead. Freezing the verdict pins the contract: the former +// in-place assignment would throw on the frozen object in strict mode. +test("Step-based task: recordStage passes stale outputs without mutating the delta verdict", async (t) => { + const {sinon, projectBuildLogger} = t.context; + + const frozenVerdict = Object.freeze({ + changedProjectResourcePaths: Object.freeze(["/changed.js"]), + changedDependencyResourcePaths: Object.freeze([]), + }); + const staleOutputs = ["/stale.js"]; + + // Fake StepRunner: drive the real recordStage hook once with the frozen verdict and a non-empty + // stale-output list, the exact condition under which the old code mutated the verdict. + class FakeStepRunner { + constructor(opts) { + this._opts = opts; + } + async runSteps() { + const ctx = this._opts.createStageContext(); + await this._opts.recordStage("s", { + ctx, + cacheInfo: frozenVerdict, + invocationData: new Map(), + staleOutputs, + foldedReads: undefined, + foldedInputs: undefined, + }); + return {anyStepExecuted: true, writtenResourcePaths: []}; + } + } + + const build = () => [{name: "s", keys: async () => [], each: async () => {}}]; + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map([ + ["stepTask", {requiresDependencies: false, stepBased: true, options: {}, taskFunction: build}], + ]), + customTasks: new Map(), + }), + }; + const buildCache = { + setTasks: sinon.stub(), + recordStageResult: sinon.stub().resolves([]), + allTasksCompleted: sinon.stub().resolves([]), + getStageId: (taskName, stepName) => + stepName === undefined ? `task/${taskName}` : `task/${taskName}::step/${stepName}`, + prepareStageExecutionAndValidateCache: sinon.stub().resolves(false), + getStepInvocationData: sinon.stub().returns(undefined), + setStepInvocationData: sinon.stub(), + getStepReturnValueStore: sinon.stub().returns(undefined), + getResolveInputValue: sinon.stub().returns(undefined), + }; + const project = getMockProject("module"); + project.getProjectResources = () => ({replayTagOperations: sinon.stub()}); + + t.context.TaskRunner = await esmock("../../../lib/build/TaskRunner.js", { + "@ui5/logger": t.context.logger, + "@ui5/fs/resourceFactory": t.context.resourceFactory, + "../../../lib/build/helpers/StepRunner.js": {default: FakeStepRunner}, + }); + + const taskRunner = createTaskRunner(t, project, {taskUtil: t.context.taskUtil, buildCache, taskDefinitions}); + await taskRunner._initTasks(); + + // Completing without throwing already proves the frozen verdict was not written to. + await t.notThrowsAsync(taskRunner._tasks["stepTask"].task(projectBuildLogger), + "the step path completes without mutating the frozen verdict"); + + t.deepEqual(frozenVerdict.changedProjectResourcePaths, ["/changed.js"], + "the verdict's changed-path list is left untouched"); + + const options = buildCache.recordStageResult.getCall(0).args[0]; + t.is(options.cacheInfo, frozenVerdict, "the same verdict object is forwarded, unmutated"); + t.deepEqual(options.changedProjectResourcePaths, ["/changed.js", "/stale.js"], + "the stale outputs are appended to the verdict's changed paths and passed as an explicit field"); +}); + + +// The recordStage hook folds the stage's per-key reads into the stage-level monitored requests via +// foldReadsInto: it dedups fold paths against the monitored paths (and against each other) rather than +// concatenating duplicates that only collapse later in the request graph. This keeps the request set that +// keys the stage minimal without moving its signature (a dropped entry was an already-present path). +test("Step-based task: recordStage dedups the fold against the monitored requests", async (t) => { + const {sinon, projectBuildLogger} = t.context; + + // A monitored workspace that reports one explicit path and a glob pattern, as a stage-level byPath and + // the keys() enumerator would. The fold then adds: a path already requested explicitly (/already.js), a + // genuinely new path (/new.js) carried twice, and a path matching the monitored pattern (kept: the + // trimmed foldReadsInto only dedups exact paths, it does not reason about pattern coverage). + t.context.resourceFactory.createMonitor = sinon.stub().callsFake((resource) => ({ + constructor: {name: "MonitoredReader"}, + getName: () => (resource?.getName ? resource.getName() : "workspace"), + getResourceRequests: () => ({paths: ["/already.js"], patterns: [["/resources/x/**"]]}), + })); + + class FakeStepRunner { + constructor(opts) { + this._opts = opts; + } + async runSteps() { + const ctx = this._opts.createStageContext(); + await this._opts.recordStage("s", { + ctx, + cacheInfo: false, + invocationData: new Map(), + staleOutputs: [], + foldedReads: { + project: {paths: ["/already.js", "/new.js", "/new.js", "/resources/x/a.js"], + patterns: ["/scan/b/*"]}, + dependencies: {paths: [], patterns: []}, + }, + foldedInputs: [], + }); + return {anyStepExecuted: true, writtenResourcePaths: []}; + } + } + + const build = () => [{name: "s", keys: async () => [], each: async () => {}}]; + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map([ + ["stepTask", {requiresDependencies: false, stepBased: true, options: {}, taskFunction: build}], + ]), + customTasks: new Map(), + }), + }; + const buildCache = { + setTasks: sinon.stub(), + recordStageResult: sinon.stub().resolves([]), + allTasksCompleted: sinon.stub().resolves([]), + getStageId: (taskName, stepName) => + stepName === undefined ? `task/${taskName}` : `task/${taskName}::step/${stepName}`, + prepareStageExecutionAndValidateCache: sinon.stub().resolves(false), + getStepInvocationData: sinon.stub().returns(undefined), + setStepInvocationData: sinon.stub(), + getStepReturnValueStore: sinon.stub().returns(undefined), + getResolveInputValue: sinon.stub().returns(undefined), + }; + const project = getMockProject("module"); + project.getProjectResources = () => ({replayTagOperations: sinon.stub()}); + + t.context.TaskRunner = await esmock("../../../lib/build/TaskRunner.js", { + "@ui5/logger": t.context.logger, + "@ui5/fs/resourceFactory": t.context.resourceFactory, + "../../../lib/build/helpers/StepRunner.js": {default: FakeStepRunner}, + }); + + const taskRunner = createTaskRunner(t, project, {taskUtil: t.context.taskUtil, buildCache, taskDefinitions}); + await taskRunner._initTasks(); + await taskRunner._tasks["stepTask"].task(projectBuildLogger); + + const {projectResourceRequests} = buildCache.recordStageResult.getCall(0).args[0]; + t.deepEqual(projectResourceRequests.patterns, [["/resources/x/**"], "/scan/b/*"], + "The monitored pattern is preserved and the cached key's folded pattern is merged in"); + t.deepEqual(projectResourceRequests.paths, ["/already.js", "/new.js", "/resources/x/a.js"], + "The already-requested path is not repeated and the duplicate is collapsed; the new paths are added " + + "once each (a pattern-covered path is kept, not reasoned about)"); +}); + + +// task declaring stepBased still runs as a legacy body, so the step runner is never driven and the runner +// outcome is not folded into recordStageResult. +test("Step-based custom task: the step-based export is ignored below Specification Version 5.0", async (t) => { + const {sinon, projectBuildLogger} = t.context; + + let ran = false; + const taskFunction = async () => { + ran = true; + }; + + const taskDefinitions = { + getTaskDefinitions: async () => ({ + standardTasks: new Map(), + customTasks: new Map([ + ["myCustom", { + taskDef: {name: "myCustom"}, + // 4.0: gte("3.0") is true (an interface is provided), gte("5.0") is false, so the + // stepBased export is not honored and the task runs as a legacy body. + task: createCustomTaskExtension(sinon, {taskFunction, gte: (v) => v === "3.0", stepBased: true}), + }], + ]), + }), + }; + + const project = getMockProject("module"); + const taskRunner = createTaskRunner(t, project, {taskDefinitions}); + await taskRunner._initTasks(); + await taskRunner._tasks["myCustom"].task(projectBuildLogger); + + t.true(ran, "The legacy task body ran"); + t.falsy(t.context.buildCache.recordStageResult.getCall(0).args[0].stepBased, + "The step-based export is ignored below 5.0, so the task did not run the step runner"); +}); + +// Builds the fixture both step-based paths share for the reporting-order tests: a two-step task whose +// bodies append to `order`, next to a projectBuildLogger whose start/end/skip reports append to the same +// log. `fullyCached` makes every stage a cache hit, so the task must be reported skipped. +function createStepReportingFixture(t, {stepBased, fullyCached = false}) { + const {sinon, projectBuildLogger} = t.context; + const order = []; + for (const method of ["startTask", "endTask", "skipTask"]) { + projectBuildLogger[method].callsFake((taskName) => order.push(`${method}:${taskName}`)); + } + + const taskName = stepBased === "custom" ? "myCustom" : "stepTask"; + const build = () => ["s1", "s2"].map((name) => ({ + name, + run: async () => { + order.push(`run:${name}`); + }, + })); + + const standardTasks = stepBased === "custom" ? new Map() : new Map([ + ["stepTask", {requiresDependencies: false, stepBased: true, options: {}, taskFunction: build}], + ]); + const customTasks = stepBased === "custom" ? new Map([ + ["myCustom", { + taskDef: {name: "myCustom"}, + task: createCustomTaskExtension(sinon, {taskFunction: build, gte: () => true, stepBased: true}), + }], + ]) : new Map(); + const taskDefinitions = {getTaskDefinitions: async () => ({standardTasks, customTasks})}; + + const buildCache = { + ...t.context.buildCache, + getStageId: (taskName, stepName) => + stepName === undefined ? `task/${taskName}` : `task/${taskName}::step/${stepName}`, + prepareStageExecutionAndValidateCache: sinon.stub().resolves(fullyCached), + // A fully cached scalar stage carries its one-entry sidecar ("scalar:0" is the implicit unit's key + // id). Without it, a full hit is treated as unrestorable and re-runs (see #canRestoreCachedStage). + getStepInvocationData: sinon.stub().returns( + fullyCached ? new Map([["scalar:0", {returns: null, tagOperations: []}]]) : undefined), + getStepReturnValueStore: sinon.stub().returns(undefined), + getResolveInputValue: sinon.stub().returns(() => undefined), + setStepInvocationData: sinon.stub(), + }; + + const project = getMockProject("module"); + project.getProjectResources = () => ({replayTagOperations: sinon.stub()}); + return {order, taskName, taskRunner: createTaskRunner(t, project, {buildCache, taskDefinitions})}; +} + +// The reporting contract a project-build-status consumer depends on: task-start announces work that is +// about to happen. A step-based task only learns its skip verdict while driving its stages, so it reports +// from the first stage that stops being a cache hit, before that stage runs anything. +for (const path of ["standard", "custom"]) { + test(`Step-based ${path} task: reports itself started before its first step runs`, async (t) => { + const {order, taskName, taskRunner} = createStepReportingFixture(t, {stepBased: path}); + await taskRunner._initTasks(); + + await taskRunner._tasks[taskName].task(t.context.projectBuildLogger); + + t.deepEqual(order, [`startTask:${taskName}`, "run:s1", "run:s2", `endTask:${taskName}`], + "The task was announced once, before any step ran, and closed after the last step"); + }); + + test(`Step-based ${path} task: a fully cached task is reported skipped, not started`, async (t) => { + const {order, taskName, taskRunner} = + createStepReportingFixture(t, {stepBased: path, fullyCached: true}); + await taskRunner._initTasks(); + + await taskRunner._tasks[taskName].task(t.context.projectBuildLogger); + + t.deepEqual(order, [`skipTask:${taskName}`], "A task whose every stage was cached only reports a skip"); + }); +} diff --git a/packages/project/test/lib/build/__helper__/ProjectBuilderFixtureTester.js b/packages/project/test/lib/build/__helper__/ProjectBuilderFixtureTester.js index dc41f4e7901..96a6b8da9b7 100644 --- a/packages/project/test/lib/build/__helper__/ProjectBuilderFixtureTester.js +++ b/packages/project/test/lib/build/__helper__/ProjectBuilderFixtureTester.js @@ -302,10 +302,21 @@ metadata: * @param {string} version The new `package.json` version (e.g. "2.0.0") */ async setSapUiCoreDependencyVersion(version) { - const pkgPath = `${this.fixturePath}/node_modules/sap.ui.core/package.json`; + const modulePath = `${this.fixturePath}/node_modules/@openui5/sap.ui.core`; + const pkgPath = `${modulePath}/package.json`; const pkg = JSON.parse(await fs.readFile(pkgPath, {encoding: "utf8"})); pkg.version = version; await fs.writeFile(pkgPath, JSON.stringify(pkg, null, "\t")); + // Keep the .library version in sync: a UI5 library project's version is read from .library. + await fs.writeFile(`${modulePath}/src/sap/ui/core/.library`, + `\n` + + `\n` + + `\tsap.ui.core\n` + + `\tSAP SE\n` + + `\tSome fancy copyright\n` + + `\t${version}\n` + + `\tSAP UI core library\n` + + `\n`); } /** diff --git a/packages/project/test/lib/build/cache/BuildStageCache.js b/packages/project/test/lib/build/cache/BuildStageCache.js new file mode 100644 index 00000000000..0c1d6b98369 --- /dev/null +++ b/packages/project/test/lib/build/cache/BuildStageCache.js @@ -0,0 +1,961 @@ +import test from "ava"; +import sinon from "sinon"; +import crypto from "node:crypto"; +import BuildStageCache from "../../../../lib/build/cache/BuildStageCache.js"; + +// Helper to create mock readers +function createMockReader(resources = []) { + const resourceMap = new Map(resources.map((r) => [r.getPath(), r])); + return { + byGlob: sinon.stub().callsFake(async (pattern) => { + // Simple pattern matching for tests + if (pattern === "/**/*") { + return Array.from(resourceMap.values()); + } + return resources.filter((r) => r.getPath().includes(pattern.replace(/[*]/g, ""))); + }), + byPath: sinon.stub().callsFake(async (path) => { + return resourceMap.get(path) || null; + }) + }; +} + +// Helper to create mock resources +function createMockResource(path, content = "test content", hash = null) { + const actualHash = hash || `hash-${path}`; + return { + getPath: () => path, + getOriginalPath: () => path, + getBuffer: async () => Buffer.from(content), + getIntegrity: async () => actualHash, + getLastModified: () => 1000, + getSize: async () => content.length, + getInode: () => 1, + getTags: () => null + }; +} + +test.afterEach.always(() => { + sinon.restore(); +}); + +// ===== CREATION AND INITIALIZATION TESTS ===== + +test("Create BuildStageCache instance", (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + t.truthy(cache, "BuildStageCache instance created"); + t.is(cache.getStageId(), "testTask", "Stage id matches"); + t.is(cache.getStepBased(), false, "Differential updates disabled"); +}); + +test("Create with differential updates enabled", (t) => { + const cache = new BuildStageCache("test.project", "testTask", true); + + t.is(cache.getStepBased(), true, "Differential updates enabled"); +}); + +test("getRootSignature: no recorded root requests returns the sha256 of an empty list", (t) => { + // A stage with no root requests short-circuits to a precomputed constant. It must equal the digest + // the previous code produced for an empty, sorted, NUL-joined signature list, so a standard build's + // stages (none read through getRootReader) keep the same root component in their stage signature. + const cache = new BuildStageCache("test.project", "testTask", false); + const expected = crypto.createHash("sha256").update("").digest("hex"); + t.is(cache.getRootSignature(), expected, + "empty root signature equals the digest of the empty join the hash loop produced"); +}); + +test("fromCache: restore BuildStageCache from cached data", (t) => { + const projectRequests = { + requestSetGraph: { + nodes: [], + nextId: 1 + }, + rootIndices: [], + deltaIndices: [], + unusedAtLeastOnce: false + }; + + const dependencyRequests = { + requestSetGraph: { + nodes: [], + nextId: 1 + }, + rootIndices: [], + deltaIndices: [], + unusedAtLeastOnce: false + }; + + const cache = BuildStageCache.fromCache({ + projectName: "test.project", + stageId: "testTask", + stepBased: false, + projectRequests, + dependencyRequests, + }); + + t.truthy(cache, "Cache restored from cached data"); + t.is(cache.getStageId(), "testTask", "Stage id preserved"); + t.is(cache.getStepBased(), false, "Differential updates setting preserved"); +}); + +// ===== METADATA ACCESS TESTS ===== + +test("getStageId: returns stage id", (t) => { + const cache = new BuildStageCache("test.project", "myTask", false); + + t.is(cache.getStageId(), "myTask", "Stage id returned"); +}); + +test("getStepBased: returns correct value", (t) => { + const cache1 = new BuildStageCache("test.project", "task1", false); + const cache2 = new BuildStageCache("test.project", "task2", true); + + t.false(cache1.getStepBased(), "Returns false when disabled"); + t.true(cache2.getStepBased(), "Returns true when enabled"); +}); + +test("hasNewOrModifiedCacheEntries: initially true for new instance", (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + // A new instance has new entries that need to be written + t.true(cache.hasNewOrModifiedCacheEntries(), "New instance has entries to write"); +}); + +test("hasNewOrModifiedCacheEntries: true after recording requests", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + const resource = createMockResource("/test.js"); + const projectReader = createMockReader([resource]); + const dependencyReader = createMockReader([]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + }); + + t.true(cache.hasNewOrModifiedCacheEntries(), "Has new entries after recording"); +}); + +// ===== SIGNATURE TESTS ===== + +test("getProjectIndexSignatures: returns signatures after recording", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + const resource = createMockResource("/test.js"); + const projectReader = createMockReader([resource]); + const dependencyReader = createMockReader([]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + }); + + const signatures = cache.getProjectIndexSignatures(); + + t.true(Array.isArray(signatures), "Returns array"); + t.true(signatures.length > 0, "Has at least one signature"); + t.is(typeof signatures[0], "string", "Signature is a string"); +}); + +test("getDependencyIndexSignatures: returns signatures after recording", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + const projectResource = createMockResource("/test.js"); + const depResource = createMockResource("/dep.js"); + const projectReader = createMockReader([projectResource]); + const dependencyReader = createMockReader([depResource]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + const dependencyRequests = { + paths: new Set(["/dep.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + dependencyRequestRecording: dependencyRequests, + projectReader, + dependencyReader, + }); + + const signatures = cache.getDependencyIndexSignatures(); + + t.true(Array.isArray(signatures), "Returns array"); + t.true(signatures.length > 0, "Has at least one signature"); + t.is(typeof signatures[0], "string", "Signature is a string"); +}); + +// ===== REQUEST RECORDING TESTS ===== + +test("recordRequests: handles project requests only", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + const resource = createMockResource("/test.js"); + const projectReader = createMockReader([resource]); + const dependencyReader = createMockReader([]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + const [projectSig, depSig] = await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + }); + + t.is(typeof projectSig, "string", "Project signature returned"); + t.is(typeof depSig, "string", "Dependency signature returned"); + t.true(projectSig.length > 0, "Project signature not empty"); + t.true(depSig.length > 0, "Dependency signature not empty"); +}); + +test("recordRequests: handles both project and dependency requests", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + const projectResource = createMockResource("/test.js"); + const depResource = createMockResource("/dep.js"); + const projectReader = createMockReader([projectResource]); + const dependencyReader = createMockReader([depResource]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + const dependencyRequests = { + paths: new Set(["/dep.js"]), + patterns: new Set() + }; + + const [projectSig, depSig] = await cache.recordRequests({ + projectRequestRecording: projectRequests, + dependencyRequestRecording: dependencyRequests, + projectReader, + dependencyReader, + }); + + t.is(typeof projectSig, "string", "Project signature returned"); + t.is(typeof depSig, "string", "Dependency signature returned"); +}); + +test("recordRequests: handles glob patterns", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + const resource1 = createMockResource("/src/test1.js"); + const resource2 = createMockResource("/src/test2.js"); + const projectReader = createMockReader([resource1, resource2]); + const dependencyReader = createMockReader([]); + + const projectRequests = { + paths: new Set(), + patterns: new Set(["/src/**/*.js"]) + }; + + const [projectSig, depSig] = await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + }); + + t.is(typeof projectSig, "string", "Project signature returned"); + t.is(typeof depSig, "string", "Dependency signature returned"); +}); + +test("recordRequests: handles empty requests", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + const projectReader = createMockReader([]); + const dependencyReader = createMockReader([]); + + const projectRequests = { + paths: new Set(), + patterns: new Set() + }; + + const [projectSig, depSig] = await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + }); + + t.is(typeof projectSig, "string", "Project signature returned"); + t.is(typeof depSig, "string", "Dependency signature returned"); +}); + +// ===== INDEX UPDATE TESTS ===== + +test("updateProjectIndices: processes changed resources", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + // First, record some requests + const resource = createMockResource("/test.js", "initial content"); + const projectReader = createMockReader([resource]); + const dependencyReader = createMockReader([]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + }); + + // Now update with changed resource + const updatedResource = createMockResource("/test.js", "updated content", "new-hash"); + const updatedReader = createMockReader([updatedResource]); + + const changed = await cache.updateProjectIndices(updatedReader, ["/test.js"]); + + t.is(typeof changed, "boolean", "Returns boolean"); +}); + +test("updateDependencyIndices: processes changed dependencies", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + // First, record some requests + const projectResource = createMockResource("/test.js"); + const depResource = createMockResource("/dep.js", "initial"); + const projectReader = createMockReader([projectResource]); + const dependencyReader = createMockReader([depResource]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + const dependencyRequests = { + paths: new Set(["/dep.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + dependencyRequestRecording: dependencyRequests, + projectReader, + dependencyReader, + }); + + // Now update with changed dependency + const updatedDepResource = createMockResource("/dep.js", "updated", "new-dep-hash"); + const updatedDepReader = createMockReader([updatedDepResource]); + + const changed = await cache.updateDependencyIndices(updatedDepReader, ["/dep.js"]); + + t.is(typeof changed, "boolean", "Returns boolean"); +}); + +test("refreshDependencyIndices: refreshes all dependency indices", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + // First, record some requests + const projectResource = createMockResource("/test.js"); + const depResource = createMockResource("/dep.js"); + const projectReader = createMockReader([projectResource]); + const dependencyReader = createMockReader([depResource]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + const dependencyRequests = { + paths: new Set(["/dep.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + dependencyRequestRecording: dependencyRequests, + projectReader, + dependencyReader, + }); + + // Refresh all indices - returns undefined when processing changes, or false if no requests + const result = await cache.refreshDependencyIndices(dependencyReader); + + t.true(result === undefined || result === false, "Returns undefined or false"); +}); + +// ===== DELTA TESTS (for differential updates) ===== + +test("getProjectIndexDeltas: returns deltas when enabled", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", true); + + const resource = createMockResource("/test.js"); + const projectReader = createMockReader([resource]); + const dependencyReader = createMockReader([]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + }); + + const deltas = cache.getProjectIndexDeltas(); + + t.true(deltas instanceof Map, "Returns Map"); +}); + +test("getDependencyIndexDeltas: returns deltas when enabled", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", true); + + const projectResource = createMockResource("/test.js"); + const depResource = createMockResource("/dep.js"); + const projectReader = createMockReader([projectResource]); + const dependencyReader = createMockReader([depResource]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + const dependencyRequests = { + paths: new Set(["/dep.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + dependencyRequestRecording: dependencyRequests, + projectReader, + dependencyReader, + }); + + const deltas = cache.getDependencyIndexDeltas(); + + t.true(deltas instanceof Map, "Returns Map"); +}); + +// ===== STAGE SIGNATURE TESTS ===== + +test("getStageSignatures: composes the [project, dependency, input, root] tuple", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + const projectResource = createMockResource("/test.js"); + const depResource = createMockResource("/dep.js"); + const projectReader = createMockReader([projectResource]); + const dependencyReader = createMockReader([depResource]); + + const [projectSig, dependencySig, inputSig, rootSig] = await cache.recordRequests({ + projectRequestRecording: {paths: new Set(["/test.js"]), patterns: new Set()}, + dependencyRequestRecording: {paths: new Set(["/dep.js"]), patterns: new Set()}, + projectReader, + dependencyReader, + }); + + const stageSignatures = cache.getStageSignatures(); + + t.deepEqual(stageSignatures, [`${projectSig}-${dependencySig}-${inputSig}-${rootSig}`], + "The single exact-match signature is the four components joined in tuple order"); +}); + +test("getStageSignatures: empty-input and empty-root components are the stable empty-set digests", + async (t) => { + // A stage that reads only resources, with no non-resource inputs and no root reads: the input and + // root slots must still carry the empty-set digests getInputSignature()/getRootSignature() return, + // so a later lookup recomposes the same signature. + const cache = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + const dependencyReader = createMockReader([createMockResource("/dep.js")]); + + await cache.recordRequests({ + projectRequestRecording: {paths: new Set(["/test.js"]), patterns: new Set()}, + dependencyRequestRecording: {paths: new Set(["/dep.js"]), patterns: new Set()}, + projectReader, + dependencyReader, + }); + + const [signature] = cache.getStageSignatures(); + const [, , inputComponent, rootComponent] = signature.split("-"); + + t.is(inputComponent, cache.getInputSignature(), "Input slot is the empty-input digest"); + t.is(rootComponent, cache.getRootSignature(), "Root slot is the empty-root digest"); + }); + +test("getStageSignatures: one signature per project x dependency index-signature combination", + async (t) => { + // With a single project request set and a single dependency request set the cartesian product is + // one signature. The product grows only as additional request sets are recorded. + const cache = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + const dependencyReader = createMockReader([createMockResource("/dep.js")]); + + await cache.recordRequests({ + projectRequestRecording: {paths: new Set(["/test.js"]), patterns: new Set()}, + dependencyRequestRecording: {paths: new Set(["/dep.js"]), patterns: new Set()}, + projectReader, + dependencyReader, + }); + + t.is(cache.getStageSignatures().length, 1, + "One project signature times one dependency signature yields one stage signature"); + }); + +// ===== SERIALIZATION TESTS ===== + +test("toCacheObjects: returns cache objects", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + const resource = createMockResource("/test.js"); + const projectReader = createMockReader([resource]); + const dependencyReader = createMockReader([]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + }); + + const [projectCache, dependencyCache] = cache.toCacheObjects(); + + t.truthy(projectCache, "Project cache object exists"); + t.truthy(dependencyCache, "Dependency cache object exists"); + t.truthy(projectCache.requestSetGraph, "Has request set graph"); + t.true(Array.isArray(projectCache.rootIndices), "Has root indices array"); +}); + +test("toCacheObjects: can restore from serialized data", async (t) => { + const cache1 = new BuildStageCache("test.project", "testTask", false); + + const resource = createMockResource("/test.js"); + const projectReader = createMockReader([resource]); + const dependencyReader = createMockReader([]); + + const projectRequests = { + paths: new Set(["/test.js"]), + patterns: new Set() + }; + + await cache1.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + }); + + const [projectCache, dependencyCache] = cache1.toCacheObjects(); + + // Restore from cache + const cache2 = BuildStageCache.fromCache({ + projectName: "test.project", + stageId: "testTask", + stepBased: false, + projectRequests: projectCache, + dependencyRequests: dependencyCache, + }); + + t.truthy(cache2, "Cache restored"); + t.is(cache2.getStageId(), "testTask", "Stage id preserved"); +}); + +// ===== EDGE CASES ===== + +test("Create with empty project name", (t) => { + const cache = new BuildStageCache("", "testTask", false); + + t.truthy(cache, "Cache created with empty project name"); + t.is(cache.getStageId(), "testTask", "Stage id still accessible"); +}); + +test("Multiple recordRequests calls accumulate", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + const resource1 = createMockResource("/test1.js"); + const resource2 = createMockResource("/test2.js"); + const projectReader = createMockReader([resource1, resource2]); + const dependencyReader = createMockReader([]); + + // First request + const projectRequests1 = { + paths: new Set(["/test1.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests1, + projectReader, + dependencyReader, + }); + + const sigsBefore = cache.getProjectIndexSignatures(); + + // Second request with different resources + const projectRequests2 = { + paths: new Set(["/test2.js"]), + patterns: new Set() + }; + + await cache.recordRequests({ + projectRequestRecording: projectRequests2, + projectReader, + dependencyReader, + }); + + const sigsAfter = cache.getProjectIndexSignatures(); + + t.true(sigsAfter.length >= sigsBefore.length, "Signatures accumulated"); +}); + +test("Handles non-existent resource paths", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + + const projectReader = createMockReader([]); + const dependencyReader = createMockReader([]); + + const projectRequests = { + paths: new Set(["/nonexistent.js"]), + patterns: new Set() + }; + + const [projectSig, depSig] = await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + }); + + t.is(typeof projectSig, "string", "Still returns signature"); + t.is(typeof depSig, "string", "Still returns dependency signature"); +}); + +test("recordRequests with unresolved probe in delta position returns a distinct signature", async (t) => { + // Shape observed in OpenUI5 after a branch switch: the first recording anchors + // a resolvable parent request set, then a subsequent recording adds a byPath + // probe for a file that no longer exists. + const cache = new BuildStageCache("test.project", "testTask", false); + + const projectReader = createMockReader([ + createMockResource("/a.js"), + ]); + const dependencyReader = createMockReader([]); + + const firstRequests = { + paths: new Set(["/a.js"]), + patterns: new Set(), + }; + const [firstProjSig] = await cache.recordRequests({ + projectRequestRecording: firstRequests, + projectReader, + dependencyReader, + }); + + const probingRequests = { + paths: new Set(["/a.js", "/optional.json"]), + patterns: new Set(), + }; + const [probingProjSig] = await cache.recordRequests({ + projectRequestRecording: probingRequests, + projectReader, + dependencyReader, + }); + + t.is(typeof probingProjSig, "string", + "Probing recording completes without throwing"); + t.not(probingProjSig, firstProjSig, + "Probing recording gets a cache key distinct from the parent's; the probed absence matters for output"); +}); + +// ===== NON-RESOURCE INPUT TRACKING ===== + +test("recordRequests: returns an input signature and flags a modified input set", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + const dependencyReader = createMockReader([]); + const projectRequests = {paths: new Set(["/test.js"]), patterns: new Set()}; + + const [, , inputSig] = await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader, + inputRecording: [{type: "env", name: "FLAG", value: "on"}], + }); + + t.is(typeof inputSig, "string", "Input signature returned"); + t.true(cache.hasNewOrModifiedCacheEntries(), "Recording an input flags the set as modified"); +}); + +test("getInputSignature: re-evaluates recorded inputs via the resolver", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + const projectRequests = {paths: new Set(["/test.js"]), patterns: new Set()}; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader: createMockReader([]), + inputRecording: [{type: "project.getVersion", name: "sap.ui.core", value: "1.120.0"}], + }); + + const sameVersion = cache.getInputSignature(() => "1.120.0"); + const bumpedVersion = cache.getInputSignature(() => "2.0.0"); + t.not(sameVersion, bumpedVersion, "A changed resolver value changes the input signature"); +}); + +test("toCacheObjects: includes an input cache object only when inputs were recorded", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + const projectRequests = {paths: new Set(["/test.js"]), patterns: new Set()}; + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader: createMockReader([]), + }); + t.is(cache.toCacheObjects()[2], undefined, "No input cache object without recorded inputs"); + + await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader: createMockReader([]), + inputRecording: [{type: "env", name: "FLAG", value: "on"}], + }); + const inputCache = cache.toCacheObjects()[2]; + t.truthy(inputCache, "Input cache object present after recording an input"); + t.deepEqual(inputCache.entries, [{type: "env", name: "FLAG"}], "Only type/name persisted"); +}); + +test("fromCache: restores recorded inputs and re-evaluates them on lookup", async (t) => { + const cache1 = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + const projectRequests = {paths: new Set(["/test.js"]), patterns: new Set()}; + + await cache1.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader: createMockReader([]), + inputRecording: [{type: "project.getVersion", name: "sap.ui.core", value: "1.120.0"}], + }); + const [projectCache, dependencyCache, inputCache] = cache1.toCacheObjects(); + + const cache2 = BuildStageCache.fromCache({ + projectName: "test.project", + stageId: "testTask", + stepBased: false, + projectRequests: projectCache, + dependencyRequests: dependencyCache, + inputSet: inputCache, + }); + + // Restored set carries only names; the resolver decides the value. + t.is(cache2.getInputSignature(() => "1.120.0"), cache1.getInputSignature(() => "1.120.0"), + "Restored input set reproduces the signature for the same resolved value"); + t.not(cache2.getInputSignature(() => "2.0.0"), cache2.getInputSignature(() => "1.120.0"), + "Restored input set still reacts to a changed resolved value"); +}); + +// ===== ROOT REQUEST TRACKING ===== + +const ROOT_REQUESTS = { + gitignore: {paths: ["/tsconfig.json"], patterns: []}, + noGitignore: {paths: [], patterns: []}, +}; + +test("recordRequests: records root requests and reflects them in the root signature", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + const rootReader = createMockReader([createMockResource("/tsconfig.json", "{}")]); + const projectRequests = {paths: new Set(["/test.js"]), patterns: new Set()}; + + t.false(cache.hasRootRequests(), "No root requests before recording"); + const emptyRootSig = cache.getRootSignature(); + + const [, , , rootSig] = await cache.recordRequests({ + projectRequestRecording: projectRequests, + projectReader, + dependencyReader: createMockReader([]), + inputRecording: [], + rootRequestRecording: ROOT_REQUESTS, + getRootReader: () => rootReader, + }); + + t.true(cache.hasRootRequests(), "Root requests recorded"); + t.is(typeof rootSig, "string"); + t.not(rootSig, emptyRootSig, "Recording a root read changes the root signature"); + t.is(rootSig, cache.getRootSignature(), "recordRequests returns the aggregated root signature"); +}); + +test("getRootSignature: is stable and empty when no root requests were recorded", async (t) => { + const a = new BuildStageCache("test.project", "testTask", false); + const b = new BuildStageCache("other.project", "otherTask", false); + const reader = createMockReader([createMockResource("/test.js")]); + // Record only project requests, leaving the root managers untouched. + await a.recordRequests({ + projectRequestRecording: {paths: new Set(["/test.js"]), patterns: new Set()}, + projectReader: reader, + dependencyReader: createMockReader([]), + }); + + t.false(a.hasRootRequests(), "A task with no root reads reports no root requests"); + t.is(a.getRootSignature(), b.getRootSignature(), + "Two caches without root requests share the same stable root signature"); +}); + +test("recordRequests: an empty root bucket leaves the manager clean", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + await cache.recordRequests({ + projectRequestRecording: {paths: new Set(["/test.js"]), patterns: new Set()}, + projectReader, + dependencyReader: createMockReader([]), + inputRecording: [], + rootRequestRecording: {gitignore: {paths: [], patterns: []}, noGitignore: {paths: [], patterns: []}}, + getRootReader: () => createMockReader([]), + }); + + t.false(cache.hasRootRequests(), "An empty root recording records no requests"); + const [, , , rootCache, rootNoGitignoreCache] = cache.toCacheObjects(); + t.is(rootCache, undefined, "No root cache object for an empty gitignore bucket"); + t.is(rootNoGitignoreCache, undefined, "No root cache object for an empty noGitignore bucket"); +}); + +test("refreshRootIndices: a changed root file changes the root signature", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + + await cache.recordRequests({ + projectRequestRecording: {paths: new Set(["/test.js"]), patterns: new Set()}, + projectReader, + dependencyReader: createMockReader([]), + inputRecording: [], + rootRequestRecording: ROOT_REQUESTS, + getRootReader: () => createMockReader([createMockResource("/tsconfig.json", "{}", "hash-v1")]), + }); + const before = cache.getRootSignature(); + + // A later build sees tsconfig.json with different content. + await cache.refreshRootIndices( + () => createMockReader([createMockResource("/tsconfig.json", "{changed}", "hash-v2")])); + + t.not(cache.getRootSignature(), before, "Root signature reflects the changed root file"); +}); + +test("toCacheObjects/fromCache: round-trips recorded root requests", async (t) => { + const cache1 = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + const rootReader = () => createMockReader([createMockResource("/tsconfig.json", "{}", "hash-root")]); + + await cache1.recordRequests({ + projectRequestRecording: {paths: new Set(["/test.js"]), patterns: new Set()}, + projectReader, + dependencyReader: createMockReader([]), + inputRecording: [], + rootRequestRecording: ROOT_REQUESTS, + getRootReader: rootReader, + }); + const [projectCache, dependencyCache, inputCache, rootCache, rootNoGitignoreCache] = + cache1.toCacheObjects(); + + t.truthy(rootCache, "useGitignore:true root cache object present"); + t.is(rootNoGitignoreCache, undefined, "useGitignore:false bucket was empty, so nothing to persist"); + + const cache2 = BuildStageCache.fromCache({ + projectName: "test.project", + stageId: "testTask", + stepBased: false, + projectRequests: projectCache, + dependencyRequests: dependencyCache, + inputSet: inputCache, + rootRequests: rootCache, + rootNoGitignoreRequests: rootNoGitignoreCache, + }); + + t.true(cache2.hasRootRequests(), "Restored cache carries the root requests"); + await cache2.refreshRootIndices(rootReader); + t.is(cache2.getRootSignature(), cache1.getRootSignature(), + "Restored root managers reproduce the root signature for unchanged content"); +}); + +test("recordRequests: a stage that stops reading root clears and persists the emptied manager", async (t) => { + const cache = new BuildStageCache("test.project", "testTask", false); + const projectReader = createMockReader([createMockResource("/test.js")]); + const projectRequestRecording = {paths: new Set(["/test.js"]), patterns: new Set()}; + const emptyRootSig = cache.getRootSignature(); + + // First build: the stage reads /tsconfig.json through the root reader. + await cache.recordRequests({ + projectRequestRecording, + projectReader, + dependencyReader: createMockReader([]), + inputRecording: [], + rootRequestRecording: ROOT_REQUESTS, + getRootReader: () => createMockReader([createMockResource("/tsconfig.json", "{}")]), + }); + t.true(cache.hasRootRequests(), "Root request recorded on the first build"); + t.not(cache.getRootSignature(), emptyRootSig, "Root signature folds in the recorded root read"); + + // Second build: the stage no longer reads any root resource. + const [, , , rootSig] = await cache.recordRequests({ + projectRequestRecording, + projectReader, + dependencyReader: createMockReader([]), + inputRecording: [], + rootRequestRecording: {gitignore: {paths: [], patterns: []}, noGitignore: {paths: [], patterns: []}}, + getRootReader: () => createMockReader([]), + }); + + t.false(cache.hasRootRequests(), "The stale root request set is cleared"); + t.is(cache.getRootSignature(), emptyRootSig, + "Root signature no longer folds in a resource the stage no longer reads"); + t.is(rootSig, emptyRootSig, "recordRequests returns the empty-root digest"); + + // The emptied manager is persisted so the stored (stale) request set is overwritten, not left + // behind for the next build to restore and keep folding into the signature. + const [, , , rootCache] = cache.toCacheObjects(); + t.truthy(rootCache, "The cleared gitignore bucket is persisted to overwrite the stored request set"); + + const restored = BuildStageCache.fromCache({ + projectName: "test.project", + stageId: "testTask", + stepBased: false, + projectRequests: {requestSetGraph: {nodes: [], nextId: 1}, rootIndices: [], deltaIndices: []}, + dependencyRequests: {requestSetGraph: {nodes: [], nextId: 1}, rootIndices: [], deltaIndices: []}, + rootRequests: rootCache, + }); + t.false(restored.hasRootRequests(), "Restoring the persisted empty manager carries no root requests"); + t.is(restored.getRootSignature(), emptyRootSig, + "The restored manager reproduces the empty-root digest, so the stale read is gone for good"); +}); + +test("fromCache: a task without root metadata restores clean root managers", (t) => { + const emptyRequests = {requestSetGraph: {nodes: [], nextId: 1}, rootIndices: [], deltaIndices: []}; + const cache = BuildStageCache.fromCache({ + projectName: "test.project", + stageId: "testTask", + stepBased: false, + projectRequests: emptyRequests, + dependencyRequests: emptyRequests, + }); + + t.false(cache.hasRootRequests(), "No root requests restored"); + t.false(cache.hasNewOrModifiedCacheEntries(), + "Restoring a task without root reads does not mark it for re-persistence"); +}); diff --git a/packages/project/test/lib/build/cache/BuildTaskCache.js b/packages/project/test/lib/build/cache/BuildTaskCache.js deleted file mode 100644 index 948f7753ad5..00000000000 --- a/packages/project/test/lib/build/cache/BuildTaskCache.js +++ /dev/null @@ -1,526 +0,0 @@ -import test from "ava"; -import sinon from "sinon"; -import BuildTaskCache from "../../../../lib/build/cache/BuildTaskCache.js"; - -// Helper to create mock readers -function createMockReader(resources = []) { - const resourceMap = new Map(resources.map((r) => [r.getPath(), r])); - return { - byGlob: sinon.stub().callsFake(async (pattern) => { - // Simple pattern matching for tests - if (pattern === "/**/*") { - return Array.from(resourceMap.values()); - } - return resources.filter((r) => r.getPath().includes(pattern.replace(/[*]/g, ""))); - }), - byPath: sinon.stub().callsFake(async (path) => { - return resourceMap.get(path) || null; - }) - }; -} - -// Helper to create mock resources -function createMockResource(path, content = "test content", hash = null) { - const actualHash = hash || `hash-${path}`; - return { - getPath: () => path, - getOriginalPath: () => path, - getBuffer: async () => Buffer.from(content), - getIntegrity: async () => actualHash, - getLastModified: () => 1000, - getSize: async () => content.length, - getInode: () => 1, - getTags: () => null - }; -} - -test.afterEach.always(() => { - sinon.restore(); -}); - -// ===== CREATION AND INITIALIZATION TESTS ===== - -test("Create BuildTaskCache instance", (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - t.truthy(cache, "BuildTaskCache instance created"); - t.is(cache.getTaskName(), "testTask", "Task name matches"); - t.is(cache.getSupportsDifferentialBuilds(), false, "Differential updates disabled"); -}); - -test("Create with differential updates enabled", (t) => { - const cache = new BuildTaskCache("test.project", "testTask", true); - - t.is(cache.getSupportsDifferentialBuilds(), true, "Differential updates enabled"); -}); - -test("fromCache: restore BuildTaskCache from cached data", (t) => { - const projectRequests = { - requestSetGraph: { - nodes: [], - nextId: 1 - }, - rootIndices: [], - deltaIndices: [], - unusedAtLeastOnce: false - }; - - const dependencyRequests = { - requestSetGraph: { - nodes: [], - nextId: 1 - }, - rootIndices: [], - deltaIndices: [], - unusedAtLeastOnce: false - }; - - const cache = BuildTaskCache.fromCache("test.project", "testTask", false, - projectRequests, dependencyRequests); - - t.truthy(cache, "Cache restored from cached data"); - t.is(cache.getTaskName(), "testTask", "Task name preserved"); - t.is(cache.getSupportsDifferentialBuilds(), false, "Differential updates setting preserved"); -}); - -// ===== METADATA ACCESS TESTS ===== - -test("getTaskName: returns task name", (t) => { - const cache = new BuildTaskCache("test.project", "myTask", false); - - t.is(cache.getTaskName(), "myTask", "Task name returned"); -}); - -test("getSupportsDifferentialBuilds: returns correct value", (t) => { - const cache1 = new BuildTaskCache("test.project", "task1", false); - const cache2 = new BuildTaskCache("test.project", "task2", true); - - t.false(cache1.getSupportsDifferentialBuilds(), "Returns false when disabled"); - t.true(cache2.getSupportsDifferentialBuilds(), "Returns true when enabled"); -}); - -test("hasNewOrModifiedCacheEntries: initially true for new instance", (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - // A new instance has new entries that need to be written - t.true(cache.hasNewOrModifiedCacheEntries(), "New instance has entries to write"); -}); - -test("hasNewOrModifiedCacheEntries: true after recording requests", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - const resource = createMockResource("/test.js"); - const projectReader = createMockReader([resource]); - const dependencyReader = createMockReader([]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests, undefined, projectReader, dependencyReader); - - t.true(cache.hasNewOrModifiedCacheEntries(), "Has new entries after recording"); -}); - -// ===== SIGNATURE TESTS ===== - -test("getProjectIndexSignatures: returns signatures after recording", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - const resource = createMockResource("/test.js"); - const projectReader = createMockReader([resource]); - const dependencyReader = createMockReader([]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests, undefined, projectReader, dependencyReader); - - const signatures = cache.getProjectIndexSignatures(); - - t.true(Array.isArray(signatures), "Returns array"); - t.true(signatures.length > 0, "Has at least one signature"); - t.is(typeof signatures[0], "string", "Signature is a string"); -}); - -test("getDependencyIndexSignatures: returns signatures after recording", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - const projectResource = createMockResource("/test.js"); - const depResource = createMockResource("/dep.js"); - const projectReader = createMockReader([projectResource]); - const dependencyReader = createMockReader([depResource]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - const dependencyRequests = { - paths: new Set(["/dep.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests, dependencyRequests, projectReader, dependencyReader); - - const signatures = cache.getDependencyIndexSignatures(); - - t.true(Array.isArray(signatures), "Returns array"); - t.true(signatures.length > 0, "Has at least one signature"); - t.is(typeof signatures[0], "string", "Signature is a string"); -}); - -// ===== REQUEST RECORDING TESTS ===== - -test("recordRequests: handles project requests only", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - const resource = createMockResource("/test.js"); - const projectReader = createMockReader([resource]); - const dependencyReader = createMockReader([]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - const [projectSig, depSig] = await cache.recordRequests( - projectRequests, undefined, projectReader, dependencyReader); - - t.is(typeof projectSig, "string", "Project signature returned"); - t.is(typeof depSig, "string", "Dependency signature returned"); - t.true(projectSig.length > 0, "Project signature not empty"); - t.true(depSig.length > 0, "Dependency signature not empty"); -}); - -test("recordRequests: handles both project and dependency requests", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - const projectResource = createMockResource("/test.js"); - const depResource = createMockResource("/dep.js"); - const projectReader = createMockReader([projectResource]); - const dependencyReader = createMockReader([depResource]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - const dependencyRequests = { - paths: new Set(["/dep.js"]), - patterns: new Set() - }; - - const [projectSig, depSig] = await cache.recordRequests( - projectRequests, dependencyRequests, projectReader, dependencyReader); - - t.is(typeof projectSig, "string", "Project signature returned"); - t.is(typeof depSig, "string", "Dependency signature returned"); -}); - -test("recordRequests: handles glob patterns", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - const resource1 = createMockResource("/src/test1.js"); - const resource2 = createMockResource("/src/test2.js"); - const projectReader = createMockReader([resource1, resource2]); - const dependencyReader = createMockReader([]); - - const projectRequests = { - paths: new Set(), - patterns: new Set(["/src/**/*.js"]) - }; - - const [projectSig, depSig] = await cache.recordRequests( - projectRequests, undefined, projectReader, dependencyReader); - - t.is(typeof projectSig, "string", "Project signature returned"); - t.is(typeof depSig, "string", "Dependency signature returned"); -}); - -test("recordRequests: handles empty requests", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - const projectReader = createMockReader([]); - const dependencyReader = createMockReader([]); - - const projectRequests = { - paths: new Set(), - patterns: new Set() - }; - - const [projectSig, depSig] = await cache.recordRequests( - projectRequests, undefined, projectReader, dependencyReader); - - t.is(typeof projectSig, "string", "Project signature returned"); - t.is(typeof depSig, "string", "Dependency signature returned"); -}); - -// ===== INDEX UPDATE TESTS ===== - -test("updateProjectIndices: processes changed resources", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - // First, record some requests - const resource = createMockResource("/test.js", "initial content"); - const projectReader = createMockReader([resource]); - const dependencyReader = createMockReader([]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests, undefined, projectReader, dependencyReader); - - // Now update with changed resource - const updatedResource = createMockResource("/test.js", "updated content", "new-hash"); - const updatedReader = createMockReader([updatedResource]); - - const changed = await cache.updateProjectIndices(updatedReader, ["/test.js"]); - - t.is(typeof changed, "boolean", "Returns boolean"); -}); - -test("updateDependencyIndices: processes changed dependencies", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - // First, record some requests - const projectResource = createMockResource("/test.js"); - const depResource = createMockResource("/dep.js", "initial"); - const projectReader = createMockReader([projectResource]); - const dependencyReader = createMockReader([depResource]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - const dependencyRequests = { - paths: new Set(["/dep.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests, dependencyRequests, projectReader, dependencyReader); - - // Now update with changed dependency - const updatedDepResource = createMockResource("/dep.js", "updated", "new-dep-hash"); - const updatedDepReader = createMockReader([updatedDepResource]); - - const changed = await cache.updateDependencyIndices(updatedDepReader, ["/dep.js"]); - - t.is(typeof changed, "boolean", "Returns boolean"); -}); - -test("refreshDependencyIndices: refreshes all dependency indices", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - // First, record some requests - const projectResource = createMockResource("/test.js"); - const depResource = createMockResource("/dep.js"); - const projectReader = createMockReader([projectResource]); - const dependencyReader = createMockReader([depResource]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - const dependencyRequests = { - paths: new Set(["/dep.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests, dependencyRequests, projectReader, dependencyReader); - - // Refresh all indices - returns undefined when processing changes, or false if no requests - const result = await cache.refreshDependencyIndices(dependencyReader); - - t.true(result === undefined || result === false, "Returns undefined or false"); -}); - -// ===== DELTA TESTS (for differential updates) ===== - -test("getProjectIndexDeltas: returns deltas when enabled", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", true); - - const resource = createMockResource("/test.js"); - const projectReader = createMockReader([resource]); - const dependencyReader = createMockReader([]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests, undefined, projectReader, dependencyReader); - - const deltas = cache.getProjectIndexDeltas(); - - t.true(deltas instanceof Map, "Returns Map"); -}); - -test("getDependencyIndexDeltas: returns deltas when enabled", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", true); - - const projectResource = createMockResource("/test.js"); - const depResource = createMockResource("/dep.js"); - const projectReader = createMockReader([projectResource]); - const dependencyReader = createMockReader([depResource]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - const dependencyRequests = { - paths: new Set(["/dep.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests, dependencyRequests, projectReader, dependencyReader); - - const deltas = cache.getDependencyIndexDeltas(); - - t.true(deltas instanceof Map, "Returns Map"); -}); - -// ===== SERIALIZATION TESTS ===== - -test("toCacheObjects: returns cache objects", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - const resource = createMockResource("/test.js"); - const projectReader = createMockReader([resource]); - const dependencyReader = createMockReader([]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests, undefined, projectReader, dependencyReader); - - const [projectCache, dependencyCache] = cache.toCacheObjects(); - - t.truthy(projectCache, "Project cache object exists"); - t.truthy(dependencyCache, "Dependency cache object exists"); - t.truthy(projectCache.requestSetGraph, "Has request set graph"); - t.true(Array.isArray(projectCache.rootIndices), "Has root indices array"); -}); - -test("toCacheObjects: can restore from serialized data", async (t) => { - const cache1 = new BuildTaskCache("test.project", "testTask", false); - - const resource = createMockResource("/test.js"); - const projectReader = createMockReader([resource]); - const dependencyReader = createMockReader([]); - - const projectRequests = { - paths: new Set(["/test.js"]), - patterns: new Set() - }; - - await cache1.recordRequests(projectRequests, undefined, projectReader, dependencyReader); - - const [projectCache, dependencyCache] = cache1.toCacheObjects(); - - // Restore from cache - const cache2 = BuildTaskCache.fromCache("test.project", "testTask", false, - projectCache, dependencyCache); - - t.truthy(cache2, "Cache restored"); - t.is(cache2.getTaskName(), "testTask", "Task name preserved"); -}); - -// ===== EDGE CASES ===== - -test("Create with empty project name", (t) => { - const cache = new BuildTaskCache("", "testTask", false); - - t.truthy(cache, "Cache created with empty project name"); - t.is(cache.getTaskName(), "testTask", "Task name still accessible"); -}); - -test("Multiple recordRequests calls accumulate", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - const resource1 = createMockResource("/test1.js"); - const resource2 = createMockResource("/test2.js"); - const projectReader = createMockReader([resource1, resource2]); - const dependencyReader = createMockReader([]); - - // First request - const projectRequests1 = { - paths: new Set(["/test1.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests1, undefined, projectReader, dependencyReader); - - const sigsBefore = cache.getProjectIndexSignatures(); - - // Second request with different resources - const projectRequests2 = { - paths: new Set(["/test2.js"]), - patterns: new Set() - }; - - await cache.recordRequests(projectRequests2, undefined, projectReader, dependencyReader); - - const sigsAfter = cache.getProjectIndexSignatures(); - - t.true(sigsAfter.length >= sigsBefore.length, "Signatures accumulated"); -}); - -test("Handles non-existent resource paths", async (t) => { - const cache = new BuildTaskCache("test.project", "testTask", false); - - const projectReader = createMockReader([]); - const dependencyReader = createMockReader([]); - - const projectRequests = { - paths: new Set(["/nonexistent.js"]), - patterns: new Set() - }; - - const [projectSig, depSig] = await cache.recordRequests( - projectRequests, undefined, projectReader, dependencyReader); - - t.is(typeof projectSig, "string", "Still returns signature"); - t.is(typeof depSig, "string", "Still returns dependency signature"); -}); - -test("recordRequests with unresolved probe in delta position returns a distinct signature", async (t) => { - // Shape observed in OpenUI5 after a branch switch: the first recording anchors - // a resolvable parent request set, then a subsequent recording adds a byPath - // probe for a file that no longer exists. - const cache = new BuildTaskCache("test.project", "testTask", false); - - const projectReader = createMockReader([ - createMockResource("/a.js"), - ]); - const dependencyReader = createMockReader([]); - - const firstRequests = { - paths: new Set(["/a.js"]), - patterns: new Set(), - }; - const [firstProjSig] = await cache.recordRequests( - firstRequests, undefined, projectReader, dependencyReader); - - const probingRequests = { - paths: new Set(["/a.js", "/optional.json"]), - patterns: new Set(), - }; - const [probingProjSig] = await cache.recordRequests( - probingRequests, undefined, projectReader, dependencyReader); - - t.is(typeof probingProjSig, "string", - "Probing recording completes without throwing"); - t.not(probingProjSig, firstProjSig, - "Probing recording gets a cache key distinct from the parent's; the probed absence matters for output"); -}); diff --git a/packages/project/test/lib/build/cache/ProjectBuildCache.js b/packages/project/test/lib/build/cache/ProjectBuildCache.js index 55823bacc09..e31a6d55416 100644 --- a/packages/project/test/lib/build/cache/ProjectBuildCache.js +++ b/packages/project/test/lib/build/cache/ProjectBuildCache.js @@ -1,8 +1,20 @@ import test from "ava"; import sinon from "sinon"; +import path from "node:path"; +import {rimraf} from "rimraf"; +import {createResource} from "@ui5/fs/resourceFactory"; import ProjectBuildCache from "../../../../lib/build/cache/ProjectBuildCache.js"; import ResourceRequestManager from "../../../../lib/build/cache/ResourceRequestManager.js"; import Cache from "../../../../lib/build/cache/Cache.js"; +import CacheManager from "../../../../lib/build/cache/CacheManager.js"; +import StepRunner from "../../../../lib/build/helpers/StepRunner.js"; + +const INTEGRATION_TEST_DIR = path.join(import.meta.dirname, "..", "..", "..", "tmp", "ProjectBuildCache"); + +test.after.always(async () => { + // Best-effort cleanup; on Windows, SQLite WAL files may linger briefly after close. + await rimraf(INTEGRATION_TEST_DIR).catch(() => {}); +}); // Helper to create mock Project instances function createMockProject(name = "test.project", id = "test-project-id") { @@ -153,11 +165,11 @@ test("Create with existing index cache", async (t) => { } } }, - tasks: [["task1", false]] + tasks: [["task/task1", false]] }; // Mock task metadata responses - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, taskName, type) => { + cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { if (type === "project") { return { requestSetGraph: { @@ -188,8 +200,8 @@ test("Create with existing index cache", async (t) => { await cache.initSourceIndex(); t.truthy(cache, "Cache created with existing index"); - const taskCache = cache.getTaskCache("task1"); - t.truthy(taskCache, "Task cache loaded from index"); + const stageCache = cache.getStageCache("task1"); + t.truthy(stageCache, "Stage cache loaded from index"); }); test("Initialize without any cache", async (t) => { @@ -212,13 +224,27 @@ test("isFresh returns false for empty cache", async (t) => { t.false(cache.isFresh(), "Empty cache is not fresh"); }); -test("getTaskCache returns undefined for non-existent task", async (t) => { +test("getStageCache returns undefined for non-existent stage", async (t) => { + const project = createMockProject(); + const cacheManager = createMockCacheManager(); + const cache = new ProjectBuildCache(project, "sig", cacheManager); + await cache.initSourceIndex(); + + t.is(cache.getStageCache("nonexistent"), undefined, "Returns undefined"); +}); + +test("getStageCache throws when a stage id is passed in place of a task name", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - t.is(cache.getTaskCache("nonexistent"), undefined, "Returns undefined"); + const error = t.throws(() => cache.getStageCache("task/task1"), + {instanceOf: Error}, "Throws for a composed stage id"); + t.true(error.message.includes("task/task1"), + "Error names the offending stage id"); + t.true(error.message.includes("expects a task name"), + "Error states the expected argument"); }); // ===== TASK MANAGEMENT TESTS ===== @@ -229,7 +255,7 @@ test("setTasks initializes project stages", async (t) => { const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["task1", "task2", "task3"]); + cache.setTasks([{taskName: "task1"}, {taskName: "task2"}, {taskName: "task3"}]); t.true(project.getProjectResources().initStages.calledOnce, "initStages called once"); t.deepEqual( @@ -375,6 +401,35 @@ test("discardIncrementalState clears the retained result signature and resets Pr "cache is no longer fresh, so the retained result signature can't serve stale stages"); }); +test("discardIncrementalState drops the failed build's partial step invocation data", async (t) => { + // A step-based task records its per-key invocation data in-memory via setStepInvocationData as it + // runs. #stepInvocationData is this build's working copy: a stage lookup stashes the matched stage's + // map here, and a running stage overwrites it. A build that throws mid-execution leaves the partial + // map behind. discardIncrementalState must drop it, or getStepInvocationData keeps returning the + // failed build's partial map. On a long-lived consumer (ui5 serve) that partial map then pairs with + // the next rebuild's stage, corrupting #selectStepsToRun / #computeStaleOutputs. The signature-matched + // map the next build needs is re-stashed from the restored stage_metadata row (embedded there, keyed + // by signature), not re-fetched by getStepInvocationData. + const project = createMockProject(); + const cacheManager = createMockCacheManager(); + + const stageId = "task/minify::step/minify"; + + const cache = new ProjectBuildCache(project, "sig", cacheManager); + await cache.initSourceIndex(); + + // A failed build records partial in-memory data for the stage. + const partial = new Map([["/partial.js", {reads: [], writes: ["/partial.js"]}]]); + cache.setStepInvocationData(stageId, partial); + t.is(cache.getStepInvocationData(stageId), partial, + "the partial data is memoized in-memory while the failed build is still live"); + + cache.discardIncrementalState(); + + t.is(cache.getStepInvocationData(stageId), undefined, + "discardIncrementalState dropped the failed build's partial step invocation data"); +}); + test("discardIncrementalState is a no-op in Cache.Off mode", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); @@ -385,78 +440,88 @@ test("discardIncrementalState is a no-op in Cache.Off mode", async (t) => { t.notThrows(() => cache.discardIncrementalState()); }); -test("prepareTaskExecutionAndValidateCache: task needs execution when no cache exists", async (t) => { +test("prepareStageExecutionAndValidateCache: task needs execution when no cache exists", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["myTask"]); - const canUseCache = await cache.prepareTaskExecutionAndValidateCache("myTask"); + cache.setTasks([{taskName: "myTask"}]); + const canUseCache = await cache.prepareStageExecutionAndValidateCache("myTask"); t.false(canUseCache, "Task cannot use cache"); t.true(project.getProjectResources().useStage.calledWith("task/myTask"), "Project switched to task stage"); }); -test("prepareTaskExecutionAndValidateCache: switches project to correct stage", async (t) => { +test("prepareStageExecutionAndValidateCache: switches project to correct stage", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["task1", "task2"]); - await cache.prepareTaskExecutionAndValidateCache("task2"); + cache.setTasks([{taskName: "task1"}, {taskName: "task2"}]); + await cache.prepareStageExecutionAndValidateCache("task2"); t.true(project.getProjectResources().useStage.calledWith("task/task2"), "Switched to task2 stage"); }); -test("recordTaskResult: creates task cache", async (t) => { +test("recordStageResult: creates stage cache", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["newTask"]); - await cache.prepareTaskExecutionAndValidateCache("newTask"); + cache.setTasks([{taskName: "newTask"}]); + await cache.prepareStageExecutionAndValidateCache("newTask"); const projectRequests = {paths: new Set(["/input.js"]), patterns: new Set()}; const dependencyRequests = {paths: new Set(), patterns: new Set()}; - await cache.recordTaskResult("newTask", projectRequests, dependencyRequests, null, false); + await cache.recordStageResult({ + taskName: "newTask", + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo: null, + }); - const taskCache = cache.getTaskCache("newTask"); - t.truthy(taskCache, "Task cache created"); + const stageCache = cache.getStageCache("newTask"); + t.truthy(stageCache, "Stage cache created"); }); -test("recordTaskResult with empty requests", async (t) => { +test("recordStageResult with empty requests", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["task1"]); - await cache.prepareTaskExecutionAndValidateCache("task1"); + cache.setTasks([{taskName: "task1"}]); + await cache.prepareStageExecutionAndValidateCache("task1"); const projectRequests = {paths: new Set(), patterns: new Set()}; const dependencyRequests = {paths: new Set(), patterns: new Set()}; - await cache.recordTaskResult("task1", projectRequests, dependencyRequests, null, false); + await cache.recordStageResult({ + taskName: "task1", + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo: null, + }); - const taskCache = cache.getTaskCache("task1"); - t.truthy(taskCache, "Task cache created even with no requests"); + const stageCache = cache.getStageCache("task1"); + t.truthy(stageCache, "Stage cache created even with no requests"); }); // ===== DELTA (CACHEINFO) PATH IN RECORDTASKRESULT TESTS ===== -test("recordTaskResult with cacheInfo: merges resources from previous stage, skipping already-written paths", +test("recordStageResult with cacheInfo: merges resources from previous stage, skipping already-written paths", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["myTask"]); - await cache.prepareTaskExecutionAndValidateCache("myTask"); + cache.setTasks([{taskName: "myTask"}]); + await cache.prepareStageExecutionAndValidateCache("myTask"); // Resources written by the delta execution const deltaWrittenRes = createMockResource("/a.js", "hash-a-new", 2000, 200, 2); @@ -491,7 +556,12 @@ test("recordTaskResult with cacheInfo: merges resources from previous stage, ski const projectRequests = {paths: new Set(), patterns: new Set()}; const dependencyRequests = {paths: new Set(), patterns: new Set()}; - await cache.recordTaskResult("myTask", projectRequests, dependencyRequests, cacheInfo, false); + await cache.recordStageResult({ + taskName: "myTask", + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo, + }); t.is(writeStub.callCount, 2, "Write called for 2 non-overlapping resources"); const writtenPaths = writeStub.getCalls().map((call) => call.args[0].getOriginalPath()); @@ -500,15 +570,15 @@ test("recordTaskResult with cacheInfo: merges resources from previous stage, ski t.false(writtenPaths.includes("/a.js"), "Already-written /a.js not merged"); }); -test("recordTaskResult with cacheInfo: calls importTagOperations with previous stage cache tags", +test("recordStageResult with cacheInfo: calls importTagOperations with previous stage cache tags", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["myTask"]); - await cache.prepareTaskExecutionAndValidateCache("myTask"); + cache.setTasks([{taskName: "myTask"}]); + await cache.prepareStageExecutionAndValidateCache("myTask"); const writeStub = sinon.stub().resolves(); project.getProjectResources().getStage.returns({ @@ -539,7 +609,12 @@ test("recordTaskResult with cacheInfo: calls importTagOperations with previous s const projectRequests = {paths: new Set(), patterns: new Set()}; const dependencyRequests = {paths: new Set(), patterns: new Set()}; - await cache.recordTaskResult("myTask", projectRequests, dependencyRequests, cacheInfo, false); + await cache.recordStageResult({ + taskName: "myTask", + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo, + }); const importStub = project.getProjectResources().importTagOperations; t.true(importStub.calledOnce, "importTagOperations called once"); @@ -549,15 +624,15 @@ test("recordTaskResult with cacheInfo: calls importTagOperations with previous s "Called with previous stage buildTagOperations"); }); -test("recordTaskResult with cacheInfo: merges tag operations with current delta ops taking precedence", +test("recordStageResult with cacheInfo: merges tag operations with current delta ops taking precedence", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["myTask"]); - await cache.prepareTaskExecutionAndValidateCache("myTask"); + cache.setTasks([{taskName: "myTask"}]); + await cache.prepareStageExecutionAndValidateCache("myTask"); // Delta execution's own tag operations — /a.js IsDebugVariant overrides previous value project.getProjectResources().getResourceTagOperations.returns({ @@ -598,7 +673,12 @@ test("recordTaskResult with cacheInfo: merges tag operations with current delta const projectRequests = {paths: new Set(), patterns: new Set()}; const dependencyRequests = {paths: new Set(), patterns: new Set()}; - await cache.recordTaskResult("myTask", projectRequests, dependencyRequests, cacheInfo, false); + await cache.recordStageResult({ + taskName: "myTask", + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo, + }); // Verify merged tags via writeCache -> cacheManager.writeStageCache await cache.writeCache(); @@ -623,15 +703,15 @@ test("recordTaskResult with cacheInfo: merges tag operations with current delta "Delta build tag for /c.js present"); }); -test("recordTaskResult with cacheInfo: uses cacheInfo.newSignature as stage signature", +test("recordStageResult with cacheInfo: uses cacheInfo.newSignature as stage signature", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["myTask"]); - await cache.prepareTaskExecutionAndValidateCache("myTask"); + cache.setTasks([{taskName: "myTask"}]); + await cache.prepareStageExecutionAndValidateCache("myTask"); const writtenRes = createMockResource("/a.js", "hash-a", 2000, 200, 2); const writeStub = sinon.stub().resolves(); @@ -660,7 +740,12 @@ test("recordTaskResult with cacheInfo: uses cacheInfo.newSignature as stage sign const projectRequests = {paths: new Set(), patterns: new Set()}; const dependencyRequests = {paths: new Set(), patterns: new Set()}; - await cache.recordTaskResult("myTask", projectRequests, dependencyRequests, cacheInfo, false); + await cache.recordStageResult({ + taskName: "myTask", + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo, + }); await cache.writeCache(); @@ -672,15 +757,15 @@ test("recordTaskResult with cacheInfo: uses cacheInfo.newSignature as stage sign "Stage signature comes from cacheInfo.newSignature"); }); -test("recordTaskResult with cacheInfo: uses getCachedWriter fallback when getWriter returns null", +test("recordStageResult with cacheInfo: uses getCachedWriter fallback when getWriter returns null", async (t) => { const project = createMockProject(); const cacheManager = createMockCacheManager(); const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["myTask"]); - await cache.prepareTaskExecutionAndValidateCache("myTask"); + cache.setTasks([{taskName: "myTask"}]); + await cache.prepareStageExecutionAndValidateCache("myTask"); const writeStub = sinon.stub().resolves(); project.getProjectResources().getStage.returns({ @@ -715,7 +800,12 @@ test("recordTaskResult with cacheInfo: uses getCachedWriter fallback when getWri const projectRequests = {paths: new Set(), patterns: new Set()}; const dependencyRequests = {paths: new Set(), patterns: new Set()}; - await cache.recordTaskResult("myTask", projectRequests, dependencyRequests, cacheInfo, false); + await cache.recordStageResult({ + taskName: "myTask", + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo, + }); t.true(getCachedWriterStub.calledOnce, "getCachedWriter used as fallback"); t.is(writeStub.callCount, 1, "Write called for 1 resource from cached writer"); @@ -723,6 +813,130 @@ test("recordTaskResult with cacheInfo: uses getCachedWriter fallback when getWri "Resource /e.js merged from getCachedWriter"); }); +// ===== STAGE / RESULT SIGNATURE INVARIANT TESTS ===== + +// B1 regression. On a dependency-only delta (the dependency moved, the project's own sources did not) +// the delta verdict's newSignature must pair the UNCHANGED project component as recorded with the new +// dependency component, so the next build's exact lookup recomputes it. The defect reverse-mapped the +// already-combined project component and combined it a second time, yielding a signature no later build +// produces. Delta tracking is only active for step-based stages, so the stage here is step-based; the +// assertion reads the newSignature's components back out and compares them to the raw current index +// signatures (available on the stage cache before and after the fix). Before the fix the project +// component is a doubly-combined hash, not the raw project index signature. +test("prepareStageExecutionAndValidateCache: a dependency-only delta keys the stage on the raw project " + + "signature, not a re-combined one (B1 regression)", async (t) => { + const project = createMockProject(); + const cacheManager = createMockCacheManager(); + + const projectResource = createMockResource("/test.js", "proj-hash", 1000, 100, 1); + const projectReader = { + byGlob: sinon.stub().resolves([projectResource]), + byPath: sinon.stub().callsFake((p) => Promise.resolve(p === "/test.js" ? projectResource : null)), + }; + project.getReader.callsFake(() => projectReader); + project.getSourceReader.callsFake(() => ({ + byGlob: sinon.stub().resolves([projectResource]), + byPath: sinon.stub().callsFake((p) => Promise.resolve(p === "/test.js" ? projectResource : null)), + })); + + const cache = new ProjectBuildCache(project, "sig", cacheManager); + await cache.initSourceIndex(); + + // validateCache sets the project and dependency readers recordStageResult records against. + const depResourceV0 = createMockResource("/dep.js", "dep-v0", 1000, 100, 2); + const depReaderV0 = { + byGlob: sinon.stub().resolves([depResourceV0]), + byPath: sinon.stub().callsFake((p) => Promise.resolve(p === "/dep.js" ? depResourceV0 : null)), + }; + await cache.validateCache(depReaderV0, {prepareForBuild: true}); + + // A step-based stage: differential (delta) tracking is active only for step-based stages. + cache.setTasks([{taskName: "stepTask", stepNames: ["s"]}]); + + project.getProjectResources().getStage.returns({ + getId: () => "task/stepTask::step/s", + getWriter: sinon.stub().returns({byGlob: sinon.stub().resolves([])}), + }); + + // Build #1: full execution records the stage over /test.js (project) and /dep.js@dep-v0 (dependency). + t.is(await cache.prepareStageExecutionAndValidateCache("stepTask", "s"), false, "Build #1 has no cache"); + await cache.recordStageResult({ + taskName: "stepTask", + projectResourceRequests: {paths: new Set(["/test.js"]), patterns: new Set()}, + dependencyResourceRequests: {paths: new Set(["/dep.js"]), patterns: new Set()}, + cacheInfo: null, + stepBased: true, + stepName: "s", + }); + + // A dependency resource changed while the project's sources did not: move the stage's dependency + // index to a delta (dep-v0 -> dep-v1). The changed resource differs in size and mtime so + // isResourceUnchanged does not short-circuit it as unchanged. + const depResourceV1 = createMockResource("/dep.js", "dep-v1", 2000, 200, 2); + const depReaderV1 = { + byGlob: sinon.stub().resolves([depResourceV1]), + byPath: sinon.stub().callsFake((p) => Promise.resolve(p === "/dep.js" ? depResourceV1 : null)), + }; + const stageCache = cache.getStageCache("stepTask", "s"); + await stageCache.updateDependencyIndices(depReaderV1, ["/dep.js"]); + + // Build #2: exact lookup misses (dependency moved), the dependency-only delta hits. + const cacheInfo = await cache.prepareStageExecutionAndValidateCache("stepTask", "s"); + t.truthy(cacheInfo, "Build #2 finds the stage via the dependency-only delta"); + t.not(cacheInfo, true, "Build #2 is a delta, not a full hit"); + t.deepEqual(cacheInfo.changedProjectResourcePaths, [], + "A dependency-only delta reports no changed project resources"); + + // The delta's newSignature is the [project, dependency, input, root] tuple. Its project component + // must be the raw project index signature (unchanged this build), and its dependency component the + // new dependency index signature, so the next build's exact lookup reproduces it. + const [projectComponent, dependencyComponent] = cacheInfo.newSignature.split("-"); + t.is(projectComponent, stageCache.getProjectIndexSignatures()[0], + "newSignature's project component is the raw project index signature, not a re-combined hash"); + t.is(dependencyComponent, stageCache.getDependencyIndexSignatures()[0], + "newSignature's dependency component is the updated dependency index signature"); +}); + +// F2 invariant. The result signature's dependency component is positional, so the candidate list +// (#getPossibleResultStageSignatures) and the stored signature (#getResultStageSignature) must cover +// the same stages in the same order. Deriving both from the single stage order turns a stage that is +// declared but never recorded into a loud failure instead of a silently shortened, never-matching +// dependency list. Before the fix allTasksCompleted computed a result signature over whichever stages +// happened to be in #currentStageSignatures and did not throw. +test("allTasksCompleted throws when a declared stage never received a signature (F2 invariant)", + async (t) => { + const project = createMockProject(); + const cacheManager = createMockCacheManager(); + + const resource = createMockResource("/test.js", "hash1", 1000, 100, 1); + project.getSourceReader.callsFake(() => ({ + byGlob: sinon.stub().resolves([resource]), + byPath: sinon.stub().callsFake((p) => Promise.resolve(p === "/test.js" ? resource : null)), + })); + + const cache = new ProjectBuildCache(project, "sig", cacheManager); + await cache.initSourceIndex(); + + // Two stages are declared, but only the first is prepared and recorded. The second never receives + // a #currentStageSignatures entry. + cache.setTasks([{taskName: "taskA"}, {taskName: "taskB"}]); + await cache.prepareStageExecutionAndValidateCache("taskA"); + project.getProjectResources().getStage.returns({ + getId: () => "task/taskA", + getWriter: sinon.stub().returns({byGlob: sinon.stub().resolves([])}), + }); + await cache.recordStageResult({ + taskName: "taskA", + projectResourceRequests: {paths: new Set(), patterns: new Set()}, + dependencyResourceRequests: {paths: new Set(), patterns: new Set()}, + cacheInfo: null, + }); + + const error = await t.throwsAsync(() => cache.allTasksCompleted()); + t.regex(error.message, /stage task\/taskB has no current stage signature/, + "Fails loudly instead of storing a result signature no later lookup could reproduce"); + }); + // ===== RESOURCE CHANGE TRACKING TESTS ===== test("projectSourcesChanged: marks cache as requiring validation", async (t) => { @@ -1012,11 +1226,11 @@ test("_refreshDependencyIndices: updates dependency indices", async (t) => { } } }, - tasks: [["task1", false]] + tasks: [["task/task1", false]] }; // Mock task metadata responses - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, taskName, type) => { + cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { if (type === "project") { return { requestSetGraph: { @@ -1126,6 +1340,144 @@ test("writeCache: skips writing unchanged caches", async (t) => { t.is(secondCallCount, firstCallCount + 1, "Index written each time"); }); +// Records a step-based stage end to end (prepare + setStepInvocationData + recordStageResult) so +// writeCache has a queued stage to persist, mirroring the TaskRunner's step-stage hook order +// (setStepInvocationData runs immediately before recordStageResult). +async function recordStepStage(cache, taskName, stepName, invocationData) { + cache.setTasks([{taskName, stepNames: [stepName]}]); + await cache.prepareStageExecutionAndValidateCache(taskName, stepName); + const stageId = cache.getStageId(taskName, stepName); + if (invocationData !== undefined) { + cache.setStepInvocationData(stageId, invocationData); + } + await cache.recordStageResult({ + taskName, + stepName, + stepBased: true, + projectResourceRequests: {paths: new Set(), patterns: new Set()}, + dependencyResourceRequests: {paths: new Set(), patterns: new Set()}, + cacheInfo: null, + }); + return stageId; +} + +test("writeCache embeds a recorded step stage's invocation map in its stage_metadata row", async (t) => { + // The per-key map travels inside the stage's own stage_metadata row (keyed by stage signature), so a + // later lookup pairs the stage output with the map recorded under exactly that signature. It is + // persisted as [[keyId, entry], ...] pairs since JSON has no Map. + const project = createMockProject(); + const cacheManager = createMockCacheManager(); + const cache = new ProjectBuildCache(project, "sig", cacheManager); + await cache.initSourceIndex(); + + const map = new Map([["k1", {reads: ["/a.js"], writes: ["/a.js"]}]]); + const stageId = await recordStepStage(cache, "minify", "minify", map); + + await cache.writeCache(); + + const stageWrite = cacheManager.writeStageCache.getCalls().find((c) => c.args[2] === stageId); + t.truthy(stageWrite, "the recorded stage's metadata row is written"); + t.deepEqual(stageWrite.args[4].stepInvocationData, [["k1", {reads: ["/a.js"], writes: ["/a.js"]}]], + "the step invocation map is embedded in the stage row as [[keyId, entry], ...] pairs"); +}); + +test("writeCache embeds an emptied step invocation map as []", async (t) => { + // A map step whose key set dropped to zero this build re-records under a new signature; its stage row + // must carry [] so the next build that matches it sees no stale keys rather than the previous run's. + const project = createMockProject(); + const cacheManager = createMockCacheManager(); + const cache = new ProjectBuildCache(project, "sig", cacheManager); + await cache.initSourceIndex(); + + const stageId = await recordStepStage(cache, "minify", "minify", new Map()); + + await cache.writeCache(); + + const stageWrite = cacheManager.writeStageCache.getCalls().find((c) => c.args[2] === stageId); + t.truthy(stageWrite, "the recorded stage's metadata row is written"); + t.deepEqual(stageWrite.args[4].stepInvocationData, [], + "an empty per-key map is embedded as []"); +}); + +test("writeCache omits stepInvocationData for a legacy stage", async (t) => { + // A legacy task records no per-key map, so its stage row carries no stepInvocationData field. + const project = createMockProject(); + const cacheManager = createMockCacheManager(); + const cache = new ProjectBuildCache(project, "sig", cacheManager); + await cache.initSourceIndex(); + + cache.setTasks([{taskName: "legacyTask"}]); + await cache.prepareStageExecutionAndValidateCache("legacyTask"); + await cache.recordStageResult({ + taskName: "legacyTask", + projectResourceRequests: {paths: new Set(), patterns: new Set()}, + dependencyResourceRequests: {paths: new Set(), patterns: new Set()}, + cacheInfo: null, + }); + + await cache.writeCache(); + + const stageWrite = cacheManager.writeStageCache.getCalls().find((c) => c.args[2] === "task/legacyTask"); + t.truthy(stageWrite, "the legacy stage's metadata row is written"); + t.is(stageWrite.args[4].stepInvocationData, undefined, + "a legacy stage omits the stepInvocationData field"); +}); + +test("writeCache does not rewrite a stage that was not re-recorded", async (t) => { + // stage_metadata (with its embedded step map) is written only for a stage recorded this build. A later + // writeCache with nothing newly recorded leaves the row untouched, preserving the "write only when + // changed" cost that the embedded map inherits for free. + const project = createMockProject(); + const cacheManager = createMockCacheManager(); + const cache = new ProjectBuildCache(project, "sig", cacheManager); + await cache.initSourceIndex(); + + const map = new Map([["k1", {reads: ["/a.js"], writes: ["/a.js"]}]]); + const stageId = await recordStepStage(cache, "minify", "minify", map); + await cache.writeCache(); + const writesAfterFirst = cacheManager.writeStageCache.getCalls().filter((c) => c.args[2] === stageId).length; + t.is(writesAfterFirst, 1, "the stage row is written on the build that recorded it"); + + // A second writeCache without recording the stage again must not rewrite its row. + await cache.writeCache(); + const writesAfterSecond = cacheManager.writeStageCache.getCalls().filter((c) => c.args[2] === stageId).length; + t.is(writesAfterSecond, 1, "an unchanged stage row is not rewritten on a later build"); +}); + +test("step return storage buffers per unit and flushes one transaction per step", async (t) => { + const project = createMockProject(); + const cacheManager = createMockCacheManager(); + const cache = new ProjectBuildCache(project, "sig", cacheManager); + await cache.initSourceIndex(); + + const store = cache.getStepReturnValueStore(); + + // The driver calls store() once per returning unit. Three units each return one resource. + const resA = createMockResource("/out/a.js", "int-a", 10, 12, 1); + const resB = createMockResource("/out/b.js", "int-b", 20, 12, 2); + const resC = createMockResource("/out/c.js", "int-c", 30, 12, 3); + + const txBefore = cacheManager.transaction.callCount; + const dA = await store.store([resA]); + const dB = await store.store([resB]); + const dC = await store.store([resC]); + t.is(cacheManager.transaction.callCount, txBefore, "storing a unit's return opens no transaction"); + + // Descriptors are built from the metadata #prepareStageResources already computed, identical to + // re-reading each resource's integrity/size/lastModified/inode. + t.deepEqual(dA, [{path: "/out/a.js", integrity: "int-a", size: 12, lastModified: 10, inode: 1}]); + t.deepEqual(dB, [{path: "/out/b.js", integrity: "int-b", size: 12, lastModified: 20, inode: 2}]); + t.deepEqual(dC, [{path: "/out/c.js", integrity: "int-c", size: 12, lastModified: 30, inode: 3}]); + + store.flush(); + t.is(cacheManager.transaction.callCount, txBefore + 1, + "flush writes every buffered unit's content in exactly one transaction"); + t.is(cacheManager.putCompressedContent.callCount, 3, "each unit's content written once"); + + store.flush(); + t.is(cacheManager.transaction.callCount, txBefore + 1, "an empty flush opens no transaction"); +}); + // ===== EDGE CASES ===== test("Create cache with empty project name", async (t) => { @@ -1172,8 +1524,8 @@ async function buildCacheWithTaskResult(resources, writtenPaths = []) { await cache.initSourceIndex(); // Set up and execute a task - cache.setTasks(["myTask"]); - await cache.prepareTaskExecutionAndValidateCache("myTask"); + cache.setTasks([{taskName: "myTask"}]); + await cache.prepareStageExecutionAndValidateCache("myTask"); // Simulate task writing some resources const writtenResources = writtenPaths.map( @@ -1188,7 +1540,12 @@ async function buildCacheWithTaskResult(resources, writtenPaths = []) { const projectRequests = {paths: new Set(), patterns: new Set()}; const dependencyRequests = {paths: new Set(), patterns: new Set()}; - await cache.recordTaskResult("myTask", projectRequests, dependencyRequests, null, false); + await cache.recordStageResult({ + taskName: "myTask", + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo: null, + }); return {cache, project, cacheManager}; } @@ -1298,8 +1655,8 @@ test("freezeUntransformedSources: throws when source file not found", async (t) const cache = new ProjectBuildCache(project, "test-sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["myTask"]); - await cache.prepareTaskExecutionAndValidateCache("myTask"); + cache.setTasks([{taskName: "myTask"}]); + await cache.prepareStageExecutionAndValidateCache("myTask"); project.getProjectResources().getStage.returns({ getId: () => "task/myTask", @@ -1310,7 +1667,12 @@ test("freezeUntransformedSources: throws when source file not found", async (t) const projectRequests = {paths: new Set(), patterns: new Set()}; const dependencyRequests = {paths: new Set(), patterns: new Set()}; - await cache.recordTaskResult("myTask", projectRequests, dependencyRequests, null, false); + await cache.recordStageResult({ + taskName: "myTask", + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo: null, + }); const error = await t.throwsAsync(() => cache.allTasksCompleted()); t.true(error.message.includes("not found during CAS freeze"), @@ -1376,11 +1738,14 @@ async function buildCacheWithWarmCacheAndTaskResult({ children } }, - tasks: [["myTask", 0]] + tasks: [["task/myTask", 0]] }; // Mock task metadata for the cached task - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, taskName, type) => { + cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { + if (type === "input") { + return null; + } return { requestSetGraph: {nodes: [], nextId: 1}, rootIndices: [], @@ -1404,8 +1769,8 @@ async function buildCacheWithWarmCacheAndTaskResult({ await cache.initSourceIndex(); // Set up and execute a task - cache.setTasks(["myTask"]); - await cache.prepareTaskExecutionAndValidateCache("myTask"); + cache.setTasks([{taskName: "myTask"}]); + await cache.prepareStageExecutionAndValidateCache("myTask"); // Simulate task writing some resources const writtenResources = taskWrittenPaths.map( @@ -1420,7 +1785,12 @@ async function buildCacheWithWarmCacheAndTaskResult({ const projectRequests = {paths: new Set(), patterns: new Set()}; const dependencyRequests = {paths: new Set(), patterns: new Set()}; - await cache.recordTaskResult("myTask", projectRequests, dependencyRequests, null, false); + await cache.recordStageResult({ + taskName: "myTask", + projectResourceRequests: projectRequests, + dependencyResourceRequests: dependencyRequests, + cacheInfo: null, + }); return {cache, project, cacheManager}; } @@ -1588,10 +1958,13 @@ test("restoreFrozenSources: cache miss skips gracefully", async (t) => { } } }, - tasks: [["task1", false]] + tasks: [["task/task1", false]] }; cacheManager.readIndexCache.returns(indexCache); - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, taskName, type) => { + cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { + if (type === "input") { + return null; + } return { requestSetGraph: {nodes: [], nextId: 1}, rootIndices: [], @@ -1672,10 +2045,13 @@ test("restoreFrozenSources: cache hit creates CAS reader", async (t) => { } } }, - tasks: [["task1", false]] + tasks: [["task/task1", false]] }; cacheManager.readIndexCache.returns(indexCache); - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, taskName, type) => { + cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { + if (type === "input") { + return null; + } return { requestSetGraph: {nodes: [], nextId: 1}, rootIndices: [], @@ -1790,7 +2166,10 @@ async function createCacheInRestoringState({ tasks }; - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, taskName, type) => { + cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { + if (type === "input") { + return null; + } return { requestSetGraph: {nodes: [], nextId: 1}, rootIndices: [], @@ -1820,7 +2199,7 @@ test("validateCache prepareForBuild=true: skips _refreshDependencyIndices when n const {cache, refreshSpy, mockDependencyReader} = await createCacheInRestoringState(); // Do NOT call dependencyResourcesChanged — simulates warm cache with no upstream changes. - // In RESTORING_DEPENDENCY_INDICES state, cached dependency indices (from BuildTaskCache.fromCache) + // In RESTORING_DEPENDENCY_INDICES state, cached dependency indices (from BuildStageCache.fromCache) // are already correct, so _refreshDependencyIndices can be skipped. await cache.validateCache(mockDependencyReader, {prepareForBuild: true}); @@ -1966,8 +2345,8 @@ test("validateCache: Cache.Force throws when source changes are detected", async // the delta-merge input. // // Each test drives a successful build, a second attempt where task B "throws" -// (modeled by omitting its `recordTaskResult` call), and a retry. Assertions -// inspect the arguments passed to the taskCache / projectResources mocks on the retry. +// (modeled by omitting its `recordStageResult` call), and a retry. Assertions +// inspect the arguments passed to the stageCache / projectResources mocks on the retry. // Points the project's getStage mock at a stage with the given id whose writer // returns `written` from byGlob. Pass `write` to capture merge writes. @@ -1983,17 +2362,20 @@ function stubStage(project, stageId, {written = [], write} = {}) { } // Records a task result with empty project/dependency request sets. -function recordEmptyResult(cache, taskName, cacheInfo = null, isDelta = false) { - return cache.recordTaskResult( - taskName, {paths: new Set(), patterns: new Set()}, - {paths: new Set(), patterns: new Set()}, cacheInfo, isDelta); +function recordEmptyResult(cache, taskName, cacheInfo = null) { + return cache.recordStageResult({ + taskName, + projectResourceRequests: {paths: new Set(), patterns: new Set()}, + dependencyResourceRequests: {paths: new Set(), patterns: new Set()}, + cacheInfo, + }); } test("Fail-then-succeed: #writtenResultResourcePaths accumulates across failed attempts (documented behavior)", async (t) => { // After taskA records in a failed build and taskB throws, /a.js is left in - // #writtenResultResourcePaths. On retry, prepareTaskExecutionAndValidateCache - // calls taskCache.updateProjectIndices(reader, #writtenResultResourcePaths). + // #writtenResultResourcePaths. On retry, prepareStageExecutionAndValidateCache + // calls stageCache.updateProjectIndices(reader, #writtenResultResourcePaths). // // Benign in practice: updateProjectIndices re-fetches each path through the // retry's fresh reader and re-hashes. Unchanged content yields the same @@ -2021,14 +2403,14 @@ test("Fail-then-succeed: #writtenResultResourcePaths accumulates across failed a }; // Failed build attempt: taskA runs successfully and writes /a.js. - cache.setTasks(["taskA", "taskB"]); - await cache.prepareTaskExecutionAndValidateCache("taskA"); + cache.setTasks([{taskName: "taskA"}, {taskName: "taskB"}]); + await cache.prepareStageExecutionAndValidateCache("taskA"); const writtenA = createMockResource("/a.js", "hash-a-built", 2000, 200, 1); stubStage(project, "task/taskA", {written: [writtenA]}); await recordEmptyResult(cache, "taskA"); - // taskB "throws": no recordTaskResult call. The failed build leaves + // taskB "throws": no recordStageResult call. The failed build leaves // #writtenResultResourcePaths containing ["/a.js"] since allTasksCompleted // (which would clear it) never runs. @@ -2037,12 +2419,12 @@ test("Fail-then-succeed: #writtenResultResourcePaths accumulates across failed a // The retry claims taskA again. Currently /a.js is passed as a "changed" // path to updateProjectIndices even though it did not change on disk. - cache.setTasks(["taskA", "taskB"]); + cache.setTasks([{taskName: "taskA"}, {taskName: "taskB"}]); const updateProjectIndicesStub = sinon.stub( - cache.getTaskCache("taskA"), "updateProjectIndices").resolves(); + cache.getStageCache("taskA"), "updateProjectIndices").resolves(); stubStage(project, "task/taskA"); - await cache.prepareTaskExecutionAndValidateCache("taskA"); + await cache.prepareStageExecutionAndValidateCache("taskA"); t.true(updateProjectIndicesStub.called, "updateProjectIndices is called on retry with the leaked paths"); @@ -2082,8 +2464,8 @@ test("Fail-then-succeed: #currentStageSignatures from failed attempt does not li await cache.validateCache(mockDependencyReader, {prepareForBuild: true}); // Failed attempt: taskA records with a distinctive signature. - cache.setTasks(["taskA", "taskB"]); - await cache.prepareTaskExecutionAndValidateCache("taskA"); + cache.setTasks([{taskName: "taskA"}, {taskName: "taskB"}]); + await cache.prepareStageExecutionAndValidateCache("taskA"); stubStage(project, "task/taskA"); await recordEmptyResult(cache, "taskA"); @@ -2091,13 +2473,13 @@ test("Fail-then-succeed: #currentStageSignatures from failed attempt does not li // Retry. await cache.validateCache(mockDependencyReader, {prepareForBuild: true}); - cache.setTasks(["taskA", "taskB"]); + cache.setTasks([{taskName: "taskA"}, {taskName: "taskB"}]); - await cache.prepareTaskExecutionAndValidateCache("taskA"); + await cache.prepareStageExecutionAndValidateCache("taskA"); stubStage(project, "task/taskA"); await recordEmptyResult(cache, "taskA"); - await cache.prepareTaskExecutionAndValidateCache("taskB"); + await cache.prepareStageExecutionAndValidateCache("taskB"); stubStage(project, "task/taskB"); await recordEmptyResult(cache, "taskB"); @@ -2121,7 +2503,7 @@ test("Fail-then-succeed: delta merge does not resurrect resources from a stage a // Build 2 (failed): source /b.js is deleted; taskA runs, taskB throws. // Build 3 (retry): source /b.js is still gone. taskA runs in delta mode // with cacheInfo.previousStageCache pointing at build 1's stage entry. - // The delta merge at recordTaskResult (line 900-907) reads previousStageCache + // The delta merge at recordStageResult (line 900-907) reads previousStageCache // and writes every resource not overlaid by the current delta. If it merges // the stale /b.js, /b.js becomes visible in the retry's output even though // the source file no longer exists. @@ -2130,8 +2512,8 @@ test("Fail-then-succeed: delta merge does not resurrect resources from a stage a const cache = new ProjectBuildCache(project, "sig", cacheManager); await cache.initSourceIndex(); - cache.setTasks(["deltaTask"]); - await cache.prepareTaskExecutionAndValidateCache("deltaTask"); + cache.setTasks([{taskName: "deltaTask"}]); + await cache.prepareStageExecutionAndValidateCache("deltaTask"); // The retry's delta task writes only the changed /a.js. const retryA = createMockResource("/a.js", "hash-a-new", 3000, 300, 1); @@ -2161,7 +2543,7 @@ test("Fail-then-succeed: delta merge does not resurrect resources from a stage a changedDependencyResourcePaths: [], }; - await recordEmptyResult(cache, "deltaTask", cacheInfo, true); + await recordEmptyResult(cache, "deltaTask", cacheInfo); // Without the fix, the merge writes every previous resource whose path is not // in the delta's writtenResourcePaths. /b.js is not overlaid, so it gets @@ -2275,7 +2657,7 @@ async function createCacheWithDependencyGlob({ }, }, }, - tasks: [[taskName, false]], + tasks: [[`task/${taskName}`, false]], // Persisted dependency-set identity from the previous build. validateCache compares the // identity passed at the next build against this; a mismatch forces the refresh. availableDependencies: oldDependencySetIdentity, @@ -2285,6 +2667,9 @@ async function createCacheWithDependencyGlob({ if (type === "dependencies") { return depCacheObject; } + if (type === "input") { + return null; + } return {requestSetGraph: {nodes: [], nextId: 1}, rootIndices: [], deltaIndices: [], unusedAtLeastOnce: false}; }); @@ -2318,13 +2703,13 @@ test("validateCache: added transitive dependency refreshes dependency index (bui t.not(oldDepSignature, expectedDepSignature, "precondition: adding libB must change the dependency index signature"); - t.is(cache.getTaskCache(taskName).getDependencyIndexSignatures()[0], oldDepSignature, + t.is(cache.getStageCache(taskName).getDependencyIndexSignatures()[0], oldDepSignature, "restored dependency index starts at the old signature"); await cache.validateCache(newDependencyReader, {prepareForBuild: true, dependencySetIdentity: newDependencySetIdentity}); - t.is(cache.getTaskCache(taskName).getDependencyIndexSignatures()[0], expectedDepSignature, + t.is(cache.getStageCache(taskName).getDependencyIndexSignatures()[0], expectedDepSignature, "dependency index must reflect the added transitive dependency after validateCache"); }); @@ -2347,7 +2732,7 @@ test("validateCache: removed transitive dependency refreshes dependency index", await cache.validateCache(newDependencyReader, {prepareForBuild: true, dependencySetIdentity: newDependencySetIdentity}); - t.is(cache.getTaskCache(taskName).getDependencyIndexSignatures()[0], expectedDepSignature, + t.is(cache.getStageCache(taskName).getDependencyIndexSignatures()[0], expectedDepSignature, "dependency index must reflect the removed transitive dependency after validateCache"); }); @@ -2373,6 +2758,107 @@ test("validateCache: dependency-set change is general, not specific to the build await cache.validateCache(newDependencyReader, {prepareForBuild: true, dependencySetIdentity: newDependencySetIdentity}); - t.is(cache.getTaskCache(taskName).getDependencyIndexSignatures()[0], expectedDepSignature, + t.is(cache.getStageCache(taskName).getDependencyIndexSignatures()[0], expectedDepSignature, "dependency index must refresh for any dependency-globbing task, not just buildThemes"); }); + +// Integration: CAS-backed step return values through a real SQLite CacheManager. Proves a returned +// resource is correct after a full build (stored), an unchanged rebuild (every unit restored from CAS), +// and a delta build (the changed unit re-runs, the others restore from CAS). The map step's reassembled +// returns are observed through a needs consumer, since runSteps() itself returns nothing. +function createIntegrationWorkspace(sources = []) { + const store = new Map(sources.map((res) => [res.getPath(), res])); + return { + getName: () => "workspace", + byGlob: async () => [...store.values()], + byPath: async (virPath) => store.get(virPath) ?? null, + write: async (resource) => { + store.set(resource.getPath(), resource); + }, + }; +} + +test.serial("Integration: step return values round-trip through the real CAS", async (t) => { + const testDir = path.join(INTEGRATION_TEST_DIR, `steps-${Date.now()}-${Math.random().toString(36).slice(2)}`); + const cacheManager = new CacheManager(path.join(testDir, "buildCache")); + t.teardown(() => cacheManager.close()); + + const project = {getName: () => "test.project", getId: () => "test-project-id"}; + const buildCache = new ProjectBuildCache(project, "build-sig", cacheManager, Cache.Default); + const returnValueStore = buildCache.getStepReturnValueStore(); + + // A producer map step over keys "a" and "b": each unit reads its per-key input (so a change to that + // input re-runs the owning unit) and returns a freshly-built resource. A consumer scalar step captures + // the producer's reassembled returns via needs, the observable route now that runSteps returns nothing. + let captured; + const buildSteps = (marker, ran) => [ + { + name: "g", + keys: async () => ["a", "b"], + each: async (key, {workspace}) => { + ran?.push(key); + await workspace.byPath(`/in/${key}`); + return createResource({path: `/out/${key}.js`, string: `${marker}-${key}`}); + }, + }, + { + name: "check", + needs: ["g"], + run: async ({needs}) => { + captured = needs.g; + }, + }, + ]; + + // Drive a StepRunner with in-memory per-stage hooks, keeping the real + // CAS-backed returnValueStore so returns genuinely round-trip through SQLite. `verdicts` gives each + // step's cache verdict (false = full run, true = fully cached, or a delta cacheInfo); `previousData` + // carries each step's per-key invocation data forward; `captureInvocation` records it back out. + function runBuild({marker, ran, verdicts = {}, previousData = new Map(), captureInvocation}) { + const workspace = createIntegrationWorkspace(); + return new StepRunner({ + steps: buildSteps(marker, ran), + returnValueStore, + prepareStage: async (step) => (step in verdicts ? verdicts[step] : false), + getPreviousInvocationData: (step) => previousData.get(step), + createStageContext: () => ({workspace, taskUtil: {}, monitoredTaskUtil: {}}), + recordStage: async (step, outcome) => { + captureInvocation?.(step, outcome.invocationData); + return []; + }, + }).runSteps(); + } + + // Build 1: full build. Every unit runs; returns are stored in the CAS. + let producerData; + await runBuild({marker: "content", captureInvocation: (step, data) => { + if (step === "g") { + producerData = data; + } + }}); + t.is(await captured[0].getString(), "content-a", "Build 1 hands back the fresh resource for a"); + t.is(await captured[1].getString(), "content-b", "Build 1 hands back the fresh resource for b"); + + // Build 2: unchanged rebuild. No changed paths, so no producer unit re-runs; both returns come from CAS. + // Only the producer's persisted data is carried forward; the consumer re-runs to capture the returns. + const ran2 = []; + await runBuild({ + marker: "content", ran: ran2, + verdicts: {g: {changedProjectResourcePaths: [], changedDependencyResourcePaths: []}}, + previousData: new Map([["g", producerData]]), + }); + t.deepEqual(ran2, [], "No producer unit re-ran on the unchanged rebuild"); + t.is(await captured[0].getString(), "content-a", "Unchanged rebuild restored a from the CAS"); + t.is(await captured[1].getString(), "content-b", "Unchanged rebuild restored b from the CAS"); + + // Build 3: delta build. Only /in/a changed, so unit "a" re-runs (fresh) while "b" restores from CAS. + const ran3 = []; + await runBuild({ + marker: "fresh", ran: ran3, + verdicts: {g: {changedProjectResourcePaths: ["/in/a"], changedDependencyResourcePaths: []}}, + previousData: new Map([["g", producerData]]), + }); + t.deepEqual(ran3, ["a"], "Delta build re-ran only the unit whose input changed"); + t.is(await captured[0].getString(), "fresh-a", "Delta build handed back the re-run unit's fresh return"); + t.is(await captured[1].getString(), "content-b", "Delta build restored the unchanged unit b from the CAS"); +}); diff --git a/packages/project/test/lib/build/cache/ResourceRequestManager.js b/packages/project/test/lib/build/cache/ResourceRequestManager.js index 5e9ad65a443..df16050075a 100644 --- a/packages/project/test/lib/build/cache/ResourceRequestManager.js +++ b/packages/project/test/lib/build/cache/ResourceRequestManager.js @@ -302,6 +302,52 @@ test("ResourceRequestManager: Adding requests marks as modified", async (t) => { t.true(manager.hasNewOrModifiedCacheEntries(), "Has modified entries after adding requests"); }); +test("ResourceRequestManager: Reusing an unchanged request set leaves the manager clean", async (t) => { + // A step-based stage records the same request set on every delta build. When that set already exists in + // the restored graph, #addRequestSet reuses the existing node and nothing persisted changes, so the + // manager must stay clean: otherwise every delta build re-serializes the whole request graph to SQLite. + const resources = new Map([ + ["/a.js", createMockResource("/a.js", "hash-a")], + ["/b.js", createMockResource("/b.js", "hash-b")], + ]); + const reader = createMockReader(resources); + + // Record the set once, serialize, and restore a clean manager from the cache. + const manager1 = new ResourceRequestManager("test.project", "myTask", false); + await manager1.addRequests({paths: ["/a.js", "/b.js"], patterns: []}, reader); + const cacheData = manager1.toCacheObject(); + const manager2 = ResourceRequestManager.fromCache("test.project", "myTask", false, cacheData); + t.false(manager2.hasNewOrModifiedCacheEntries(), "Restored manager starts clean"); + + // Record the identical set again: the exact-match reuse must not flag the manager dirty. + await manager2.addRequests({paths: ["/a.js", "/b.js"], patterns: []}, reader); + + t.false(manager2.hasNewOrModifiedCacheEntries(), + "Reusing an unchanged request set leaves the manager clean"); + t.is(manager2.toCacheObject(), undefined, "A clean manager serializes to nothing"); +}); + +test("ResourceRequestManager: Adding a new request set to a restored manager flags it dirty", async (t) => { + // The counterpart to the reuse case: a genuinely new request set (not present in the restored graph) is a + // cache modification and must flag the manager dirty so it is persisted. + const resources = new Map([ + ["/a.js", createMockResource("/a.js", "hash-a")], + ["/b.js", createMockResource("/b.js", "hash-b")], + ]); + const reader = createMockReader(resources); + + const manager1 = new ResourceRequestManager("test.project", "myTask", false); + await manager1.addRequests({paths: ["/a.js"], patterns: []}, reader); + const cacheData = manager1.toCacheObject(); + const manager2 = ResourceRequestManager.fromCache("test.project", "myTask", false, cacheData); + t.false(manager2.hasNewOrModifiedCacheEntries(), "Restored manager starts clean"); + + // A request set that was never recorded before: a new node is created. + await manager2.addRequests({paths: ["/a.js", "/b.js"], patterns: []}, reader); + + t.true(manager2.hasNewOrModifiedCacheEntries(), "A new request set flags the manager dirty"); +}); + // ===== toCacheObject TESTS ===== test("ResourceRequestManager: Serialize to cache object", async (t) => { @@ -829,9 +875,9 @@ test("ResourceRequestManager: fully empty root recording gets a distinguished si "Distinct root recordings with all-unresolved reads produce distinct signatures"); }); -test("ResourceRequestManager: BuildTaskCache-shape flow with unresolved probe (integration-ish)", +test("ResourceRequestManager: BuildStageCache-shape flow with unresolved probe (integration-ish)", async (t) => { - // Mirrors what BuildTaskCache.recordRequests does with two consecutive + // Mirrors what BuildStageCache.recordRequests does with two consecutive // addRequests recordings that share a parent but differ by one probed path. const readerBefore = createMockReader(new Map([ ["/a.js", createMockResource("/a.js", "hash-a")], @@ -892,3 +938,29 @@ test("ResourceRequestManager: Serialization round-trip with multiple request set t.true(manager2.hasNewOrModifiedCacheEntries(), "Restored manager has new entries"); }); + +test("ResourceRequestManager: clear() empties the manager and flags it for re-persistence", async (t) => { + const manager = new ResourceRequestManager("test.project", "myStage#root", false); + const reader = createMockReader(new Map([["/tsconfig.json", createMockResource("/tsconfig.json")]])); + await manager.addRequests({paths: ["/tsconfig.json"], patterns: []}, reader); + + t.true(manager.hasRequests(), "Manager has requests after recording"); + t.true(manager.getIndexSignatures().length > 0, "A recorded request yields a signature"); + t.false(manager.wasCleared(), "A manager that recorded requests was not cleared"); + + manager.clear(); + + t.false(manager.hasRequests(), "clear() empties the manager"); + t.deepEqual(manager.getIndexSignatures(), [], "No signatures remain after clear()"); + t.true(manager.wasCleared(), "wasCleared() reports the manager was cleared"); + t.true(manager.hasNewOrModifiedCacheEntries(), "A cleared manager is marked for re-persistence"); + t.truthy(manager.toCacheObject(), "A cleared manager serializes its now-empty state to overwrite the stored one"); +}); + +test("ResourceRequestManager: wasCleared() is false for a fresh and a restored manager", (t) => { + t.false(new ResourceRequestManager("test.project", "myStage#root", false).wasCleared(), + "A fresh manager was not cleared"); + const restored = ResourceRequestManager.fromCache("test.project", "myStage#root", false, + {requestSetGraph: {nodes: [], nextId: 1}, rootIndices: [], deltaIndices: []}); + t.false(restored.wasCleared(), "A restored manager was not cleared"); +}); diff --git a/packages/project/test/lib/build/cache/StageCache.js b/packages/project/test/lib/build/cache/StageCache.js index 3cf85f8ca79..62581e7ecb9 100644 --- a/packages/project/test/lib/build/cache/StageCache.js +++ b/packages/project/test/lib/build/cache/StageCache.js @@ -71,3 +71,29 @@ test("discardPending removes the latest write for an overwritten signature", (t) t.is(cache.getCacheForSignature("task/a", "sig-x"), null, "Overwritten-and-then-discarded entry is removed from the in-memory map"); }); + +test("getCacheForSignature returns each signature's own step invocation data", (t) => { + // The step-based stage's per-key map travels with the stage under its signature, so a stage cached + // under two signatures (e.g. a dependency at v1 then v2) keeps a distinct map per signature. This is + // the in-memory half of the fix that keeps the map from pairing with a different run's stage output. + const cache = new StageCache(); + const stageId = "task/enhanceManifest::step/enhanceManifest"; + const mapV1 = new Map([["/manifest.json", {reads: ["/manifest.json"], writes: ["/manifest.json"]}]]); + const mapV2 = new Map([["/manifest.json", {reads: ["/manifest.json", "/dep"], writes: ["/manifest.json"]}]]); + + cache.addSignature(stageId, "sig-v1", fakeStage("v1"), [], new Map(), new Map(), mapV1); + cache.addSignature(stageId, "sig-v2", fakeStage("v2"), [], new Map(), new Map(), mapV2); + + t.is(cache.getCacheForSignature(stageId, "sig-v1").stepInvocationData, mapV1, + "the v1 signature returns v1's per-key map, not the most-recently-written one"); + t.is(cache.getCacheForSignature(stageId, "sig-v2").stepInvocationData, mapV2, + "the v2 signature returns v2's per-key map"); +}); + +test("getCacheForSignature carries no step invocation data for a legacy stage", (t) => { + const cache = new StageCache(); + cache.addSignature("task/legacy", "sig", fakeStage("l"), [], new Map(), new Map()); + + t.is(cache.getCacheForSignature("task/legacy", "sig").stepInvocationData, undefined, + "a legacy stage recorded without a per-key map carries undefined"); +}); diff --git a/packages/project/test/lib/build/cache/index/TaskInputSet.js b/packages/project/test/lib/build/cache/index/TaskInputSet.js new file mode 100644 index 00000000000..508179e55e0 --- /dev/null +++ b/packages/project/test/lib/build/cache/index/TaskInputSet.js @@ -0,0 +1,180 @@ +import test from "ava"; +import crypto from "node:crypto"; +import TaskInputSet, {normalizeInputValue} from "../../../../../lib/build/cache/index/TaskInputSet.js"; + +test("normalizeInputValue: passes strings through", (t) => { + t.is(normalizeInputValue("value"), "value"); + t.is(normalizeInputValue(""), ""); +}); + +test("normalizeInputValue: maps undefined and null to undefined", (t) => { + t.is(normalizeInputValue(undefined), undefined); + t.is(normalizeInputValue(null), undefined); +}); + +test("normalizeInputValue: stringifies primitives", (t) => { + t.is(normalizeInputValue(true), "true"); + t.is(normalizeInputValue(false), "false"); + t.is(normalizeInputValue(42), "42"); +}); + +test("normalizeInputValue: serializes objects with sorted keys", (t) => { + // Key order must not matter: both objects normalize to the same string. + t.is( + normalizeInputValue({b: 1, a: 2}), + normalizeInputValue({a: 2, b: 1}), + "objects with the same entries in different key order normalize equally"); + t.is(normalizeInputValue({a: 2, b: 1}), `{"a":2,"b":1}`); +}); + +test("normalizeInputValue: keeps array order", (t) => { + t.is(normalizeInputValue(["b", "a"]), `["b","a"]`); + t.not(normalizeInputValue(["a", "b"]), normalizeInputValue(["b", "a"])); +}); + +test("isEmpty: true for no entries, false once entries exist", (t) => { + t.true(new TaskInputSet().isEmpty()); + t.false(new TaskInputSet([{type: "env", name: "FOO", value: "bar"}]).isEmpty()); +}); + +test("getEntries: returns entries sorted by type then name", (t) => { + const set = new TaskInputSet([ + {type: "env", name: "B", value: "2"}, + {type: "env", name: "A", value: "1"}, + {type: "isRootProject", name: "", value: "true"}, + ]); + t.deepEqual(set.getEntries(), [ + {type: "env", name: "A", value: "1"}, + {type: "env", name: "B", value: "2"}, + {type: "isRootProject", name: "", value: "true"}, + ]); +}); + +test("constructor: deduplicates by type+name, last value wins", (t) => { + const set = new TaskInputSet([ + {type: "env", name: "FOO", value: "first"}, + {type: "env", name: "FOO", value: "second"}, + ]); + t.deepEqual(set.getEntries(), [{type: "env", name: "FOO", value: "second"}]); +}); + +test("getSignature: stable across entry order, sensitive to values", (t) => { + const a = new TaskInputSet([ + {type: "env", name: "A", value: "1"}, + {type: "env", name: "B", value: "2"}, + ]); + const b = new TaskInputSet([ + {type: "env", name: "B", value: "2"}, + {type: "env", name: "A", value: "1"}, + ]); + t.is(a.getSignature(), b.getSignature(), "entry order does not affect the signature"); + + const changed = new TaskInputSet([ + {type: "env", name: "A", value: "1"}, + {type: "env", name: "B", value: "changed"}, + ]); + t.not(a.getSignature(), changed.getSignature(), "a changed value changes the signature"); +}); + +test("getSignature: empty set is a stable, fixed digest", (t) => { + t.is(new TaskInputSet().getSignature(), new TaskInputSet().getSignature()); + t.not(new TaskInputSet().getSignature(), + new TaskInputSet([{type: "env", name: "FOO", value: "bar"}]).getSignature()); +}); + +test("getSignature: empty set returns the sha256 of no input", (t) => { + // The empty-set short-circuit must return the exact digest the hash loop produced for zero entries, + // so a stage that recorded no inputs keeps the same signature it had before the short-circuit. + const expected = crypto.createHash("sha256").digest("hex"); + t.is(new TaskInputSet().getSignature(), expected, + "getSignature on an empty set equals the digest of zero hashed entries"); + t.is(new TaskInputSet().getSignatureWithCurrentValues(() => "x"), expected, + "getSignatureWithCurrentValues on an empty set returns the same empty digest"); +}); + +test("getEntries: relational sort order matches the previous localeCompare order for recorded shapes", (t) => { + // The sort switched from String.localeCompare (ICU collation) to a plain code-point comparison on + // the composite `type\0name` key. For the ASCII type/name identifiers the recorder stores, the two + // orders must agree; otherwise the input signature would move and miss the cache on first run. + const entries = [ + {type: "project.getVersion", name: "sap.ui.unified", value: "1"}, + {type: "env", name: "UI5_TASK_INPUT", value: "1"}, + {type: "env", name: "NODE_ENV", value: "1"}, + {type: "time", name: "", value: "1"}, + {type: "isRootProject", name: "", value: "1"}, + {type: "getDependencies", name: "sap.m", value: "1"}, + {type: "project.getVersion", name: "sap.ui.core", value: "1"}, + {type: "getDependencies", name: "", value: "1"}, + ]; + const relational = new TaskInputSet(entries).getEntries().map((e) => `${e.type}\0${e.name}`); + const localeOrder = entries + .map((e) => `${e.type}\0${e.name}`) + .sort((a, b) => a.localeCompare(b)); + t.deepEqual(relational, localeOrder, + "code-point order equals localeCompare order for the stored type/name shapes"); +}); + +test("getSignature: unset value does not collide with empty string", (t) => { + const unset = new TaskInputSet([{type: "env", name: "FOO", value: undefined}]); + const empty = new TaskInputSet([{type: "env", name: "FOO", value: ""}]); + t.not(unset.getSignature(), empty.getSignature()); +}); + +test("getSignatureWithCurrentValues: resolver re-derives values", (t) => { + // Recorded values are irrelevant here; only the resolver output feeds the signature. + const set = new TaskInputSet([ + {type: "project.getVersion", name: "sap.ui.core", value: "1.120.0"}, + ]); + const recorded = set.getSignature(); + + const sameValue = set.getSignatureWithCurrentValues(() => "1.120.0"); + t.is(sameValue, recorded, "resolving to the recorded value reproduces the recorded signature"); + + const bumped = set.getSignatureWithCurrentValues(() => "2.0.0"); + t.not(bumped, recorded, "resolving to a new value changes the signature"); +}); + +test("getSignatureWithCurrentValues: default resolver reads process.env for env inputs", (t) => { + const set = new TaskInputSet([{type: "env", name: "UI5_TASK_INPUT_SET_TEST", value: undefined}]); + t.teardown(() => { + delete process.env.UI5_TASK_INPUT_SET_TEST; + }); + + delete process.env.UI5_TASK_INPUT_SET_TEST; + const unsetSig = set.getSignatureWithCurrentValues(); + + process.env.UI5_TASK_INPUT_SET_TEST = "now-set"; + const setSig = set.getSignatureWithCurrentValues(); + + t.not(unsetSig, setSig, "changing the environment changes the default-resolved signature"); +}); + +test("toCacheObject/fromCache: round-trips entry names and types, drops values", (t) => { + const set = new TaskInputSet([ + {type: "env", name: "FOO", value: "bar"}, + {type: "project.getVersion", name: "sap.ui.core", value: "1.120.0"}, + ]); + const cacheObject = set.toCacheObject(); + t.is(cacheObject.version, 1); + t.deepEqual(cacheObject.entries, [ + {type: "env", name: "FOO"}, + {type: "project.getVersion", name: "sap.ui.core"}, + ], "values are not persisted"); + + const restored = TaskInputSet.fromCache(cacheObject); + t.deepEqual(restored.getEntries(), [ + {type: "env", name: "FOO", value: undefined}, + {type: "project.getVersion", name: "sap.ui.core", value: undefined}, + ], "restored entries carry no value"); +}); + +test("fromCache: null or undefined yields an empty set", (t) => { + t.true(TaskInputSet.fromCache(null).isEmpty()); + t.true(TaskInputSet.fromCache(undefined).isEmpty()); +}); + +test("fromCache: unsupported version throws", (t) => { + t.throws(() => TaskInputSet.fromCache({version: 2, entries: []}), { + message: "Unsupported TaskInputSet version: 2", + }); +}); diff --git a/packages/project/test/lib/build/cache/stageSignature.js b/packages/project/test/lib/build/cache/stageSignature.js new file mode 100644 index 00000000000..28b2cf96fcb --- /dev/null +++ b/packages/project/test/lib/build/cache/stageSignature.js @@ -0,0 +1,51 @@ +import test from "ava"; +import { + STAGE_SIGNATURE_SEPARATOR, + STAGE_SIG_DEPENDENCY_INDEX, + createStageSignature, + splitStageSignature, +} from "../../../../lib/build/cache/stageSignature.js"; + +// A stage signature is the explicit tuple [project, dependency, input, root]. These tests pin the +// composition and decomposition the two cache classes share, including the empty-input and empty-root +// cases where the input and/or root components are an empty-set digest rather than absent. + +const PROJECT = "a".repeat(64); +const DEPENDENCY = "b".repeat(64); +const INPUT = "c".repeat(64); +const ROOT = "d".repeat(64); + +test("createStageSignature joins the four components in tuple order", (t) => { + t.is( + createStageSignature([PROJECT, DEPENDENCY, INPUT, ROOT]), + `${PROJECT}${STAGE_SIGNATURE_SEPARATOR}${DEPENDENCY}${STAGE_SIGNATURE_SEPARATOR}` + + `${INPUT}${STAGE_SIGNATURE_SEPARATOR}${ROOT}`); +}); + +test("splitStageSignature reverses createStageSignature losslessly", (t) => { + const components = [PROJECT, DEPENDENCY, INPUT, ROOT]; + t.deepEqual(splitStageSignature(createStageSignature(components)), components, + "Round-trips the exact components"); +}); + +test("STAGE_SIG_DEPENDENCY_INDEX addresses the dependency component", (t) => { + const components = [PROJECT, DEPENDENCY, INPUT, ROOT]; + t.is(splitStageSignature(createStageSignature(components))[STAGE_SIG_DEPENDENCY_INDEX], DEPENDENCY, + "The dependency component is read back out by index"); +}); + +test("The separator cannot occur inside a hex component, so the split is unambiguous", (t) => { + // Every component is a SHA-256 hex digest; the separator is a single character absent from [0-9a-f]. + t.false(PROJECT.includes(STAGE_SIGNATURE_SEPARATOR)); + t.is(splitStageSignature(createStageSignature([PROJECT, DEPENDENCY, INPUT, ROOT])).length, 4, + "Exactly four components are recovered"); +}); + +test("Empty-input and empty-root components round-trip like any other", (t) => { + // A stage that recorded no non-resource inputs and no root reads still contributes a hex digest in + // each of those two slots (the digest of an empty set), so the tuple shape is uniform. + const emptyInputDigest = "e".repeat(64); + const emptyRootDigest = "f".repeat(64); + const signature = createStageSignature([PROJECT, DEPENDENCY, emptyInputDigest, emptyRootDigest]); + t.deepEqual(splitStageSignature(signature), [PROJECT, DEPENDENCY, emptyInputDigest, emptyRootDigest]); +}); diff --git a/packages/project/test/lib/build/definitions/application.js b/packages/project/test/lib/build/definitions/application.js index cc6fb8eabee..e0267bf634d 100644 --- a/packages/project/test/lib/build/definitions/application.js +++ b/packages/project/test/lib/build/definitions/application.js @@ -50,32 +50,33 @@ test("Standard build", (t) => { t.deepEqual(Object.fromEntries(tasks), { escapeNonAsciiCharacters: { + stepBased: true, options: { encoding: "UTF-412", pattern: "/**/*.properties" } }, replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/**/*.{js,json}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/**/*.{js,json}" }, - supportsDifferentialBuilds: true, }, minify: { + stepBased: true, options: { pattern: [ "/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, - enhanceManifest: {}, + enhanceManifest: {stepBased: true}, generateFlexChangesBundle: {}, generateComponentPreload: { options: { @@ -133,32 +134,33 @@ test("Standard build with legacy spec version", (t) => { t.deepEqual(Object.fromEntries(tasks), { escapeNonAsciiCharacters: { + stepBased: true, options: { encoding: "UTF-412", pattern: "/**/*.properties" } }, replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/**/*.{js,json}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/**/*.{js,json}" }, - supportsDifferentialBuilds: true, }, minify: { + stepBased: true, options: { pattern: [ "/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, - enhanceManifest: {}, + enhanceManifest: {stepBased: true}, generateFlexChangesBundle: {}, generateComponentPreload: { options: { @@ -249,32 +251,33 @@ test("Custom bundles", async (t) => { t.deepEqual(Object.fromEntries(tasks), { escapeNonAsciiCharacters: { + stepBased: true, options: { encoding: "UTF-412", pattern: "/**/*.properties" } }, replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/**/*.{js,json}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/**/*.{js,json}" }, - supportsDifferentialBuilds: true, }, minify: { + stepBased: true, options: { pattern: [ "/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, - enhanceManifest: {}, + enhanceManifest: {stepBased: true}, generateFlexChangesBundle: {}, generateComponentPreload: { options: { @@ -399,6 +402,7 @@ test("Minification excludes", (t) => { const taskDefinition = tasks.get("minify"); t.deepEqual(taskDefinition, { + stepBased: true, options: { pattern: [ "/**/*.js", @@ -406,7 +410,6 @@ test("Minification excludes", (t) => { "!/resources/**.html", ] }, - supportsDifferentialBuilds: true, }, "Correct minify task definition"); }); @@ -426,13 +429,13 @@ test("Minification excludes not applied for legacy specVersion", (t) => { const taskDefinition = tasks.get("minify"); t.deepEqual(taskDefinition, { + stepBased: true, options: { pattern: [ "/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, "Correct minify task definition"); }); diff --git a/packages/project/test/lib/build/definitions/component.js b/packages/project/test/lib/build/definitions/component.js index d586f9a77b1..4baf6ca9a13 100644 --- a/packages/project/test/lib/build/definitions/component.js +++ b/packages/project/test/lib/build/definitions/component.js @@ -49,32 +49,33 @@ test("Standard build", (t) => { t.deepEqual(Object.fromEntries(tasks), { escapeNonAsciiCharacters: { + stepBased: true, options: { encoding: "UTF-412", pattern: "/**/*.properties" } }, replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/**/*.{js,json}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/**/*.{js,json}" }, - supportsDifferentialBuilds: true, }, minify: { + stepBased: true, options: { pattern: [ "/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, - enhanceManifest: {}, + enhanceManifest: {stepBased: true}, generateFlexChangesBundle: {}, generateComponentPreload: { options: { @@ -151,32 +152,33 @@ test("Custom bundles", async (t) => { t.deepEqual(Object.fromEntries(tasks), { escapeNonAsciiCharacters: { + stepBased: true, options: { encoding: "UTF-412", pattern: "/**/*.properties" } }, replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/**/*.{js,json}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/**/*.{js,json}" }, - supportsDifferentialBuilds: true, }, minify: { + stepBased: true, options: { pattern: [ "/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, - enhanceManifest: {}, + enhanceManifest: {stepBased: true}, generateFlexChangesBundle: {}, generateComponentPreload: { options: { @@ -287,6 +289,7 @@ test("Minification excludes", (t) => { const taskDefinition = tasks.get("minify"); t.deepEqual(taskDefinition, { + stepBased: true, options: { pattern: [ "/**/*.js", @@ -294,7 +297,6 @@ test("Minification excludes", (t) => { "!/resources/**.html", ] }, - supportsDifferentialBuilds: true, }, "Correct minify task definition"); }); diff --git a/packages/project/test/lib/build/definitions/library.js b/packages/project/test/lib/build/definitions/library.js index 22eaf7fb8bd..c02b1cd1b26 100644 --- a/packages/project/test/lib/build/definitions/library.js +++ b/packages/project/test/lib/build/definitions/library.js @@ -61,29 +61,30 @@ test("Standard build", async (t) => { const generateJsdocTaskDefinition = tasks.get("generateJsdoc"); t.deepEqual(Object.fromEntries(tasks), { escapeNonAsciiCharacters: { + stepBased: true, options: { encoding: "UTF-412", pattern: "/**/*.properties" } }, replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/**/*.{js,library,css,less,theme,html}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/**/*.{js,json,library,css,less,theme,html}" }, - supportsDifferentialBuilds: true, }, replaceBuildtime: { + stepBased: true, options: { pattern: "/resources/sap/ui/{Global,core/Core}.js" }, - supportsDifferentialBuilds: true, }, generateJsdoc: { determineBuildSignature: generateJsdocTaskDefinition.determineBuildSignature, @@ -97,22 +98,23 @@ test("Standard build", async (t) => { } }, minify: { + stepBased: true, options: { pattern: [ "/resources/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, generateLibraryManifest: {}, - enhanceManifest: {}, + enhanceManifest: {stepBased: true}, generateLibraryPreload: { options: { excludes: [], skipBundles: [] } }, buildThemes: { + stepBased: true, requiresDependencies: true, options: { projectName: "project.b", @@ -177,6 +179,7 @@ test("Standard build (framework project)", (t) => { }); t.deepEqual(tasks.get("generateThemeDesignerResources"), { + stepBased: true, requiresDependencies: true, options: { version: "version" } @@ -199,29 +202,30 @@ test("Standard build with legacy spec version", (t) => { const generateJsdocTaskDefinition = tasks.get("generateJsdoc"); t.deepEqual(Object.fromEntries(tasks), { escapeNonAsciiCharacters: { + stepBased: true, options: { encoding: "UTF-412", pattern: "/**/*.properties" } }, replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/**/*.{js,library,css,less,theme,html}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/**/*.{js,json,library,css,less,theme,html}" }, - supportsDifferentialBuilds: true, }, replaceBuildtime: { + stepBased: true, options: { pattern: "/resources/sap/ui/{Global,core/Core}.js" }, - supportsDifferentialBuilds: true, }, generateJsdoc: { determineBuildSignature: generateJsdocTaskDefinition.determineBuildSignature, @@ -235,22 +239,23 @@ test("Standard build with legacy spec version", (t) => { } }, minify: { + stepBased: true, options: { pattern: [ "/resources/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, generateLibraryManifest: {}, - enhanceManifest: {}, + enhanceManifest: {stepBased: true}, generateLibraryPreload: { options: { excludes: [], skipBundles: [] } }, buildThemes: { + stepBased: true, requiresDependencies: true, options: { projectName: "project.b", @@ -329,29 +334,30 @@ test("Custom bundles", async (t) => { t.deepEqual(Object.fromEntries(tasks), { escapeNonAsciiCharacters: { + stepBased: true, options: { encoding: "UTF-412", pattern: "/**/*.properties" } }, replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/**/*.{js,library,css,less,theme,html}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/**/*.{js,json,library,css,less,theme,html}" }, - supportsDifferentialBuilds: true, }, replaceBuildtime: { + stepBased: true, options: { pattern: "/resources/sap/ui/{Global,core/Core}.js" }, - supportsDifferentialBuilds: true, }, generateJsdoc: { determineBuildSignature: generateJsdocTaskDefinition.determineBuildSignature, @@ -365,16 +371,16 @@ test("Custom bundles", async (t) => { } }, minify: { + stepBased: true, options: { pattern: [ "/resources/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, generateLibraryManifest: {}, - enhanceManifest: {}, + enhanceManifest: {stepBased: true}, generateLibraryPreload: { options: { excludes: [], @@ -389,6 +395,7 @@ test("Custom bundles", async (t) => { taskFunction: generateBundleTaskDefinition.taskFunction }, buildThemes: { + stepBased: true, requiresDependencies: true, options: { projectName: "project.b", @@ -493,6 +500,7 @@ test("Minification excludes", (t) => { const taskDefinition = tasks.get("minify"); t.deepEqual(taskDefinition, { + stepBased: true, options: { pattern: [ "/resources/**/*.js", @@ -500,7 +508,6 @@ test("Minification excludes", (t) => { "!/resources/**.html", ] }, - supportsDifferentialBuilds: true, }, "Correct minify task definition"); }); @@ -521,13 +528,13 @@ test("Minification excludes not applied for legacy specVersion", (t) => { const taskDefinition = tasks.get("minify"); t.deepEqual(taskDefinition, { + stepBased: true, options: { pattern: [ "/resources/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, "Correct minify task definition"); }); @@ -633,6 +640,7 @@ test("buildThemes: Project is not root", (t) => { const taskDefinition = tasks.get("buildThemes"); t.deepEqual(taskDefinition, { + stepBased: true, requiresDependencies: true, options: { projectName: "project.b", @@ -688,6 +696,7 @@ test("buildThemes: CSS Variables enabled", (t) => { const taskDefinition = tasks.get("buildThemes"); t.deepEqual(taskDefinition, { + stepBased: true, requiresDependencies: true, options: { projectName: "project.b", @@ -711,29 +720,30 @@ test("Standard build: nulled taskFunction to skip tasks", (t) => { const generateJsdocTaskDefinition = tasks.get("generateJsdoc"); t.deepEqual(Object.fromEntries(tasks), { escapeNonAsciiCharacters: { + stepBased: true, options: { encoding: "UTF-412", pattern: "/**/*.properties" } }, replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/**/*.{js,library,css,less,theme,html}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/**/*.{js,json,library,css,less,theme,html}" }, - supportsDifferentialBuilds: true, }, replaceBuildtime: { + stepBased: true, options: { pattern: "/resources/sap/ui/{Global,core/Core}.js" }, - supportsDifferentialBuilds: true, }, generateJsdoc: { determineBuildSignature: generateJsdocTaskDefinition.determineBuildSignature, @@ -747,22 +757,23 @@ test("Standard build: nulled taskFunction to skip tasks", (t) => { } }, minify: { + stepBased: true, options: { pattern: [ "/resources/**/*.js", "!**/*.support.js", ] }, - supportsDifferentialBuilds: true, }, generateLibraryManifest: {}, - enhanceManifest: {}, + enhanceManifest: {stepBased: true}, generateLibraryPreload: { options: { excludes: [], skipBundles: [] } }, buildThemes: { + stepBased: true, requiresDependencies: true, options: { projectName: "project.b", diff --git a/packages/project/test/lib/build/definitions/themeLibrary.js b/packages/project/test/lib/build/definitions/themeLibrary.js index 798bb49b1e9..cd9209a03c6 100644 --- a/packages/project/test/lib/build/definitions/themeLibrary.js +++ b/packages/project/test/lib/build/definitions/themeLibrary.js @@ -50,20 +50,21 @@ test("Standard build", (t) => { const generateThemeDesignerResourcesTaskFunction = tasks.get("generateThemeDesignerResources"); t.deepEqual(Object.fromEntries(tasks), { replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/resources/**/*.{less,theme}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/resources/**/*.{less,theme}" }, - supportsDifferentialBuilds: true, }, buildThemes: { + stepBased: true, requiresDependencies: true, options: { projectName: "project.b", @@ -94,6 +95,7 @@ test("Standard build (framework project)", (t) => { }); t.deepEqual(tasks.get("generateThemeDesignerResources"), { + stepBased: true, requiresDependencies: true, options: { version: "version" } @@ -109,20 +111,21 @@ test("Standard build for non root project", (t) => { }); t.deepEqual(Object.fromEntries(tasks), { replaceCopyright: { + stepBased: true, options: { copyright: "copyright", pattern: "/resources/**/*.{less,theme}" }, - supportsDifferentialBuilds: true, }, replaceVersion: { + stepBased: true, options: { version: "version", pattern: "/resources/**/*.{less,theme}" }, - supportsDifferentialBuilds: true, }, buildThemes: { + stepBased: true, requiresDependencies: true, options: { projectName: "project.b", @@ -150,6 +153,7 @@ test("CSS variables enabled", (t) => { const taskDefinition = tasks.get("buildThemes"); t.deepEqual(taskDefinition, { + stepBased: true, requiresDependencies: true, options: { projectName: "project.b", diff --git a/packages/project/test/lib/build/helpers/BuildContext.js b/packages/project/test/lib/build/helpers/BuildContext.js index 1e67e125c87..610f2a8beb4 100644 --- a/packages/project/test/lib/build/helpers/BuildContext.js +++ b/packages/project/test/lib/build/helpers/BuildContext.js @@ -60,6 +60,34 @@ test("getGraph", (t) => { t.deepEqual(buildContext.getGraph(), graph, "Returned correct value"); }); +test("getBuildTime: defaults to a timestamp at construction", (t) => { + const {BuildContext} = t.context; + const graph = {getRoot: () => ({getType: () => "library"})}; + const buildContext = new BuildContext(graph, "taskRepository"); + + t.true(buildContext.getBuildTime() instanceof Date, "Returns a Date before any run"); +}); + +test("getBuildTime: stable until refreshed, advances on refresh", (t) => { + const {BuildContext} = t.context; + const graph = {getRoot: () => ({getType: () => "library"})}; + const buildContext = new BuildContext(graph, "taskRepository"); + + const clock = sinon.useFakeTimers(new Date(2026, 8, 25, 14, 0, 0).getTime()); + t.teardown(() => clock.restore()); + + buildContext.refreshBuildTime(); + const first = buildContext.getBuildTime(); + // A second read within the same run must return the same instant, not a fresh Date. + t.is(buildContext.getBuildTime(), first, "Same instance until the next refresh"); + + clock.tick(60 * 60 * 1000); // Advance one hour + buildContext.refreshBuildTime(); + const second = buildContext.getBuildTime(); + t.not(second, first, "A new run gets a fresh timestamp"); + t.is(second.getTime(), first.getTime() + 60 * 60 * 1000, "Timestamp advanced by the elapsed time"); +}); + test("getTaskRepository", (t) => { const {BuildContext} = t.context; diff --git a/packages/project/test/lib/build/helpers/MonitoredTaskUtil.js b/packages/project/test/lib/build/helpers/MonitoredTaskUtil.js new file mode 100644 index 00000000000..6e3e3d5740b --- /dev/null +++ b/packages/project/test/lib/build/helpers/MonitoredTaskUtil.js @@ -0,0 +1,324 @@ +import test from "ava"; +import sinonGlobal from "sinon"; +import MonitoredTaskUtil from "../../../../lib/build/helpers/MonitoredTaskUtil.js"; + +test.beforeEach((t) => { + const sinon = t.context.sinon = sinonGlobal.createSandbox(); + + // A fake project whose accessors return fixed values. getVersion is the canonical tracked input. + t.context.coreProject = { + getName: () => "sap.ui.core", + getVersion: () => "1.120.0", + getType: () => "library", + getSpecVersion: () => "5.0", // not a tracked accessor: passes through unrecorded + }; + + // A fake TaskUtil (the raw instance a standard task receives). Tracked reads return fixed values. + t.context.taskUtil = { + STANDARD_TAGS: {IsBundle: "ui5:IsBundle"}, + getEnv: sinon.stub().callsFake((name) => (name === "SET" ? "on" : undefined)), + getTime: sinon.stub().callsFake((granularity) => (granularity === "year" ? "2026" : "2026-09-25")), + isRootProject: sinon.stub().returns(true), + getDependencies: sinon.stub().returns(["dep.a", "dep.b"]), + getProject: sinon.stub().callsFake((name) => { + if (name === undefined || name === "sap.ui.core") { + return t.context.coreProject; + } + return undefined; + }), + setTag: sinon.stub(), + resourceFactory: {createResource: sinon.stub()}, + }; +}); + +test.afterEach.always((t) => { + t.context.sinon.restore(); +}); + +test("delegates untracked members unchanged", (t) => { + const {taskUtil} = t.context; + const monitored = new MonitoredTaskUtil(taskUtil); + + t.is(monitored.STANDARD_TAGS, taskUtil.STANDARD_TAGS, "data property passes through by reference"); + t.is(monitored.resourceFactory, taskUtil.resourceFactory, "resourceFactory passes through"); + + monitored.setTag("resource", "tag", true); + t.true(taskUtil.setTag.calledOnceWithExactly("resource", "tag", true), "setTag delegates to the target"); +}); + +test("records getEnv reads", (t) => { + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + + t.is(monitored.getEnv("SET"), "on", "returns the underlying value"); + t.is(monitored.getEnv("UNSET"), undefined); + + t.deepEqual(monitored.getInputRecording(), [ + {type: "env", name: "SET", value: "on"}, + {type: "env", name: "UNSET", value: undefined}, + ]); +}); + +test("records getTime reads keyed by granularity", (t) => { + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + + t.is(monitored.getTime("year"), "2026", "returns the underlying quantized value"); + t.is(monitored.getTime("day"), "2026-09-25"); + + // The granularity is recorded as the input name, the quantized bucket as the value. + t.deepEqual(monitored.getInputRecording(), [ + {type: "time", name: "year", value: "2026"}, + {type: "time", name: "day", value: "2026-09-25"}, + ]); +}); + +test("does not record getBuildTime reads", (t) => { + // getBuildTime is an untracked passthrough: unlike getTime it must not fold into the signature, + // so a getBuildTime read produces no input recording even though it returns the underlying value. + const buildTime = new Date(2026, 8, 25, 14, 7, 3); + t.context.taskUtil.getBuildTime = t.context.sinon.stub().returns(buildTime); + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + + t.is(monitored.getBuildTime(), buildTime, "returns the underlying Date"); + // A tracked getTime read still records, so the recording holds only the getTime entry. + monitored.getTime("year"); + + t.deepEqual(monitored.getInputRecording(), [ + {type: "time", name: "year", value: "2026"}, + ], "getBuildTime left no entry; only the tracked getTime read was recorded"); +}); + +test("records isRootProject reads", (t) => { + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + t.is(monitored.isRootProject(), true); + t.deepEqual(monitored.getInputRecording(), [ + {type: "isRootProject", name: "", value: "true"}, + ]); +}); + +test("records getDependencies reads, resolving the default project name", (t) => { + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + + t.deepEqual(monitored.getDependencies("sap.ui.core"), ["dep.a", "dep.b"]); + // Called without a name: records under the project being built (getProject().getName()). + monitored.getDependencies(); + + t.deepEqual(monitored.getInputRecording(), [ + {type: "getDependencies", name: "sap.ui.core", value: `["dep.a","dep.b"]`}, + ], "both reads resolve to the same project name and collapse into one entry"); +}); + +test("records tracked project accessors keyed by project name", (t) => { + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + + const project = monitored.getProject("sap.ui.core"); + t.is(project.getVersion(), "1.120.0", "returns the underlying value"); + t.is(project.getType(), "library"); + + t.deepEqual(monitored.getInputRecording(), [ + {type: "project.getVersion", name: "sap.ui.core", value: "1.120.0"}, + {type: "project.getType", name: "sap.ui.core", value: "library"}, + ]); +}); + +test("does not record untracked project accessors", (t) => { + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + + const project = monitored.getProject("sap.ui.core"); + t.is(project.getSpecVersion(), "5.0", "untracked accessor still delegates"); + + t.deepEqual(monitored.getInputRecording(), [], "getSpecVersion is not recorded"); +}); + +// Builds a fake AbstractReader-like reader that answers byPath/byGlob and exposes the +// _byPath/_byGlob hooks a real MonitoredReader delegates to. Records nothing itself; the +// MonitoredReader wrapping it is what records the requests. +function fakeReader(name) { + return { + getName: () => name, + byPath: async (virPath) => ({getPath: () => virPath}), + byGlob: async () => [], + _byPath: async (virPath) => ({getPath: () => virPath}), + _byGlob: async () => [], + }; +} + +// The empty root bucket, reused by resource-request assertions that expect no root reads. +const EMPTY_ROOT = { + gitignore: {paths: [], patterns: []}, + noGitignore: {paths: [], patterns: []}, +}; + +// In the shared fixture, getProject() with no argument resolves to sap.ui.core, so that project is +// the one being built. Reads of its reader are project requests; reads of any other project's reader +// are dependency requests. +test("captures reads of the current project's getReader() as project requests", async (t) => { + t.context.coreProject.getReader = () => fakeReader("sap.ui.core reader"); + + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + await monitored.getProject().getReader().byPath("/resources/sap/ui/core/library.js"); + + t.deepEqual(monitored.getResourceRequests(), { + project: {paths: ["/resources/sap/ui/core/library.js"], patterns: []}, + dependencies: {paths: [], patterns: []}, + root: EMPTY_ROOT, + }, "reads of the project being built land in the project bucket"); +}); + +test("captures reads of a dependency's getReader() as dependency requests", async (t) => { + const depProject = {getName: () => "my.dep", getReader: () => fakeReader("my.dep reader")}; + t.context.taskUtil.getProject.callsFake((name) => { + if (name === undefined || name === "sap.ui.core") { + return t.context.coreProject; + } + if (name === "my.dep") { + return depProject; + } + return undefined; + }); + + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + await monitored.getProject("my.dep").getReader().byGlob("/resources/my/dep/**"); + + t.deepEqual(monitored.getResourceRequests(), { + project: {paths: [], patterns: []}, + dependencies: {paths: [], patterns: ["/resources/my/dep/**"]}, + root: EMPTY_ROOT, + }, "reads of a dependency's reader land in the dependency bucket"); +}); + +test("routes reads to the project or dependency bucket by project identity", async (t) => { + t.context.coreProject.getReader = () => fakeReader("sap.ui.core reader"); + const depProject = {getName: () => "my.dep", getReader: () => fakeReader("my.dep reader")}; + t.context.taskUtil.getProject.callsFake((name) => { + if (name === undefined || name === "sap.ui.core") { + return t.context.coreProject; + } + if (name === "my.dep") { + return depProject; + } + return undefined; + }); + + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + // Reading the current project by its explicit name still routes to the project bucket. + await monitored.getProject("sap.ui.core").getReader().byPath("/resources/sap/ui/core/library.js"); + await monitored.getProject("my.dep").getReader().byPath("/resources/my/dep/thing.js"); + + t.deepEqual(monitored.getResourceRequests(), { + project: {paths: ["/resources/sap/ui/core/library.js"], patterns: []}, + dependencies: {paths: ["/resources/my/dep/thing.js"], patterns: []}, + root: EMPTY_ROOT, + }); +}); + +test("getResourceRequests returns empty buckets when no project reader was accessed", (t) => { + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + t.deepEqual(monitored.getResourceRequests(), { + project: {paths: [], patterns: []}, + dependencies: {paths: [], patterns: []}, + root: EMPTY_ROOT, + }); +}); + +test("captures a byPath read of the current project's root reader (default useGitignore)", async (t) => { + t.context.coreProject.getRootReader = () => fakeReader("sap.ui.core root reader"); + + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + await monitored.getProject().getRootReader().byPath("/tsconfig.json"); + + const {root} = monitored.getResourceRequests(); + t.deepEqual(root.gitignore, { + // The default useGitignore:true bucket implicitly tracks the root .gitignore, whose content + // decides what globs recorded under it match. + paths: ["/tsconfig.json", "/.gitignore"], + patterns: [], + }, "the tsconfig read lands in the gitignore root bucket alongside the .gitignore input"); + t.deepEqual(root.noGitignore, {paths: [], patterns: []}); +}); + +test("routes root reads to the gitignore or noGitignore bucket by the useGitignore flag", async (t) => { + t.context.coreProject.getRootReader = sinonGlobal.stub() + .callsFake(() => fakeReader("sap.ui.core root reader")); + + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + const project = monitored.getProject(); + await project.getRootReader().byPath("/tsconfig.json"); // default: useGitignore true + await project.getRootReader({useGitignore: false}).byGlob("/node_modules/lodash/**"); + + const {root} = monitored.getResourceRequests(); + t.deepEqual(root.gitignore, { + paths: ["/tsconfig.json", "/.gitignore"], + patterns: [], + }); + t.deepEqual(root.noGitignore, { + // An explicit /node_modules glob opts in: recorded as-is, no ignore negations, and no + // implicit .gitignore (that input only joins the useGitignore:true bucket). + paths: [], + patterns: ["/node_modules/lodash/**"], + }, "an explicit node_modules glob is recorded unchanged in the noGitignore bucket"); +}); + +test("applies the default node_modules/.git ignore to wide root globs", async (t) => { + t.context.coreProject.getRootReader = () => fakeReader("sap.ui.core root reader"); + + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + await monitored.getProject().getRootReader({useGitignore: false}).byGlob("/**"); + + const {root} = monitored.getResourceRequests(); + t.deepEqual(root.noGitignore, { + paths: [], + patterns: [["/**", "!/node_modules/**", "!/.git/**"]], + }, "a wide glob is bounded away from node_modules and .git on record"); +}); + +test("does not wrap a dependency's root reader", async (t) => { + const depRootReader = fakeReader("my.dep root reader"); + const depProject = {getName: () => "my.dep", getRootReader: () => depRootReader}; + t.context.taskUtil.getProject.callsFake((name) => { + if (name === undefined || name === "sap.ui.core") { + return t.context.coreProject; + } + if (name === "my.dep") { + return depProject; + } + return undefined; + }); + + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + // A dependency's root reader passes through unwrapped: it is the same reference and its reads + // are not recorded (root requests re-materialize against the built project's root only). + t.is(monitored.getProject("my.dep").getRootReader(), depRootReader, "dependency root reader passes through"); + await monitored.getProject("my.dep").getRootReader().byPath("/tsconfig.json"); + + t.deepEqual(monitored.getResourceRequests().root, EMPTY_ROOT, "dependency root reads stay untracked"); +}); + +test("getProject returns the underlying falsy value for an unknown project", (t) => { + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + t.is(monitored.getProject("does.not.exist"), undefined); + t.deepEqual(monitored.getInputRecording(), []); +}); + +test("getInputRecording is not visible as an underlying member", (t) => { + // The monitor answers getInputRecording itself; the wrapped taskUtil has no such method. + t.is(typeof t.context.taskUtil.getInputRecording, "undefined"); + const monitored = new MonitoredTaskUtil(t.context.taskUtil); + t.is(typeof monitored.getInputRecording, "function"); +}); + +test("preserves a limited interface shape (custom read-only task)", (t) => { + // A spec-version interface without setTag: the monitor must not add it. + const readOnlyInterface = { + getTag: t.context.sinon.stub().returns("tagValue"), + getEnv: t.context.sinon.stub().returns("v"), + isRootProject: t.context.sinon.stub().returns(false), + }; + const monitored = new MonitoredTaskUtil(readOnlyInterface); + + t.is(monitored.setTag, undefined, "setTag stays absent"); + t.is(monitored.getTag("r", "t"), "tagValue", "getTag delegates"); + monitored.isRootProject(); + t.deepEqual(monitored.getInputRecording(), [ + {type: "isRootProject", name: "", value: "false"}, + ]); +}); diff --git a/packages/project/test/lib/build/helpers/ProjectBuildContext.js b/packages/project/test/lib/build/helpers/ProjectBuildContext.js index e5e1d30737a..43ade112b54 100644 --- a/packages/project/test/lib/build/helpers/ProjectBuildContext.js +++ b/packages/project/test/lib/build/helpers/ProjectBuildContext.js @@ -21,6 +21,7 @@ function createBuildContextStub(overrides = {}) { return { getGraph: () => ({}), getTaskRepository: () => ({}), + getBuildTime: () => new Date(), ...overrides }; } @@ -434,3 +435,101 @@ test("getBuildMetadata: has no build-manifest", (t) => { ); t.is(projectBuildContext.getBuildMetadata(), null, "Project has no build manifest"); }); + +test("resolveInputValue: env reads process.env", (t) => { + const buildContext = createBuildContextStub(); + const project = {getName: () => "project", getType: () => "type"}; + const projectBuildContext = new ProjectBuildContext(buildContext, project); + + t.teardown(() => { + delete process.env.UI5_PROJECT_BUILD_CONTEXT_TEST; + }); + delete process.env.UI5_PROJECT_BUILD_CONTEXT_TEST; + t.is(projectBuildContext.resolveInputValue("env", "UI5_PROJECT_BUILD_CONTEXT_TEST"), undefined, + "unset variable resolves to undefined"); + process.env.UI5_PROJECT_BUILD_CONTEXT_TEST = "value"; + t.is(projectBuildContext.resolveInputValue("env", "UI5_PROJECT_BUILD_CONTEXT_TEST"), "value"); +}); + +test("resolveInputValue: isRootProject normalizes the boolean", (t) => { + const rootProject = {getName: () => "root", getType: () => "type"}; + const buildContext = createBuildContextStub({getRootProject: () => rootProject}); + const projectBuildContext = new ProjectBuildContext(buildContext, rootProject); + + t.is(projectBuildContext.resolveInputValue("isRootProject", ""), "true"); +}); + +test("resolveInputValue: getDependencies normalizes the array via the graph", (t) => { + const getDependencies = sinon.stub().returns(["dep.a", "dep.b"]); + const buildContext = createBuildContextStub({ + getGraph: () => ({getDependencies, getTaskRepository: () => ({})}), + getTaskRepository: () => ({}), + }); + const project = {getName: () => "project", getType: () => "type"}; + const projectBuildContext = new ProjectBuildContext(buildContext, project); + + t.is(projectBuildContext.resolveInputValue("getDependencies", "project"), `["dep.a","dep.b"]`); + t.true(getDependencies.calledWith("project")); +}); + +test("resolveInputValue: project.getVersion reads the dependency version from the graph", (t) => { + const coreProject = {getName: () => "sap.ui.core", getVersion: () => "2.0.0"}; + const getProject = sinon.stub().callsFake((name) => (name === "sap.ui.core" ? coreProject : undefined)); + const buildContext = createBuildContextStub({ + getGraph: () => ({getProject}), + }); + const project = {getName: () => "project", getType: () => "type"}; + const projectBuildContext = new ProjectBuildContext(buildContext, project); + + t.is(projectBuildContext.resolveInputValue("project.getVersion", "sap.ui.core"), "2.0.0"); +}); + +test("resolveInputValue: unresolvable project yields undefined", (t) => { + const getProject = sinon.stub().returns(undefined); + const buildContext = createBuildContextStub({ + getGraph: () => ({getProject}), + }); + const project = {getName: () => "project", getType: () => "type"}; + const projectBuildContext = new ProjectBuildContext(buildContext, project); + + t.is(projectBuildContext.resolveInputValue("project.getVersion", "gone"), undefined, + "a project no longer in the graph resolves to undefined"); +}); + +test("resolveInputValue: time re-derives the bucket for the granularity from the build time", (t) => { + // 25 September 2026, 14:07:03 local. The lookup side must quantize the build run's shared + // timestamp (via getBuildTime), so assert against that instant's buckets. + const buildTime = new Date(2026, 8, 25, 14, 7, 3); + const buildContext = createBuildContextStub({getBuildTime: () => buildTime}); + const project = {getName: () => "project", getType: () => "type"}; + const projectBuildContext = new ProjectBuildContext(buildContext, project); + + t.is(projectBuildContext.resolveInputValue("time", "year"), "2026"); + t.is(projectBuildContext.resolveInputValue("time", "hour"), "2026-09-25T14"); +}); + +test("resolveInputValue: time with an unknown granularity yields undefined", (t) => { + const buildContext = createBuildContextStub(); + const project = {getName: () => "project", getType: () => "type"}; + const projectBuildContext = new ProjectBuildContext(buildContext, project); + + // A corrupt cache row must miss the cache, not crash the lookup. + t.is(projectBuildContext.resolveInputValue("time", "minute"), undefined); +}); + +test("resolveInputValue: unknown type yields undefined", (t) => { + const buildContext = createBuildContextStub(); + const project = {getName: () => "project", getType: () => "type"}; + const projectBuildContext = new ProjectBuildContext(buildContext, project); + + t.is(projectBuildContext.resolveInputValue("unknownType", "x"), undefined); +}); + +test("getBuildTime delegates to the build context", (t) => { + const buildTime = new Date(2026, 8, 25, 14, 7, 3); + const buildContext = createBuildContextStub({getBuildTime: () => buildTime}); + const project = {getName: () => "project", getType: () => "type"}; + const projectBuildContext = new ProjectBuildContext(buildContext, project); + + t.is(projectBuildContext.getBuildTime(), buildTime, "Returns the build context's timestamp"); +}); diff --git a/packages/project/test/lib/build/helpers/StepRunner.js b/packages/project/test/lib/build/helpers/StepRunner.js new file mode 100644 index 00000000000..b72fb1242a4 --- /dev/null +++ b/packages/project/test/lib/build/helpers/StepRunner.js @@ -0,0 +1,1406 @@ +import test from "ava"; +import StepRunner from "../../../../lib/build/helpers/StepRunner.js"; + +function createResource(resourcePath, content = resourcePath) { + return { + getPath: () => resourcePath, + getIntegrity: async () => `sha256-${content}`, + getString: async () => content, + }; +} + +// A filesystem-backed resource as the project source reader yields it: lastModified and a statically-known +// size are present (so #keyId's cheap tier applies and never reads content), while getIntegrity() would read +// and hash the content. getIntegrity throws here so a test proves the cheap tier did NOT fall through to it. +function createFsResource(resourcePath, {content = resourcePath, lastModified = 1000, size} = {}) { + return { + getPath: () => resourcePath, + getLastModified: () => lastModified, + hasSize: () => true, + getSize: async () => size ?? content.length, + getIntegrity: async () => { + throw new Error(`getIntegrity() must not be called for ${resourcePath} on the cheap key tier`); + }, + getString: async () => content, + }; +} + +// A memory-backed or generated resource: no lastModified, so #keyId falls back to the integrity tier. The +// Memory adapter and resources produced by a task carry no filesystem stat, matching this shape. +function createMemoryResource(resourcePath, content = resourcePath) { + return { + getPath: () => resourcePath, + getLastModified: () => undefined, + hasSize: () => true, + getSize: async () => content.length, + getIntegrity: async () => `sha256-${content}`, + getString: async () => content, + }; +} + +function createWorkspace(initial = []) { + const store = new Map(initial.map((res) => [res.getPath(), res])); + return { + getName: () => "workspace", + byGlob: async () => [...store.values()], + byPath: async (virPath) => store.get(virPath) ?? null, + write: async (resource) => { + store.set(resource.getPath(), resource); + }, + store, + }; +} + +// A workspace fake that additionally records the write order and the trailing write arguments, so the +// key-order flush and the argument handling of a concurrent map step are observable. +function createRecordingWorkspace(initial = []) { + const store = new Map(initial.map((res) => [res.getPath(), res])); + const writeOrder = []; + const writeArgs = []; + return { + getName: () => "workspace", + byGlob: async () => [...store.values()], + byPath: async (virPath) => store.get(virPath) ?? null, + write: async (resource, ...args) => { + writeOrder.push(resource.getPath()); + writeArgs.push(args); + store.set(resource.getPath(), resource); + }, + store, + writeOrder, + writeArgs, + }; +} + +// content by integrity so store() and a later restore() round-trip the exact bytes, exactly as the real +// SQLite CAS does across two builds. +function createReturnValueStore() { + const cas = new Map(); + return { + cas, + store: async (resources) => Promise.all(resources.map(async (res) => { + const integrity = await res.getIntegrity(); + cas.set(integrity, await res.getString()); + return {path: res.getPath(), integrity}; + })), + restore: ({path, integrity}) => ({ + getPath: () => path, + getIntegrity: async () => integrity, + getString: async () => cas.get(integrity), + restored: true, + }), + }; +} + +// Drives a StepRunner with in-memory per-stage hooks, mirroring what the TaskRunner wires around a real +// build cache. The harness owns the fakes so tests can inspect what each +// stage recorded (its per-key invocationData, folded reads/inputs, stale outputs) without the StepRunner +// exposing task-level fold accessors anymore. +// +// - prepareStage(step): returns the stage's cache verdict. Defaults to false (full run) for every step; +// pass `cacheVerdicts` to return `true` (fully cached) or a delta cacheInfo object for named steps. +// - getPreviousInvocationData(step): returns the step's previous per-key data from `previousData`. +// - createStageContext(): fresh recording context around a shared workspace/dependencies/taskUtil. +// - recordStage(step, outcome): captures the outcome under `recorded[step]` and returns the stage's +// written paths (the union of its keys' writes), which runSteps aggregates. +function makeDriver({ + steps, options, workspace = createWorkspace(), dependencies, taskUtil = {}, returnValueStore, + resolveInputValue, applyTagOperations, signal, cacheVerdicts = {}, previousData = new Map(), + notifyStepExecution, +}) { + const recorded = new Map(); + const runner = new StepRunner({ + steps, + options, + returnValueStore, + resolveInputValue, + applyTagOperations, + signal, + notifyStepExecution, + prepareStage: async (step) => (step in cacheVerdicts ? cacheVerdicts[step] : false), + getPreviousInvocationData: (step) => previousData.get(step), + // Mirrors the TaskRunner hook: reopens the stage with a live writer and demotes the full hit to a + // full re-run. The harness workspace is always writable, so reopening is a no-op here. + reopenStage: async () => false, + createStageContext: () => ({workspace, dependencies, taskUtil, monitoredTaskUtil: taskUtil}), + recordStage: async (step, outcome) => { + recorded.set(step, outcome); + const written = new Set(); + for (const data of outcome.invocationData.values()) { + (data.writes ?? []).forEach((path) => written.add(path)); + } + return [...written]; + }, + }); + return {runner, recorded, workspace}; +} + +// The per-key invocation data a stage recorded this build (the map keyed by keyId), for the assertions +// that previously inspected runner.getInvocationData().get(step). +function invocationDataOf(recorded, step) { + return recorded.get(step)?.invocationData; +} + +// --- Step-factory API (runSteps) --- + +test("runSteps runs a scalar step once and records its single unit", async (t) => { + const {runner, recorded, workspace} = makeDriver({ + steps: [ + {name: "s", run: async ({workspace}) => { + await workspace.write(createResource("/out")); + }}, + ], + }); + + await runner.runSteps(); + + t.true(workspace.store.has("/out"), "Scalar step's write persisted"); + t.is(invocationDataOf(recorded, "s").size, 1, "Scalar step recorded one implicit unit"); +}); + +test("runSteps runs a map step's each once per enumerated key", async (t) => { + const ran = []; + const {runner, recorded, workspace} = makeDriver({ + steps: [ + {name: "m", keys: async () => ["a", "b"], each: async (key, {workspace}) => { + ran.push(key); + await workspace.write(createResource(`/out/${key}`)); + }}, + ], + }); + + await runner.runSteps(); + + t.deepEqual(ran.sort(), ["a", "b"], "each ran once per key"); + t.is(invocationDataOf(recorded, "m").size, 2, "Map step recorded one unit per key"); + t.true(workspace.store.has("/out/a") && workspace.store.has("/out/b"), "Both keys' writes persisted"); +}); + +test("Concurrent map-step keys writing the same path throw the shared message", async (t) => { + const {runner} = makeDriver({ + steps: [ + {name: "m", keys: async () => ["a", "b"], each: async (key, {workspace}) => { + await workspace.write(createResource("/same")); + }}, + ], + }); + + const err = await t.throwsAsync(runner.runSteps()); + // The exact user-visible message, shared with the uncached runner through @ui5/fs/internal/stepWriteBuffer. + t.is(err.message, + "Concurrent map-step keys must not write the same resource path /same. " + + "Pass {sequential: true} if a later key must build on an earlier key's writes.", + "The same-path guard surfaces the shared message verbatim"); +}); + +test("A concurrent map step flushes its writes in key order", async (t) => { + const workspace = createRecordingWorkspace(); + const {runner} = makeDriver({ + workspace, + steps: [ + {name: "m", keys: async () => ["a", "b", "c"], each: async (key, {workspace}) => { + // Reverse the natural completion order so the key-order flush is observable. + if (key === "a") { + await new Promise((resolve) => setTimeout(resolve, 15)); + } + await workspace.write(createResource(`/${key}.out`)); + }}, + ], + }); + + await runner.runSteps(); + + t.deepEqual(workspace.writeOrder, ["/a.out", "/b.out", "/c.out"], + "Buffered writes flushed in key order regardless of completion order"); +}); + +test("A concurrent map step replays each key's write options through the flush", async (t) => { + const workspace = createRecordingWorkspace(); + const {runner} = makeDriver({ + workspace, + steps: [ + {name: "m", keys: async () => ["with", "without"], each: async (key, {workspace}) => { + if (key === "with") { + await workspace.write(createResource("/with"), {drain: true, readOnly: false}); + } else { + // No options: RecordingReaderWriter overrides _write, so AbstractReaderWriter.write has + // already defaulted options by the time it buffers; the flush replays that one object. + await workspace.write(createResource("/without")); + } + }}, + ], + }); + + await runner.runSteps(); + + t.deepEqual(workspace.writeOrder, ["/with", "/without"], "Flushed in key order"); + t.deepEqual(workspace.writeArgs, + [[{drain: true, readOnly: false}], [{drain: false, readOnly: false}]], + "The with-options write replays its options; the without-options write replays the defaulted options"); +}); + +test("A step must be either scalar or map", async (t) => { + const {runner} = makeDriver({steps: [{name: "bad"}]}); + await t.throwsAsync(runner.runSteps(), + {message: /Step 'bad' must be either a scalar step .* or a map step/}); + + const {runner: both} = makeDriver({ + steps: [{name: "bad", run: async () => {}, keys: async () => [], each: async () => {}}], + }); + await t.throwsAsync(both.runSteps(), + {message: /Step 'bad' must be either a scalar step .* or a map step/}); +}); + +test("A step's needs may only reference an earlier step", async (t) => { + const {runner} = makeDriver({ + steps: [{name: "a", needs: ["later"], run: async () => {}}, {name: "later", run: async () => {}}], + }); + await t.throwsAsync(runner.runSteps(), + {message: /Step 'a' needs 'later', which is not an earlier step/}); +}); + +test("A factory returning a non-array throws before any step runs", async (t) => { + const {runner} = makeDriver({steps: undefined}); + await t.throwsAsync(runner.runSteps(), + {message: "Step factory must return an array of step objects, got undefined"}); +}); + +test("A non-object step element names its index", async (t) => { + const {runner} = makeDriver({steps: [{name: "ok", run: async () => {}}, "nope"]}); + await t.throwsAsync(runner.runSteps(), + {message: "Step at index 1 must be an object, got a string"}); +}); + +test("A step without a name names its index", async (t) => { + const {runner} = makeDriver({steps: [{run: async () => {}}]}); + await t.throwsAsync(runner.runSteps(), + {message: "Step at index 0 must have a non-empty string 'name'"}); + + const {runner: empty} = makeDriver({steps: [{name: "", run: async () => {}}]}); + await t.throwsAsync(empty.runSteps(), + {message: "Step at index 0 must have a non-empty string 'name'"}); +}); + +test("A duplicate step name throws", async (t) => { + const {runner} = makeDriver({ + steps: [{name: "dup", run: async () => {}}, {name: "dup", run: async () => {}}], + }); + await t.throwsAsync(runner.runSteps(), {message: "Duplicate step name 'dup'"}); +}); + +test("A half-defined map step names the missing half", async (t) => { + const {runner: noEach} = makeDriver({steps: [{name: "m", keys: async () => []}]}); + await t.throwsAsync(noEach.runSteps(), + {message: "Map step 'm' must define both 'keys' and 'each' functions"}); + + const {runner: noKeys} = makeDriver({steps: [{name: "m", each: async () => {}}]}); + await t.throwsAsync(noKeys.runSteps(), + {message: "Map step 'm' must define both 'keys' and 'each' functions"}); +}); + +test("A step's needs declared as a string throws the array message, not a per-character error", async (t) => { + // Regression: 'needs' used to be iterated with for..of, so a string typo iterated characters and + // reported the first character as a missing earlier step. It must be rejected as a non-array instead. + const {runner} = makeDriver({ + steps: [{name: "a", run: async () => {}}, {name: "b", needs: "a", run: async () => {}}], + }); + await t.throwsAsync(runner.runSteps(), + {message: "Step 'b' 'needs' must be an array of earlier step names, got a string"}); +}); + +test("A non-string needs entry names its index", async (t) => { + const {runner} = makeDriver({ + steps: [{name: "a", run: async () => {}}, {name: "b", needs: ["a", 42], run: async () => {}}], + }); + await t.throwsAsync(runner.runSteps(), + {message: "Step 'b' 'needs' entries must be strings; entry 1 is a number"}); +}); + +test("A step needing itself throws, since it is not an earlier step", async (t) => { + const {runner} = makeDriver({steps: [{name: "a", needs: ["a"], run: async () => {}}]}); + await t.throwsAsync(runner.runSteps(), + {message: "Step 'a' needs 'a', which is not an earlier step"}); +}); + +test("A scalar producer's serializable return is injected into a consumer via needs", async (t) => { + let seen; + const {runner} = makeDriver({ + steps: [ + {name: "scan", run: async () => ({hasThemes: true})}, + {name: "use", needs: ["scan"], run: async ({needs}) => { + seen = needs.scan; + }}, + ], + }); + + await runner.runSteps(); + + t.deepEqual(seen, {hasThemes: true}, "The producer's return arrived as needs.scan"); +}); + +test("A producer return reaches a map step's keys and each via needs", async (t) => { + const keysSaw = []; + const eachSaw = []; + const {runner} = makeDriver({ + steps: [ + {name: "scan", run: async () => ({wanted: ["x", "y"]})}, + {name: "build", needs: ["scan"], keys: async ({needs}) => { + keysSaw.push(needs.scan); + return needs.scan.wanted; + }, each: async (key, {needs}) => { + eachSaw.push([key, needs.scan.wanted.length]); + }}, + ], + }); + + await runner.runSteps(); + + t.deepEqual(keysSaw, [{wanted: ["x", "y"]}], "keys saw the producer return"); + t.deepEqual(eachSaw.sort(), [["x", 2], ["y", 2]], "each saw the producer return per key"); +}); + +test("A resource return is injected into a consumer and stored in the CAS", async (t) => { + const returnValueStore = createReturnValueStore(); + let consumed; + const {runner} = makeDriver({ + returnValueStore, + steps: [ + {name: "make", run: async () => createResource("/made", "made-content")}, + {name: "use", needs: ["make"], run: async ({needs}) => { + consumed = await needs.make.getString(); + }}, + ], + }); + + await runner.runSteps(); + + t.is(consumed, "made-content", "The producer's returned resource arrived as needs.make"); + t.deepEqual([...returnValueStore.cas.keys()], ["sha256-made-content"], + "The returned resource's content was stored in the CAS"); +}); + +test("Delta build re-runs only the map key whose recorded read changed", async (t) => { + const build1 = makeDriver({ + steps: [ + {name: "m", keys: async () => ["a", "b"], each: async (key, {workspace}) => { + await workspace.byPath(`/in/${key}`); // recorded read + await workspace.write(createResource(`/out/${key}`)); + }}, + ], + }); + await build1.runner.runSteps(); + + const ran = []; + const cacheInfo = {changedProjectResourcePaths: ["/in/a"], changedDependencyResourcePaths: []}; + const build2 = makeDriver({ + cacheVerdicts: {m: cacheInfo}, + previousData: new Map([["m", invocationDataOf(build1.recorded, "m")]]), + steps: [ + {name: "m", keys: async () => ["a", "b"], each: async (key, {workspace}) => { + ran.push(key); + await workspace.byPath(`/in/${key}`); + await workspace.write(createResource(`/out/${key}`)); + }}, + ], + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, ["a"], "Only the key whose recorded read changed re-ran"); +}); + +test("Delta build re-runs a consumer when its producer's return changed", async (t) => { + const stepsFor = (ran) => [ + {name: "scan", run: async ({workspace}) => { + const res = await workspace.byPath("/in"); + return {v: res ? await res.getString() : "none"}; + }}, + {name: "use", needs: ["scan"], run: async ({needs, workspace}) => { + ran.push("use"); + await workspace.write(createResource("/use.out", JSON.stringify(needs.scan))); + }}, + ]; + + const build1 = makeDriver({ + workspace: createWorkspace([createResource("/in", "old")]), steps: stepsFor([]), + }); + await build1.runner.runSteps(); + + const ran = []; + // scan re-runs because its recorded read /in changed, so its return advances; use re-runs because the + // producer return it consumed changed. + const cacheInfo = {changedProjectResourcePaths: ["/in"], changedDependencyResourcePaths: []}; + const build2 = makeDriver({ + workspace: createWorkspace([createResource("/in", "new")]), + cacheVerdicts: {scan: cacheInfo, use: cacheInfo}, + previousData: new Map([ + ["scan", invocationDataOf(build1.recorded, "scan")], + ["use", invocationDataOf(build1.recorded, "use")], + ]), + steps: stepsFor(ran), + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, ["use"], "The consumer re-ran because the producer's return changed"); +}); + +test("Delta build keeps a map consumer cached when its producer is restored unchanged", async (t) => { + // A scalar step always re-runs on a delta verdict (a single implicit unit cannot be pruned), so the + // delta-path "keep the consumer cached when its producer is unchanged" behavior is exercised through a + // map consumer: its one key stays cached because the producer return it consumed did not change. + const stepsFor = (ran) => [ + {name: "scan", run: async ({workspace}) => { + const res = await workspace.byPath("/in"); + ran.push("scan"); + return {v: res ? await res.getString() : "none"}; + }}, + {name: "use", needs: ["scan"], keys: async () => ["k"], + each: async (key, {needs, workspace}) => { + ran.push("use"); + await workspace.write(createResource("/use.out", JSON.stringify(needs.scan))); + }}, + ]; + + const build1 = makeDriver({ + workspace: createWorkspace([createResource("/in", "v")]), steps: stepsFor([]), + }); + await build1.runner.runSteps(); + + const ran = []; + // scan is fully cached (verdict true), so it restores its return unchanged; use is a delta with no + // changed paths, so its key stays cached because the producer return did not change. + const cacheInfo = {changedProjectResourcePaths: [], changedDependencyResourcePaths: []}; + const build2 = makeDriver({ + workspace: createWorkspace([createResource("/in", "v")]), + cacheVerdicts: {scan: true, use: cacheInfo}, + previousData: new Map([ + ["scan", invocationDataOf(build1.recorded, "scan")], + ["use", invocationDataOf(build1.recorded, "use")], + ]), + steps: stepsFor(ran), + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, [], "Neither the restored producer nor its cached map consumer re-ran"); +}); + +test("Delta build re-runs a scalar step whose glob gains a newly matching file", async (t) => { + // A scalar step globs for themes. On build 1 the workspace has none, so the step records no reads and + // returns hasThemes:false. On build 2 a file matching the glob is added. The recorder stores resolved + // paths, not the glob pattern, so the added file is in no previous read and the per-unit reads delta + // cannot select the step. A scalar step is a single implicit unit, so it must re-run on any delta + // verdict rather than serve its stale cached return. + const stepsFor = (ran) => [ + {name: "scan", run: async ({workspace}) => { + ran.push("scan"); + const matches = await workspace.byGlob("/themes/**/library.source.less"); + return {hasThemes: matches.length > 0}; + }}, + ]; + + const build1 = makeDriver({workspace: createWorkspace([]), steps: stepsFor([])}); + await build1.runner.runSteps(); + t.deepEqual( + [...invocationDataOf(build1.recorded, "scan").values()][0].reads, [], + "Build 1 recorded no reads because the glob matched nothing"); + + const ran = []; + // The stage signature changed (the stage monitor recorded the glob), so prepareStage returns a delta + // verdict. The added path intersects none of scan's recorded (empty) reads. + const cacheInfo = { + changedProjectResourcePaths: ["/themes/my_theme/library.source.less"], + changedDependencyResourcePaths: [], + }; + const build2 = makeDriver({ + workspace: createWorkspace([createResource("/themes/my_theme/library.source.less")]), + cacheVerdicts: {scan: cacheInfo}, + previousData: new Map([["scan", invocationDataOf(build1.recorded, "scan")]]), + steps: stepsFor(ran), + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, ["scan"], "The scalar step re-ran on the delta despite no recorded read changing"); +}); + +test("Delta build re-runs a scalar step whose glob loses its last matching file", async (t) => { + // The removal direction: build 1 globs one matching file (recorded as a read), build 2 removes it. The + // removed path intersects the recorded read, so the reads delta alone would already re-run the step; + // this locks that a scalar step still re-runs when its only matching file is deleted. + const stepsFor = (ran) => [ + {name: "scan", run: async ({workspace}) => { + ran.push("scan"); + const matches = await workspace.byGlob("/themes/**/library.source.less"); + return {hasThemes: matches.length > 0}; + }}, + ]; + + const build1 = makeDriver({ + workspace: createWorkspace([createResource("/themes/my_theme/library.source.less")]), + steps: stepsFor([]), + }); + await build1.runner.runSteps(); + + const ran = []; + const cacheInfo = { + changedProjectResourcePaths: ["/themes/my_theme/library.source.less"], + changedDependencyResourcePaths: [], + }; + const build2 = makeDriver({ + workspace: createWorkspace([]), + cacheVerdicts: {scan: cacheInfo}, + previousData: new Map([["scan", invocationDataOf(build1.recorded, "scan")]]), + steps: stepsFor(ran), + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, ["scan"], "The scalar step re-ran when its last matching file was removed"); +}); + +test("Full stage-cache hit re-runs a consumer when its producer's return changed", async (t) => { + // A consumer that reads no resources has a constant stage signature, so its stage is a full cache hit + // (verdict true) even when a producer it needs re-ran with a changed return. The needs return is + // excluded from the stage signature, so nothing in the stage lookup catches the change; the full-hit + // path must check needs itself and re-run rather than serve stale cached output. + const stepsFor = (ran) => [ + {name: "scan", run: async ({workspace}) => { + const res = await workspace.byPath("/in"); + return {v: res ? await res.getString() : "none"}; + }}, + {name: "use", needs: ["scan"], run: async ({needs, workspace}) => { + ran.push("use"); + await workspace.write(createResource("/use.out", JSON.stringify(needs.scan))); + }}, + ]; + + const build1 = makeDriver({ + workspace: createWorkspace([createResource("/in", "old")]), steps: stepsFor([]), + }); + await build1.runner.runSteps(); + + const ran = []; + // scan re-runs on a delta because its recorded read /in changed, advancing its return signature. + // use's own stage signature is unchanged (it reads nothing), so its verdict is a full hit (true). + const cacheInfo = {changedProjectResourcePaths: ["/in"], changedDependencyResourcePaths: []}; + const build2 = makeDriver({ + workspace: createWorkspace([createResource("/in", "new")]), + cacheVerdicts: {scan: cacheInfo, use: true}, + previousData: new Map([ + ["scan", invocationDataOf(build1.recorded, "scan")], + ["use", invocationDataOf(build1.recorded, "use")], + ]), + steps: stepsFor(ran), + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, ["use"], + "The consumer re-ran despite a full stage-cache hit because the producer's return changed"); + t.is(build2.workspace.store.get("/use.out") && + await build2.workspace.store.get("/use.out").getString(), JSON.stringify({v: "new"}), + "The re-run consumer wrote the fresh producer return, not stale output"); +}); + +test("Full stage-cache hit stays cached when the producer's return is unchanged", async (t) => { + // The complement of the previous test: a full-hit consumer whose producer restored unchanged must + // NOT re-run, so the fast path is preserved for the common case. + const stepsFor = (ran) => [ + {name: "scan", run: async ({workspace}) => { + const res = await workspace.byPath("/in"); + return {v: res ? await res.getString() : "none"}; + }}, + {name: "use", needs: ["scan"], run: async ({needs, workspace}) => { + ran.push("use"); + await workspace.write(createResource("/use.out", JSON.stringify(needs.scan))); + }}, + ]; + + const build1 = makeDriver({ + workspace: createWorkspace([createResource("/in", "v")]), steps: stepsFor([]), + }); + await build1.runner.runSteps(); + + const ran = []; + // scan restores unchanged (full hit, no change), use is a full hit too. The producer return did not + // change, so use stays cached. + const build2 = makeDriver({ + workspace: createWorkspace([createResource("/in", "v")]), + cacheVerdicts: {scan: true, use: true}, + previousData: new Map([ + ["scan", invocationDataOf(build1.recorded, "scan")], + ["use", invocationDataOf(build1.recorded, "use")], + ]), + steps: stepsFor(ran), + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, [], "A full-hit consumer stays cached when its producer's return is unchanged"); +}); + +test("Full stage-cache hit does not replay per-key tags: setStage owns the full-hit tag operations", async (t) => { + // On a full hit the stage was installed via ProjectResources.setStage with its complete cached tag + // operations (captured at record time, including the keys enumerator's), which reach the live tag + // collection through #applyCachedResourceTags when a later stage reads over it. Replaying each key's + // subset again would be redundant work, so #restoreCachedStage must not call the applyTagOperations hook. + // The delta path still replays (see the next test), because there the stage re-runs and is re-recorded. + let replayCount = 0; + const previous = new Map([ + ["key-1", {returns: null, tagOperations: [ + {op: "set", path: "/out", tag: "ui5:IsDebugVariant", value: true}, + ]}], + ]); + const {runner} = makeDriver({ + steps: [{name: "s", run: async () => undefined}], + cacheVerdicts: {s: true}, + previousData: new Map([["s", previous]]), + applyTagOperations: () => replayCount++, + }); + + await runner.runSteps(); + + t.is(replayCount, 0, + "A full-hit restore does not replay per-key tags; setStage already installed the complete set"); +}); + +test("Delta stage hit still replays a restored key's tags so the re-recorded stage keeps them", async (t) => { + // Contrast with the full-hit test: on a delta build the stage re-runs and is re-recorded, so a key + // served from cache must have its tags replayed into the monitored collection to be captured and + // persisted again. This locks that the delta-path replay in #runGroup stays. + let replayCount = 0; + const previous = new Map([ + ["string:/a", {returns: null, reads: [], dependencyReads: [], inputs: [], needsInputs: [], writes: [], + tagOperations: [{op: "set", path: "/a", tag: "ui5:IsDebugVariant", value: true}]}], + ]); + // A map step whose single key "/a" is unchanged, so the delta leaves it cached rather than re-running it. + const {runner} = makeDriver({ + workspace: createWorkspace([createResource("/a")]), + steps: [{name: "m", keys: async () => ["/a"], each: async () => undefined}], + cacheVerdicts: {m: {changedProjectResourcePaths: [], changedDependencyResourcePaths: []}}, + previousData: new Map([["m", previous]]), + applyTagOperations: () => replayCount++, + }); + + await runner.runSteps(); + + t.is(replayCount, 1, "The restored key's tags were replayed on the delta path"); +}); + +test("A scalar producer's full hit with no sidecar re-runs instead of handing undefined to a consumer", async (t) => { + // The stage result and its per-key sidecar are two independent rows (independent write conditions), so + // a stage_metadata hit can arrive with no matching steps row (previous === undefined). Restoring it + // would store an undefined return for the scalar producer, which crashes a consumer dereferencing it + // through needs. #canRestoreCachedStage rejects the unrestorable full hit so the producer re-runs. + const ran = []; + const steps = [ + {name: "scan", run: async ({workspace}) => { + ran.push("scan"); + return {hasThemes: (await workspace.byGlob("/themes/*")).length > 0}; + }}, + {name: "use", needs: ["scan"], run: async ({needs, workspace}) => { + ran.push("use"); + // Dereferences the producer return exactly as generateThemeDesignerResources does; this throws + // if scan handed down undefined. + await workspace.write(createResource("/use.out", JSON.stringify(needs.scan.hasThemes))); + }}, + ]; + + const {runner, workspace} = makeDriver({ + workspace: createWorkspace([createResource("/themes/a", "x")]), + // scan reports a full hit, but no previousData is seeded for it: the sidecar is absent. + cacheVerdicts: {scan: true, use: false}, + steps, + }); + + await runner.runSteps(); + + t.deepEqual(ran, ["scan", "use"], "scan re-ran rather than restoring an undefined return"); + t.is(await workspace.store.get("/use.out").getString(), "true", + "The consumer saw the freshly produced return"); +}); + +test("A zero-key map step gated behind a needs flag re-runs when the flag flips (finding 6)", async (t) => { + // generateThemeDesignerResources' themes map step gates keys() behind needs.scan.hasThemes. With no + // themes it enumerates zero keys and persists an empty sidecar, so on the next build its stage is a + // full hit with an empty previous map. #needsReturnChanged cannot examine an empty map, so a flipped + // scan return would be missed. #canRestoreCachedStage treats an empty previous on a needs-declaring + // step as unrestorable, forcing a re-run that re-enumerates keys() against the new needs. + const makeSteps = (ran) => [ + {name: "scan", run: async ({workspace}) => ({ + hasThemes: (await workspace.byGlob("/themes/*")).length > 0, + })}, + {name: "themes", needs: ["scan"], + keys: async ({needs, workspace}) => needs.scan.hasThemes ? workspace.byGlob("/themes/*") : [], + each: async (theme, {workspace}) => { + ran.push(theme.getPath()); + await workspace.write(createResource(`${theme.getPath()}.css`, "built")); + }}, + ]; + + // Build 1: no themes. scan.hasThemes is false, themes enumerates zero keys and records an empty map. + const build1 = makeDriver({workspace: createWorkspace([]), steps: makeSteps([])}); + await build1.runner.runSteps(); + t.is(invocationDataOf(build1.recorded, "themes").size, 0, "themes recorded zero keys with no themes"); + + // Build 2: the first theme is added. scan re-runs (its glob gained a match) and now returns + // {hasThemes: true}. themes reads nothing when gated, so its stage signature did not move: a full hit. + const ran = []; + const build2 = makeDriver({ + workspace: createWorkspace([createResource("/themes/base/library.source.less", "less")]), + cacheVerdicts: { + scan: {changedProjectResourcePaths: ["/themes/base/library.source.less"], + changedDependencyResourcePaths: []}, + themes: true, + }, + previousData: new Map([ + ["scan", invocationDataOf(build1.recorded, "scan")], + ["themes", invocationDataOf(build1.recorded, "themes")], + ]), + steps: makeSteps(ran), + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, ["/themes/base/library.source.less"], + "themes re-enumerated and built the newly added theme despite its full stage-cache hit"); +}); + +test("A map producer's return signature is invariant under key order", async (t) => { + // The producer returns the same per-key values on both builds but enumerates its keys in a different + // order (entries follow keys() order, which for the shipped tasks is workspace.byGlob(...) order). A + // positional signature would move with the order and re-run the consumer for nothing; the signature + // must be order-independent so the consumer stays cached when nothing it consumes changed. + const makeSteps = (keys, ran) => [ + {name: "make", keys: async () => keys, each: async (key) => ({v: key})}, + {name: "use", needs: ["make"], run: async () => { + ran.push("use"); + }}, + ]; + + const build1 = makeDriver({steps: makeSteps(["a", "b"], [])}); + await build1.runner.runSteps(); + + const ran = []; + // make takes a delta verdict with no changed paths, so all its keys stay cached (restored), but it + // enumerates them in reversed order this build. use is a full hit; it re-runs only if make's return + // signature changed. + const noChange = {changedProjectResourcePaths: [], changedDependencyResourcePaths: []}; + const build2 = makeDriver({ + cacheVerdicts: {make: noChange, use: true}, + previousData: new Map([ + ["make", invocationDataOf(build1.recorded, "make")], + ["use", invocationDataOf(build1.recorded, "use")], + ]), + steps: makeSteps(["b", "a"], ran), + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, [], "The consumer stayed cached because the producer return is order-independent"); +}); + +test("A map producer's changed return re-runs its consumer", async (t) => { + // The complement of the invariance test: when a key's return changes, the producer's signature + // must change so the consumer re-runs rather than serving stale output. + const makeSteps = (value, ran) => [ + {name: "make", keys: async () => ["a", "b"], each: async (key, {workspace}) => { + await workspace.byPath(`/in/${key}`); // recorded read, so a changed path re-runs this key + return {v: key === "a" ? value : key}; + }}, + {name: "use", needs: ["make"], run: async () => { + ran.push("use"); + }}, + ]; + + const build1 = makeDriver({steps: makeSteps("a", [])}); + await build1.runner.runSteps(); + + const ran = []; + // make re-runs key "a" because its recorded read changed, advancing the producer return. use is a full + // hit; it must re-run because the producer return it consumed changed. + const cacheInfo = {changedProjectResourcePaths: ["/in/a"], changedDependencyResourcePaths: []}; + const build2 = makeDriver({ + cacheVerdicts: {make: cacheInfo, use: true}, + previousData: new Map([ + ["make", invocationDataOf(build1.recorded, "make")], + ["use", invocationDataOf(build1.recorded, "use")], + ]), + steps: makeSteps("a2", ran), + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, ["use"], "The consumer re-ran because the producer's return changed"); +}); + +test("A producer's return signature is a fixed-size digest, not a raw serialization", async (t) => { + // The consumer records each producer it needs under needsInputs[].value, which is the producer's + // return signature. That value must be a SHA-256 hex digest regardless of producer shape (finding 5): + // a raw map serialization is O(producer key count) and is copied into every consumer unit, so a map + // producer feeding a map consumer would persist an O(producers x consumers) sidecar. A digest keeps the + // recorded value O(1) and consistent with every other signature in the cache system. + const digest = /^[0-9a-f]{64}$/; + + // Map producer: enough keys that a raw serialization would be visibly long. + const mapBuild = makeDriver({ + steps: [ + {name: "make", keys: async () => ["a", "b", "c", "d"], each: async (key) => ({v: key})}, + {name: "use", needs: ["make"], run: async () => {}}, + ], + }); + await mapBuild.runner.runSteps(); + const mapNeeds = [...invocationDataOf(mapBuild.recorded, "use").values()][0].needsInputs; + t.is(mapNeeds.length, 1, "The consumer recorded the one producer it needs"); + t.is(mapNeeds[0].name, "make", "Recorded under the producer's name"); + t.regex(mapNeeds[0].value, digest, "A map producer's return signature is a SHA-256 hex digest"); + + // Scalar producer: its branch is hashed too, so a large scalar value does not land verbatim either. + const scalarBuild = makeDriver({ + steps: [ + {name: "make", run: async () => ({wanted: ["x", "y", "z"]})}, + {name: "use", needs: ["make"], run: async () => {}}, + ], + }); + await scalarBuild.runner.runSteps(); + const scalarNeeds = [...invocationDataOf(scalarBuild.recorded, "use").values()][0].needsInputs; + t.regex(scalarNeeds[0].value, digest, "A scalar producer's return signature is a SHA-256 hex digest"); +}); + +test("A map step honors sequential so a later key reads an earlier key's write", async (t) => { + let secondSawFirst = false; + const {runner} = makeDriver({ + steps: [ + {name: "m", sequential: true, keys: async () => ["first", "second"], each: async (key, {workspace}) => { + if (key === "first") { + await workspace.write(createResource("/shared")); + } else { + secondSawFirst = !!(await workspace.byPath("/shared")); + } + }}, + ], + }); + + await runner.runSteps(); + + t.true(secondSawFirst, "Sequential map step made the first key's write visible to the second"); +}); + +test("A later step sees an earlier step's write", async (t) => { + // The stages share one workspace here (the harness's single fake); in the real pipeline a later + // stage reads an earlier stage's writer through the prioritized reader stack. + let laterSaw = false; + const {runner} = makeDriver({ + steps: [ + {name: "first", run: async ({workspace}) => { + await workspace.write(createResource("/from-first")); + }}, + {name: "second", run: async ({workspace}) => { + laterSaw = !!(await workspace.byPath("/from-first")); + }}, + ], + }); + + await runner.runSteps(); + + t.true(laterSaw, "The second step read the first step's write through the stage"); +}); + +test("A removed map key's output is reported stale", async (t) => { + const build1 = makeDriver({ + steps: [ + {name: "m", keys: async () => ["a", "b"], each: async (key, {workspace}) => { + await workspace.write(createResource(`/out/${key}`)); + }}, + ], + }); + await build1.runner.runSteps(); + + const cacheInfo = {changedProjectResourcePaths: [], changedDependencyResourcePaths: []}; + const build2 = makeDriver({ + cacheVerdicts: {m: cacheInfo}, + previousData: new Map([["m", invocationDataOf(build1.recorded, "m")]]), + steps: [ + {name: "m", keys: async () => ["a"], each: async (key, {workspace}) => { + await workspace.write(createResource(`/out/${key}`)); + }}, + ], + }); + await build2.runner.runSteps(); + + t.deepEqual(build2.recorded.get("m").staleOutputs, ["/out/b"], "The dropped key's output is stale"); +}); + +test("runSteps over an empty step list does nothing", async (t) => { + // An empty step list still drives the task's single stage (prepareStage/recordStage with undefined), + // so the empty stage caches; nothing is recorded per key. + const {runner, recorded} = makeDriver({steps: []}); + const {anyStepExecuted} = await runner.runSteps(); + t.true(anyStepExecuted, "An empty step list ran its stage (nothing cached to skip)"); + t.is(invocationDataOf(recorded, undefined).size, 0, "No per-key invocation data recorded"); + t.deepEqual(recorded.get(undefined).staleOutputs, [], "No stale outputs"); +}); + +test("An empty step list served from cache reports the task as skipped", async (t) => { + const {runner, recorded} = makeDriver({steps: [], cacheVerdicts: {undefined: true}}); + const {anyStepExecuted} = await runner.runSteps(); + t.false(anyStepExecuted, "A fully-cached empty stage counts as skipped"); + t.is(recorded.size, 0, "Nothing recorded when the empty stage is served from cache"); +}); + +test("options passed to the runner reaches each step's context", async (t) => { + let seenScalar; + let seenEach; + const {runner} = makeDriver({ + options: {pattern: "/**/*.js"}, + steps: [ + {name: "s", run: async ({options}) => { + seenScalar = options; + }}, + {name: "m", keys: async ({options}) => [options.pattern], each: async (key, {options}) => { + seenEach = options; + }}, + ], + }); + + await runner.runSteps(); + + t.deepEqual(seenScalar, {pattern: "/**/*.js"}, "Scalar step received the task options"); + t.deepEqual(seenEach, {pattern: "/**/*.js"}, "Map step received the task options"); +}); + +test("A re-run key's dropped output is stale while a cached key's output is kept", async (t) => { + const stepsFor = (paths) => [ + {name: "m", keys: async () => ["a", "b"], each: async (key, {workspace}) => { + await workspace.byPath(`/in/${key}`); // recorded read + for (const path of paths[key]) { + await workspace.write(createResource(path)); + } + }}, + ]; + + const build1 = makeDriver({steps: stepsFor({a: ["/out/a", "/out/a.extra"], b: ["/out/b"]})}); + await build1.runner.runSteps(); + + // Only key 'a' re-runs, and it writes one path less than before. Key 'b' is served from cache, so its + // output must survive even though this build never wrote it. + const cacheInfo = {changedProjectResourcePaths: ["/in/a"], changedDependencyResourcePaths: []}; + const build2 = makeDriver({ + cacheVerdicts: {m: cacheInfo}, + previousData: new Map([["m", invocationDataOf(build1.recorded, "m")]]), + steps: stepsFor({a: ["/out/a"], b: ["/out/b"]}), + }); + await build2.runner.runSteps(); + + t.deepEqual(build2.recorded.get("m").staleOutputs, ["/out/a.extra"], + "Only the path the re-run key stopped writing is stale"); +}); + +test("A removed key's output stays when a cached key still writes it", async (t) => { + const stepsFor = (keys) => [ + {name: "m", sequential: true, keys: async () => keys, each: async (key, {workspace}) => { + await workspace.write(createResource("/out/shared", "shared")); + await workspace.write(createResource(`/out/${key}`)); + }}, + ]; + + const build1 = makeDriver({steps: stepsFor(["a", "b"])}); + await build1.runner.runSteps(); + + // Key 'b' is gone and key 'a' is served from cache, so '/out/shared' is written by nobody this build. + // It is still owned by the cached key, so dropping key 'b' must not take it down. + const cacheInfo = {changedProjectResourcePaths: [], changedDependencyResourcePaths: []}; + const build2 = makeDriver({ + cacheVerdicts: {m: cacheInfo}, + previousData: new Map([["m", invocationDataOf(build1.recorded, "m")]]), + steps: stepsFor(["a"]), + }); + await build2.runner.runSteps(); + + t.deepEqual(build2.recorded.get("m").staleOutputs, ["/out/b"], + "Only the removed key's exclusive output is stale"); +}); + +// --- Key identity (#keyId) --- + +test("Key identity is stable across builds for an unchanged resource and uses the cheap tier", async (t) => { + // The same filesystem-backed resource (same lastModified + size) on two builds. getIntegrity throws, so + // the build only completes if #keyId used the lastModified + size tier and never read the content. + const stepsFor = () => [ + {name: "m", keys: async ({workspace}) => workspace.byGlob(), each: async () => {}}, + ]; + + const build1 = makeDriver({ + workspace: createWorkspace([createFsResource("/in/a", {lastModified: 1000, size: 3})]), + steps: stepsFor(), + }); + await build1.runner.runSteps(); + const keyId1 = [...invocationDataOf(build1.recorded, "m").keys()]; + + const build2 = makeDriver({ + workspace: createWorkspace([createFsResource("/in/a", {lastModified: 1000, size: 3})]), + steps: stepsFor(), + }); + await build2.runner.runSteps(); + const keyId2 = [...invocationDataOf(build2.recorded, "m").keys()]; + + t.deepEqual(keyId2, keyId1, "An unchanged resource keeps the same key identity across builds"); +}); + +test("Key identity changes when a resource's content changes", async (t) => { + // A content edit moves lastModified (and here size), so the cheap tier yields a new key identity. + const stepsFor = () => [ + {name: "m", keys: async ({workspace}) => workspace.byGlob(), each: async () => {}}, + ]; + + const build1 = makeDriver({ + workspace: createWorkspace([createFsResource("/in/a", {content: "old", lastModified: 1000, size: 3})]), + steps: stepsFor(), + }); + await build1.runner.runSteps(); + const [keyId1] = [...invocationDataOf(build1.recorded, "m").keys()]; + + const build2 = makeDriver({ + workspace: createWorkspace([createFsResource("/in/a", {content: "newer", lastModified: 2000, size: 5})]), + steps: stepsFor(), + }); + await build2.runner.runSteps(); + const [keyId2] = [...invocationDataOf(build2.recorded, "m").keys()]; + + t.not(keyId2, keyId1, "A changed resource yields a different key identity"); +}); + +test("A content change drops the previous output rather than serving it stale", async (t) => { + // The property the integrity hash guaranteed, now carried by lastModified + size: when a key resource's + // content changes, its key identity changes, so the old key disappears and its output is dropped as stale, + // while the unit re-runs under the new key producing fresh output. A stale (not re-run, not dropped) key + // would keep serving the old output. The output path is derived from the content so the drop is observable: + // the old key wrote /out/old, the re-run writes /out/new, and /out/old must be reported stale. + const stepsFor = (ran) => [ + {name: "m", keys: async ({workspace}) => workspace.byGlob(), each: async (key, {workspace}) => { + const content = await key.getString(); + ran?.push(content); + await workspace.write(createResource(`/out/${content}`)); + }}, + ]; + + const build1 = makeDriver({ + workspace: createWorkspace([createFsResource("/in/a", {content: "old", lastModified: 1000, size: 3})]), + steps: stepsFor(), + }); + await build1.runner.runSteps(); + const previous = invocationDataOf(build1.recorded, "m"); + + // A delta build whose changed-path verdict does NOT list /in/a: the re-run is driven solely by the new key + // identity, exactly the case the integrity hash existed to cover (a mtime-moving edit the stage's own + // changed-path delta did not surface, e.g. because no unit recorded a read of /in/a). + const ran = []; + const build2 = makeDriver({ + workspace: createWorkspace([createFsResource("/in/a", {content: "new", lastModified: 2000, size: 3})]), + cacheVerdicts: {m: {changedProjectResourcePaths: [], changedDependencyResourcePaths: []}}, + previousData: new Map([["m", previous]]), + steps: stepsFor(ran), + }); + await build2.runner.runSteps(); + + t.deepEqual(ran, ["new"], "The changed-content key re-ran"); + t.true(build2.workspace.store.has("/out/new"), "The re-run produced fresh output"); + t.deepEqual(build2.recorded.get("m").staleOutputs, ["/out/old"], + "The previous key's output is dropped as stale, not served from cache"); +}); + +test("Key identity falls back to integrity when lastModified is missing", async (t) => { + // A memory-backed or generated resource has no lastModified, so #keyId uses the integrity tier. Identity + // is still stable for identical content and changes with content. + const stepsFor = () => [ + {name: "m", keys: async ({workspace}) => workspace.byGlob(), each: async () => {}}, + ]; + + const build1 = makeDriver({ + workspace: createWorkspace([createMemoryResource("/mem/a", "same")]), steps: stepsFor(), + }); + await build1.runner.runSteps(); + const [stable1] = [...invocationDataOf(build1.recorded, "m").keys()]; + + const build2 = makeDriver({ + workspace: createWorkspace([createMemoryResource("/mem/a", "same")]), steps: stepsFor(), + }); + await build2.runner.runSteps(); + const [stable2] = [...invocationDataOf(build2.recorded, "m").keys()]; + + t.is(stable2, stable1, "A memory resource with unchanged content keeps its integrity-tier key identity"); + + const build3 = makeDriver({ + workspace: createWorkspace([createMemoryResource("/mem/a", "changed")]), steps: stepsFor(), + }); + await build3.runner.runSteps(); + const [changed3] = [...invocationDataOf(build3.recorded, "m").keys()]; + + t.not(changed3, stable1, "A memory resource's changed content yields a different integrity-tier key"); +}); + +test("A filesystem key and a memory key never collide on the same path", async (t) => { + // The tier prefixes (m/s vs i) keep a stat-tiered key distinct from an integrity-tiered key for the same + // path, so a resource that changes provenance between builds is treated as new rather than aliasing. + const stepsFor = (resource) => [ + {name: "m", keys: async () => [resource], each: async () => {}}, + ]; + + const fsBuild = makeDriver({steps: stepsFor(createFsResource("/x", {lastModified: 1000, size: 3}))}); + await fsBuild.runner.runSteps(); + const [fsKey] = [...invocationDataOf(fsBuild.recorded, "m").keys()]; + + const memBuild = makeDriver({steps: stepsFor(createMemoryResource("/x", "abc"))}); + await memBuild.runner.runSteps(); + const [memKey] = [...invocationDataOf(memBuild.recorded, "m").keys()]; + + t.not(fsKey, memKey, "A stat-tiered key and an integrity-tiered key for the same path differ"); +}); + +test("A step's needs is frozen, so one unit cannot leak into its siblings", async (t) => { + const seen = []; + let keysError; + let eachError; + const {runner} = makeDriver({ + steps: [ + {name: "produce", run: async () => ({v: "original"})}, + {name: "consume", needs: ["produce"], keys: async ({needs}) => { + keysError = t.throws(() => { + needs.produce = {v: "from keys"}; + }, {instanceOf: TypeError}); + return ["a", "b"]; + }, sequential: true, each: async (key, {needs}) => { + seen.push([key, needs.produce.v]); + eachError ??= t.throws(() => { + needs.produce = {v: `from ${key}`}; + }, {instanceOf: TypeError}); + }}, + ], + }); + + await runner.runSteps(); + + t.truthy(keysError, "Assigning to needs from the keys enumerator throws"); + t.truthy(eachError, "Assigning to needs from a unit throws"); + t.deepEqual(seen, [["a", "original"], ["b", "original"]], + "Every unit sees the producer's return, unaffected by its siblings"); +}); + +test("notifyStepExecution fires before the first executing step does any work", async (t) => { + const order = []; + const {runner} = makeDriver({ + notifyStepExecution: (isDifferentialBuild) => order.push(`notify:${isDifferentialBuild}`), + steps: [ + {name: "s1", run: async () => { + order.push("s1"); + }}, + {name: "s2", run: async () => { + order.push("s2"); + }}, + ], + }); + + await runner.runSteps(); + + t.deepEqual(order, ["notify:false", "s1", "s2"], + "The task is announced once, before the first step runs"); +}); + +test("notifyStepExecution reports the first executing stage's delta verdict", async (t) => { + const build1 = makeDriver({ + steps: [ + {name: "s1", run: async ({workspace}) => { + await workspace.write(createResource("/out/1")); + }}, + {name: "s2", run: async ({workspace}) => { + await workspace.byPath("/in"); + await workspace.write(createResource("/out/2")); + }}, + ], + }); + await build1.runner.runSteps(); + + const notified = []; + const build2 = makeDriver({ + notifyStepExecution: (isDifferentialBuild) => notified.push(isDifferentialBuild), + // s1 is served from cache entirely, so the first stage that executes is the delta stage s2. + cacheVerdicts: { + s1: true, + s2: {changedProjectResourcePaths: ["/in"], changedDependencyResourcePaths: []}, + }, + previousData: new Map([ + ["s1", invocationDataOf(build1.recorded, "s1")], + ["s2", invocationDataOf(build1.recorded, "s2")], + ]), + steps: [ + {name: "s1", run: async ({workspace}) => { + await workspace.write(createResource("/out/1")); + }}, + {name: "s2", run: async ({workspace}) => { + await workspace.byPath("/in"); + await workspace.write(createResource("/out/2")); + }}, + ], + }); + await build2.runner.runSteps(); + + t.deepEqual(notified, [true], "Reported once, as a differential build"); +}); + +test("notifyStepExecution is not called when every step is served from cache", async (t) => { + // A real full hit carries the stage's one-entry scalar sidecar, so seed it from a prior build: + // a full hit with no sidecar is treated as unrestorable and re-runs (see #canRestoreCachedStage). + const build1 = makeDriver({ + steps: [ + {name: "s1", run: async () => undefined}, + {name: "s2", run: async () => undefined}, + ], + }); + await build1.runner.runSteps(); + + let notified = 0; + const {runner} = makeDriver({ + notifyStepExecution: () => notified++, + cacheVerdicts: {s1: true, s2: true}, + previousData: new Map([ + ["s1", invocationDataOf(build1.recorded, "s1")], + ["s2", invocationDataOf(build1.recorded, "s2")], + ]), + steps: [ + {name: "s1", run: async () => undefined}, + {name: "s2", run: async () => undefined}, + ], + }); + + const {anyStepExecuted} = await runner.runSteps(); + + t.false(anyStepExecuted, "A fully cached task counts as skipped"); + t.is(notified, 0, "A skipped task is never announced as running"); +}); + +test("The stage fold deduplicates reads shared across keys", async (t) => { + // Two keys each read the same shared path plus one of their own. The recorder stores resolved paths, so a + // path read by both keys would otherwise appear once per key in the fold. The fold must collapse it: the + // request graph keys on a Set, so a duplicated path is wasted work (an inflated recording the TaskRunner + // concatenates and the request-key set rebuilds), never a signature difference. + const {runner, recorded} = makeDriver({ + steps: [ + {name: "m", keys: async () => ["a", "b"], each: async (key, {workspace, dependencies}) => { + await workspace.byPath("/shared"); // read by every key + await workspace.byPath(`/in/${key}`); // read by this key only + await dependencies.byPath("/dep/shared"); // dependency read by every key + }}, + ], + dependencies: { + getName: () => "dependencies", + byPath: async () => null, + byGlob: async () => [], + }, + }); + + await runner.runSteps(); + + const {foldedReads} = recorded.get("m"); + t.deepEqual(foldedReads.project.paths.slice().sort(), ["/in/a", "/in/b", "/shared"], + "Each project path appears exactly once, across the union of both keys' reads"); + t.deepEqual(foldedReads.dependencies.paths, ["/dep/shared"], + "The shared dependency read is folded once, not once per key"); + t.is(foldedReads.project.paths.length, new Set(foldedReads.project.paths).size, + "The folded project paths carry no duplicates"); +}); + +test("Deduplicating the fold does not change the set of reads it represents", async (t) => { + // The signature downstream is a function of the SET of folded paths (the request graph dedups anyway), so + // deduplication must preserve that set exactly: every path any key read is present, and nothing else is. + // Compare the deduplicated fold against the union assembled by hand from the per-key invocation data. + const {runner, recorded} = makeDriver({ + steps: [ + {name: "m", keys: async () => ["a", "b", "c"], each: async (key, {workspace}) => { + await workspace.byPath("/common"); // all three keys + await workspace.byPath(key === "c" ? "/common" : `/in/${key}`); // c reads /common twice + }}, + ], + }); + + await runner.runSteps(); + + const invocationData = invocationDataOf(recorded, "m"); + const expected = new Set(); + for (const data of invocationData.values()) { + for (const path of data.reads) { + expected.add(path); + } + } + const {foldedReads} = recorded.get("m"); + t.deepEqual(new Set(foldedReads.project.paths), expected, + "The deduplicated fold represents exactly the union of every key's reads"); + t.is(foldedReads.project.paths.length, expected.size, "with one entry per unique path"); +}); + +test("A cached key's read, unseen by the stage monitor, stays in the stage fold on a delta build", async (t) => { + // The property the fold exists to preserve: on a delta build only the re-run keys read through the + // stage-level monitored readers, so a key served from cache contributes nothing the monitor sees. Its + // recorded read must still key the stage, or the next build looks the stage up under a signature missing + // that read and never finds it. The fold recovers it from the stage's complete per-key invocation data. + // + // Removing #foldStageKeys (so recordStage receives no foldedReads) makes this fail: foldedReads.project + // would not carry /in/b, the cached key's read. + const build1 = makeDriver({ + steps: [ + {name: "m", keys: async () => ["a", "b"], each: async (key, {workspace}) => { + await workspace.byPath(`/in/${key}`); // each key reads its own input + await workspace.write(createResource(`/out/${key}`)); + }}, + ], + }); + await build1.runner.runSteps(); + + // A delta that re-runs only key 'a' (its input changed). Key 'b' is served from cache: it does not run, so + // the stage monitor never observes its read of /in/b. + const cacheInfo = {changedProjectResourcePaths: ["/in/a"], changedDependencyResourcePaths: []}; + const build2 = makeDriver({ + cacheVerdicts: {m: cacheInfo}, + previousData: new Map([["m", invocationDataOf(build1.recorded, "m")]]), + steps: [ + {name: "m", keys: async () => ["a", "b"], each: async (key, {workspace}) => { + await workspace.byPath(`/in/${key}`); + await workspace.write(createResource(`/out/${key}`)); + }}, + ], + }); + await build2.runner.runSteps(); + + const {foldedReads} = build2.recorded.get("m"); + t.true(foldedReads.project.paths.includes("/in/b"), + "The cached key's read is folded into the stage's reads, though the monitor never saw it this build"); + t.true(foldedReads.project.paths.includes("/in/a"), + "The re-run key's read is folded in too"); +}); + +test("A cached key's glob pattern stays in the stage fold on a delta build (finding 4)", async (t) => { + // The pattern counterpart of the previous test. A map-step key's each issues a glob; on a delta build + // where that key is cached, its each does not re-run, so the glob is not re-issued and the stage monitor + // never sees the pattern. The pattern must still key the stage, or a newly matching file would not move + // the stage signature and the stage would stay a full hit (buildThemes' per-theme themesPattern check). + // The fold recovers the pattern from the cached key's recorded invocation data. + const stepsFor = () => [ + {name: "m", + keys: async () => ["a", "b"], // keys() issues no glob, so patterns come only from each + each: async (key, {workspace}) => { + await workspace.byPath(`/in/${key}`); // the delta driver re-runs the key whose input changed + await workspace.byGlob(`/scan/${key}/*`); // this key's own glob, dropped on cache unless folded + await workspace.write(createResource(`/out/${key}`)); + }}, + ]; + + const build1 = makeDriver({steps: stepsFor()}); + await build1.runner.runSteps(); + + // A delta that re-runs only key 'a'. Key 'b' is served from cache, so its glob /scan/b/* is not re-issued + // this build and reaches the stage request set only through the fold. + const build2 = makeDriver({ + cacheVerdicts: {m: {changedProjectResourcePaths: ["/in/a"], changedDependencyResourcePaths: []}}, + previousData: new Map([["m", invocationDataOf(build1.recorded, "m")]]), + steps: stepsFor(), + }); + await build2.runner.runSteps(); + + const {foldedReads} = build2.recorded.get("m"); + t.true(foldedReads.project.patterns.includes("/scan/b/*"), + "The cached key's glob pattern is folded in, so a newly matching file still moves the stage signature"); + t.true(foldedReads.project.patterns.includes("/scan/a/*"), + "The re-run key's glob pattern is folded in too"); +}); + diff --git a/packages/project/test/lib/build/helpers/TaskUtil.js b/packages/project/test/lib/build/helpers/TaskUtil.js index 694cb84ed43..30c85acd0bf 100644 --- a/packages/project/test/lib/build/helpers/TaskUtil.js +++ b/packages/project/test/lib/build/helpers/TaskUtil.js @@ -258,6 +258,43 @@ test("resourceFactory", (t) => { "resourceFactory function createFlatReader is available"); }); +test("getTime quantizes the build run's timestamp to the requested granularity", (t) => { + // 25 September 2026, 14:07:03 local. getTime must quantize the build run's fixed timestamp + // (from getBuildTime), not a fresh new Date(), so assert against that instant's buckets. + const buildTime = new Date(2026, 8, 25, 14, 7, 3); + const taskUtil = new TaskUtil({ + projectBuildContext: { + getBuildTime: () => buildTime + } + }); + + t.is(taskUtil.getTime("year"), "2026", "year bucket derives from the build time"); + t.is(taskUtil.getTime("hour"), "2026-09-25T14", "hour bucket derives from the build time"); +}); + +test("getTime throws for an unknown granularity", (t) => { + const taskUtil = new TaskUtil({ + projectBuildContext: { + getBuildTime: () => new Date() + } + }); + + const err = t.throws(() => taskUtil.getTime("second")); + t.is(err.message, `Invalid time granularity "second". Expected one of: year, month, day, hour`); +}); + +test("getBuildTime returns the build run's raw timestamp", (t) => { + // Unlike getTime, getBuildTime returns the underlying Date unquantized and untracked. + const buildTime = new Date(2026, 8, 25, 14, 7, 3); + const taskUtil = new TaskUtil({ + projectBuildContext: { + getBuildTime: () => buildTime + } + }); + + t.is(taskUtil.getBuildTime(), buildTime, "returns the build context's Date instance"); +}); + test("registerCleanupTask", (t) => { const registerCleanupTaskStub = sinon.stub(); const taskUtil = new TaskUtil({ @@ -450,6 +487,9 @@ test("getInterface: specVersion 3.0", (t) => { t.is(typeof interfacedTaskUtil.isRootProject, "function", "function isRootProject is provided"); t.is(typeof interfacedTaskUtil.registerCleanupTask, "function", "function registerCleanupTask is provided"); t.is(typeof interfacedTaskUtil.getProject, "function", "function registerCleanupTask is provided"); + t.is(interfacedTaskUtil.getEnv, undefined, "getEnv is not provided below specVersion 5.0"); + t.is(interfacedTaskUtil.getTime, undefined, "getTime is not provided below specVersion 5.0"); + t.is(interfacedTaskUtil.getBuildTime, undefined, "getBuildTime is not provided below specVersion 5.0"); // getProject const interfacedProject = interfacedTaskUtil.getProject("pony"); @@ -507,3 +547,63 @@ test("getInterface: specVersion 3.0", (t) => { t.is(typeof resourceFactory.createFlatReader, "function", "resourceFactory function createFlatReader is available"); }); + +test("getInterface: specVersion 5.0 exposes getBuildTime", (t) => { + const buildTime = new Date(2026, 8, 25, 14, 7, 3); + const taskUtil = new TaskUtil({ + projectBuildContext: { + getProject: sinon.stub().returns({ + getName: () => "name", + }), + getDependencies: sinon.stub().returns([]), + getBuildTime: () => buildTime, + } + }); + + const interfacedTaskUtil = taskUtil.getInterface(getSpecificationVersion("5.0")); + + t.deepEqual(Object.keys(interfacedTaskUtil), [ + "STANDARD_TAGS", + "setTag", + "clearTag", + "getTag", + "isRootProject", + "registerCleanupTask", + "getProject", + "getDependencies", + "resourceFactory", + "getEnv", + "getTime", + "getBuildTime", + ], "getBuildTime is added at specVersion 5.0"); + + t.is(typeof interfacedTaskUtil.getEnv, "function", "function getEnv is provided"); + t.is(typeof interfacedTaskUtil.getTime, "function", "function getTime is provided"); + t.is(typeof interfacedTaskUtil.getBuildTime, "function", "function getBuildTime is provided"); + t.is(interfacedTaskUtil.getBuildTime(), buildTime, "getBuildTime returns the build run's Date"); +}); + +test("getReadOnlyInterface: specVersion 5.0 exposes getBuildTime", (t) => { + const buildTime = new Date(2026, 8, 25, 14, 7, 3); + const taskUtil = new TaskUtil({ + projectBuildContext: { + getProject: sinon.stub().returns({ + getName: () => "name", + }), + getDependencies: sinon.stub().returns([]), + getBuildTime: () => buildTime, + } + }); + + const interfacedTaskUtil = taskUtil.getReadOnlyInterface(getSpecificationVersion("5.0")); + + t.is(typeof interfacedTaskUtil.getEnv, "function", "function getEnv is provided"); + t.is(typeof interfacedTaskUtil.getTime, "function", "function getTime is provided"); + t.is(typeof interfacedTaskUtil.getBuildTime, "function", "function getBuildTime is provided"); + t.is(interfacedTaskUtil.getBuildTime(), buildTime, "getBuildTime returns the build run's Date"); + + const readOnlyBelow5 = taskUtil.getReadOnlyInterface(getSpecificationVersion("3.0")); + t.is(readOnlyBelow5.getEnv, undefined, "getEnv is not provided below specVersion 5.0"); + t.is(readOnlyBelow5.getTime, undefined, "getTime is not provided below specVersion 5.0"); + t.is(readOnlyBelow5.getBuildTime, undefined, "getBuildTime is not provided below specVersion 5.0"); +}); diff --git a/packages/project/test/lib/build/helpers/quantizeTime.js b/packages/project/test/lib/build/helpers/quantizeTime.js new file mode 100644 index 00000000000..e3184ffe15e --- /dev/null +++ b/packages/project/test/lib/build/helpers/quantizeTime.js @@ -0,0 +1,60 @@ +import test from "ava"; +import {quantizeTime, TIME_GRANULARITIES} from "../../../../lib/build/helpers/quantizeTime.js"; + +// A fixed local point in time: 25 September 2026, 14:07:03. Month, day and hour are all two digits +// here; a separate test covers zero-padding of single-digit values. +const fixedDate = new Date(2026, 8, 25, 14, 7, 3); + +test("quantizes each granularity to its bucket string", (t) => { + t.is(quantizeTime("year", fixedDate), "2026"); + t.is(quantizeTime("month", fixedDate), "2026-09"); + t.is(quantizeTime("day", fixedDate), "2026-09-25"); + t.is(quantizeTime("hour", fixedDate), "2026-09-25T14"); +}); + +test("zero-pads single-digit month, day and hour", (t) => { + // 3 February 2026, 05:00 local. + const earlyDate = new Date(2026, 1, 3, 5, 0, 0); + t.is(quantizeTime("month", earlyDate), "2026-02"); + t.is(quantizeTime("day", earlyDate), "2026-02-03"); + t.is(quantizeTime("hour", earlyDate), "2026-02-03T05"); +}); + +test("coarser buckets are a prefix of finer buckets", (t) => { + const year = quantizeTime("year", fixedDate); + const month = quantizeTime("month", fixedDate); + const day = quantizeTime("day", fixedDate); + const hour = quantizeTime("hour", fixedDate); + t.true(month.startsWith(year)); + t.true(day.startsWith(month)); + t.true(hour.startsWith(day)); +}); + +test("two dates in the same bucket quantize equally, a rolled-over bucket differs", (t) => { + const jan1 = new Date(2026, 0, 1, 0, 0, 0); + const dec31 = new Date(2026, 11, 31, 23, 59, 59); + const nextYear = new Date(2027, 0, 1, 0, 0, 0); + t.is(quantizeTime("year", jan1), quantizeTime("year", dec31), "same year -> same bucket"); + t.not(quantizeTime("year", dec31), quantizeTime("year", nextYear), "year boundary -> different bucket"); +}); + +test("throws when the date argument is missing", (t) => { + // The date is mandatory so callers pass the build run's shared timestamp rather than silently + // falling back to a fresh new Date(). + const err = t.throws(() => quantizeTime("year")); + t.is(err.message, `Missing or invalid 'date' argument: expected a Date instance`); +}); + +test("throws when the date argument is not a Date", (t) => { + const err = t.throws(() => quantizeTime("year", 2026)); + t.is(err.message, `Missing or invalid 'date' argument: expected a Date instance`); +}); + +test("throws for an unknown granularity", (t) => { + const err = t.throws(() => quantizeTime("minute", fixedDate)); + t.is(err.message, `Invalid time granularity "minute". Expected one of: year, month, day, hour`); +}); + +test("TIME_GRANULARITIES lists the supported buckets", (t) => { + t.deepEqual(TIME_GRANULARITIES, ["year", "month", "day", "hour"]); +}); diff --git a/packages/project/test/lib/resources/ProjectResources.js b/packages/project/test/lib/resources/ProjectResources.js index 0d72a1a19cc..439aea24816 100644 --- a/packages/project/test/lib/resources/ProjectResources.js +++ b/packages/project/test/lib/resources/ProjectResources.js @@ -40,6 +40,32 @@ test("setFrozenSourceReader: frozen reader is included in getReader chain", (t) t.truthy(reader, "Reader returned successfully"); }); +test("reopenStage: swaps a cached read-only stage back to a writable one", (t) => { + const {pr, writer} = createProjectResources(); + pr.initStages(["task/a", "task/b"]); + pr.useStage("task/b"); + + // A full cache hit installs a read-only cached stage: it has a cached writer, no live writer. + const cachedWriter = {byGlob: sinon.stub().resolves([]), name: "cached-reader"}; + pr.setStage("task/b", cachedWriter, new Map(), new Map()); + t.is(pr.getStage().getWriter(), undefined, "Cached stage has no live writer"); + t.is(pr.getStage().getCachedWriter(), cachedWriter, "Cached stage carries the cached reader"); + + // Reopening restores a fresh live writer so the stage can be re-run. + pr.reopenStage("task/b"); + t.is(pr.getStage().getId(), "task/b", "Still on the reopened stage"); + t.is(pr.getStage().getWriter(), writer, "Reopened stage has a live writer"); + t.is(pr.getStage().getCachedWriter(), undefined, "Reopened stage dropped the cached reader"); + t.notThrows(() => pr.getWorkspace(), "The reopened stage yields a writable workspace"); +}); + +test("reopenStage: throws for an unknown stage", (t) => { + const {pr} = createProjectResources(); + pr.initStages(["task/a"]); + t.throws(() => pr.reopenStage("task/missing"), + {message: /Stage 'task\/missing' does not exist in project test\.project/}); +}); + test("setFrozenSourceReader: invalidates cached readers", (t) => { const {pr} = createProjectResources(); @@ -190,3 +216,26 @@ test("Frozen source reader takes priority over filesystem source reader", async t.is(content, frozenCASContent, "Frozen CAS reader takes priority over filesystem source reader"); }); + +test("replayTagOperations routes by tag, applies by path, and skips get operations", (t) => { + const {pr} = createProjectResources(); + + // A restored step replays these: a project-level and a build-level set, a get (no + // persistent effect), and a set-then-clear of the same tag on another path. + pr.replayTagOperations([ + {op: "set", path: "/resources/x.js", tag: "ui5:HasDebugVariant", value: true}, + {op: "set", path: "/resources/x.js", tag: "ui5:OmitFromBuildResult", value: true}, + {op: "get", path: "/resources/x.js", tag: "ui5:IsBundle"}, + {op: "set", path: "/resources/x-dbg.js", tag: "ui5:IsDebugVariant", value: true}, + {op: "clear", path: "/resources/x-dbg.js", tag: "ui5:IsDebugVariant"}, + ]); + + const {projectTagOperations, buildTagOperations} = pr.getResourceTagOperations(); + + t.deepEqual([...projectTagOperations.get("/resources/x.js")], [["ui5:HasDebugVariant", true]], + "A project-level tag is routed to the project collection"); + t.deepEqual([...projectTagOperations.get("/resources/x-dbg.js")], [["ui5:IsDebugVariant", undefined]], + "A set followed by a clear of the same tag records the clear"); + t.deepEqual([...buildTagOperations.get("/resources/x.js")], [["ui5:OmitFromBuildResult", true]], + "A build-level tag is routed to the build collection; the skipped get left no operation"); +}); diff --git a/packages/project/test/lib/specifications/extensions/Task.js b/packages/project/test/lib/specifications/extensions/Task.js index 262a72c0cf9..059468e28d6 100644 --- a/packages/project/test/lib/specifications/extensions/Task.js +++ b/packages/project/test/lib/specifications/extensions/Task.js @@ -112,18 +112,6 @@ test("getDetermineBuildSignatureCallback (ESM)", async (t) => { t.is(callback, undefined, "Returns undefined when not exported"); }); -test("getSupportsDifferentialBuildsCallback (CJS)", async (t) => { - const extension = await Specification.create(clone(basicCjsTaskInput)); - const callback = await extension.getSupportsDifferentialBuildsCallback(); - t.is(callback, undefined, "Returns undefined when not exported"); -}); - -test("getSupportsDifferentialBuildsCallback (ESM)", async (t) => { - const extension = await Specification.create(clone(basicEsmTaskInput)); - const callback = await extension.getSupportsDifferentialBuildsCallback(); - t.is(callback, undefined, "Returns undefined when not exported"); -}); - test("getExpectedOutputCallback (CJS)", async (t) => { const extension = await Specification.create(clone(basicCjsTaskInput)); const callback = await extension.getExpectedOutputCallback(); From f4a9b7f6d636fa23746fc958710e101091d015b2 Mon Sep 17 00:00:00 2001 From: Matthias Osswald Date: Wed, 7 Oct 2026 10:54:27 +0200 Subject: [PATCH 3/8] feat(builder): Convert built-in tasks to step factories The incremental build cache keys its work per step, so a built-in task has to expose its unit of work as a step factory. Convert minify, buildThemes, replaceVersion, replaceCopyright, replaceBuildtime, enhanceManifest, and escapeNonAsciiCharacters to a default build(options) factory that returns step descriptors. generateThemeDesignerResources returns three steps, the others one map step each. replaceCopyright and replaceVersion read the copyright year and version through taskUtil.getTime and taskUtil.getProject, so a changed value invalidates only the affected units on a delta build. Add runSteps.js, a cache-free step runner for @ui5/builder standalone use (outside the @ui5/project build cache). Its BufferedWriter mirrors StepRunner through the shared @ui5/fs/internal/stepWriteBuffer contract. themeBuilderWorker imports themeBuilder.js lazily so TaskRunner can import the module for plan-time step-name discovery without evaluating the less-openui5 graph. Co-authored-by: Merlin Beutlberger --- .../lib/processors/themeBuilderWorker.js | 5 +- packages/builder/lib/tasks/buildThemes.js | 265 ++++++++---------- packages/builder/lib/tasks/enhanceManifest.js | 47 ++-- .../lib/tasks/escapeNonAsciiCharacters.js | 47 ++-- .../tasks/generateThemeDesignerResources.js | 175 ++++++------ packages/builder/lib/tasks/minify.js | 93 +++--- .../builder/lib/tasks/replaceBuildtime.js | 58 ++-- .../builder/lib/tasks/replaceCopyright.js | 62 ++-- packages/builder/lib/tasks/replaceVersion.js | 53 ++-- packages/builder/lib/tasks/runSteps.js | 116 ++++++++ .../test/lib/tasks/buildThemes.integration.js | 5 +- .../builder/test/lib/tasks/buildThemes.js | 133 ++++----- .../builder/test/lib/tasks/enhanceManifest.js | 41 ++- .../lib/tasks/escapeNonAsciiCharacters.js | 39 +-- .../tasks/generateThemeDesignerResources.js | 159 +++++------ .../test/lib/tasks/minify.integration.js | 19 +- packages/builder/test/lib/tasks/minify.js | 101 ++++--- .../test/lib/tasks/replaceBuildtime.js | 37 ++- .../test/lib/tasks/replaceCopyright.js | 56 +++- .../builder/test/lib/tasks/replaceVersion.js | 3 +- packages/builder/test/lib/tasks/runSteps.js | 176 ++++++++++++ 21 files changed, 1027 insertions(+), 663 deletions(-) create mode 100644 packages/builder/lib/tasks/runSteps.js create mode 100644 packages/builder/test/lib/tasks/runSteps.js diff --git a/packages/builder/lib/processors/themeBuilderWorker.js b/packages/builder/lib/processors/themeBuilderWorker.js index 91956e566c4..ac8aeb9a778 100644 --- a/packages/builder/lib/processors/themeBuilderWorker.js +++ b/packages/builder/lib/processors/themeBuilderWorker.js @@ -1,5 +1,4 @@ import workerpool from "workerpool"; -import themeBuilder from "./themeBuilder.js"; import {createResource} from "@ui5/fs/resourceFactory"; import {Buffer} from "node:buffer"; @@ -25,6 +24,10 @@ export default async function execThemeBuild({ const fsThemeResources = deserializeResources(themeResources); const fsReader = new FsWorkerThreadInterface(fsInterfacePort); + // Load the theme builder (and its less-openui5 module graph) lazily, so the main thread can import this + // module for its FsMainThreadInterface and (de)serialize helpers, and for the step-factory's plan-time + // step-name discovery, without evaluating that graph. Only a worker that builds a theme needs it. + const themeBuilder = (await import("./themeBuilder.js")).default; const result = await themeBuilder({ resources: fsThemeResources, fs: fsReader, diff --git a/packages/builder/lib/tasks/buildThemes.js b/packages/builder/lib/tasks/buildThemes.js index c11b3435a88..2305440cd0d 100644 --- a/packages/builder/lib/tasks/buildThemes.js +++ b/packages/builder/lib/tasks/buildThemes.js @@ -61,173 +61,148 @@ async function buildThemeInWorker(taskUtil, options, transferList) { return getPool(taskUtil).exec("execThemeBuild", [options], toTransfer); } - -/** - * @public - * @module @ui5/builder/tasks/buildThemes - */ /** - * Task to build a library theme. + * Builds the given theme resources, reading imports through combo. Uses the theme-builder + * worker pool when a taskUtil is available, otherwise builds inline. * - * @public - * @function default - * @static - * - * @param {object} parameters Parameters - * @param {@ui5/fs/DuplexCollection} parameters.workspace DuplexCollection to read and write files - * @param {@ui5/fs/AbstractReader} parameters.dependencies Reader or Collection to read dependency files - * @param {@ui5/builder/tasks/TaskUtil|object} [parameters.taskUtil] TaskUtil instance. - * Required to run buildThemes in parallel execution mode. - * @param {object} parameters.options Options - * @param {string} parameters.options.projectName Project name - * @param {string} parameters.options.inputPattern Search pattern for *.less files to be built - * @param {string} [parameters.options.librariesPattern] Search pattern for .library files - * @param {string} [parameters.options.themesPattern] Search pattern for sap.ui.core theme folders - * @param {boolean} [parameters.options.compress=true] - * @returns {Promise} Promise resolving with undefined once data has been written + * @param {@ui5/fs/Resource[]} themeResources library.source.less resources to build + * @param {@ui5/fs/AbstractReader} combo Prioritized workspace+dependencies reader for import resolution + * @param {boolean} compress Whether to compress the produced CSS + * @param {object} [taskUtil] TaskUtil, required for worker-pool execution + * @returns {Promise<@ui5/fs/Resource[]>} The produced theme resources */ -export default async function({ - workspace, dependencies, taskUtil, - options: { - projectName, inputPattern, librariesPattern, themesPattern, compress, - } -}) { - const combo = new ReaderCollectionPrioritized({ - name: `theme - prioritize workspace over dependencies: ${projectName}`, - readers: [workspace, dependencies] - }); - - compress = compress === undefined ? true : compress; - - const pThemeResources = workspace.byGlob(inputPattern); - let pAvailableLibraries; - let pAvailableThemes; - if (librariesPattern) { - // If a librariesPattern is given - // we will use it to reduce the set of libraries a theme will be built for - pAvailableLibraries = combo.byGlob(librariesPattern); - } - if (themesPattern) { - // If a themesPattern is given - // we will use it to reduce the set of themes that will be built - pAvailableThemes = combo.byGlob(themesPattern, {nodir: false}); - } - - /* Don't try to build themes for libraries that are not available - (maybe replace this with something more aware of which dependencies are optional and therefore - legitimately missing and which not (fault case)) - */ - let availableLibraries; - if (pAvailableLibraries) { - availableLibraries = []; - (await pAvailableLibraries).forEach((resource) => { - const library = path.dirname(resource.getPath()); - if (!availableLibraries.includes(library)) { - availableLibraries.push(library); - } - }); - } - let availableThemes; - if (pAvailableThemes) { - availableThemes = (await pAvailableThemes) - .filter((resource) => resource.getStatInfo().isDirectory()) - .map((resource) => { - return path.basename(resource.getPath()); - }); - } - - let themeResources = await pThemeResources; - - const isAvailable = function(resource) { - let libraryAvailable = false; - let themeAvailable = false; - const resourcePath = resource.getPath(); - const themeName = path.basename(path.dirname(resourcePath)); - - if (!availableLibraries || availableLibraries.length === 0) { - libraryAvailable = true; // If no libraries are found, build themes for all libraries - } else { - for (let i = availableLibraries.length - 1; i >= 0; i--) { - if (resourcePath.startsWith(availableLibraries[i])) { - libraryAvailable = true; - } - } - } - - if (!availableThemes || availableThemes.length === 0) { - themeAvailable = true; // If no themes are found, build all themes - } else { - themeAvailable = availableThemes.includes(themeName); - } - - if (log.isLevelEnabled("verbose")) { - if (!libraryAvailable) { - log.silly(`Skipping ${resourcePath}: Library is not available`); - } - if (!themeAvailable) { - log.verbose(`Skipping ${resourcePath}: sap.ui.core theme '${themeName}' is not available. ` + - "If you experience missing themes, check whether you have added the corresponding theme " + - "library to your projects dependencies and make sure that your custom themes contain " + - "resources for the sap.ui.core namespace."); - } - } - - // Only build if library and theme are available - return libraryAvailable && themeAvailable; - }; - - if (availableLibraries || availableThemes) { - if (log.isLevelEnabled("verbose")) { - log.verbose("Filtering themes to be built:"); - if (availableLibraries) { - log.verbose(`Available libraries: ${availableLibraries.join(", ")}`); - } - if (availableThemes) { - log.verbose(`Available sap.ui.core themes: ${availableThemes.join(", ")}`); - } - } - themeResources = themeResources.filter(isAvailable); - } - - let processedResources; +async function buildThemeResources(themeResources, combo, compress, taskUtil) { const useWorkers = !process.env.UI5_CLI_NO_WORKERS && !!taskUtil; if (useWorkers) { const threadMessageHandler = new FsMainThreadInterface(fsInterface(combo)); - - processedResources = await Promise.all(themeResources.map(async (themeRes) => { + const processedResources = await Promise.all(themeResources.map(async (themeRes) => { const {port1, port2} = new MessageChannel(); threadMessageHandler.startCommunication(port1); const result = await buildThemeInWorker(taskUtil, { fsInterfacePort: port2, themeResources: await serializeResources([themeRes]), - options: { - compress, - }, + options: {compress}, }, [port2]); threadMessageHandler.endCommunication(port1); - return result; })) .then((resources) => Array.prototype.concat.apply([], resources)) .then(deserializeResources); threadMessageHandler.cleanup(); - } else { - // Do not use workerpool - const themeBuilder = (await import("../processors/themeBuilder.js")).default; - - processedResources = await themeBuilder({ - resources: themeResources, - fs: fsInterface(combo), - options: { - compress, + return processedResources; + } + + const themeBuilder = (await import("../processors/themeBuilder.js")).default; + return themeBuilder({ + resources: themeResources, + fs: fsInterface(combo), + options: {compress}, + }); +} + +/** + * Determines whether a single theme should be built, probing the gating marker and sap.ui.core theme + * folder for exactly this theme through combo. Uses targeted byPath probes so + * the probed paths are recorded as inputs of the owning step: an absent marker created later, or a present + * marker removed, invalidates exactly this theme's step on a delta build. + * + * @param {@ui5/fs/Resource} themeResource The library.source.less resource of the theme + * @param {@ui5/fs/AbstractReader} combo Prioritized workspace+dependencies reader (recording) + * @param {object} patterns + * @param {string} [patterns.librariesPattern] Marks that a library.js/.library + * marker gates the theme (set when the theme library is built as a dependency) + * @param {string} [patterns.themesPattern] Search pattern for sap.ui.core theme folders + * @returns {Promise} Whether the theme should be built + */ +async function isThemeAvailable(themeResource, combo, {librariesPattern, themesPattern}) { + const resourcePath = themeResource.getPath(); + const themeName = path.basename(path.dirname(resourcePath)); + + let libraryAvailable = true; + if (librariesPattern) { + // The library root is the namespace directory owning the theme, i.e. the path up to `/themes/`. + // Probe both marker candidates by path so an absent marker is recorded too, enabling add/remove + // deltas to invalidate this theme. + const libraryRoot = resourcePath.substring(0, resourcePath.lastIndexOf("/themes/")); + const [dotLibrary, libraryJs] = await Promise.all([ + combo.byPath(`${libraryRoot}/.library`), + combo.byPath(`${libraryRoot}/library.js`), + ]); + libraryAvailable = !!(dotLibrary || libraryJs); + if (!libraryAvailable) { + log.silly(`Skipping ${resourcePath}: Library is not available`); + } + } + + let themeAvailable = true; + if (themesPattern) { + const availableThemes = (await combo.byGlob(themesPattern, {nodir: false})) + .filter((resource) => resource.getStatInfo().isDirectory()) + .map((resource) => path.basename(resource.getPath())); + // As in the batch check: if no sap.ui.core theme folders exist at all, build all themes; the + // themesPattern only narrows the set when such folders are present. + if (availableThemes.length > 0) { + themeAvailable = availableThemes.includes(themeName); + if (!themeAvailable) { + log.verbose(`Skipping ${resourcePath}: sap.ui.core theme '${themeName}' is not available. ` + + "If you experience missing themes, check whether you have added the corresponding theme " + + "library to your projects dependencies and make sure that your custom themes contain " + + "resources for the sap.ui.core namespace."); } - }); + } } - await Promise.all(processedResources.map((resource) => { - return workspace.write(resource); - })); + return libraryAvailable && themeAvailable; +} + +/** + * @public + * @module @ui5/builder/tasks/buildThemes + */ +/** + * Task to build a library theme. + * + * A step-based task: the default export is a factory returning one map step with a key per theme's + * library.source.less. A step probes only the gating marker and imports of its own theme, + * so a delta build rebuilds only the affected theme and leaves the others served from cache. Standalone + * invocation runs every theme through @ui5/builder's runSteps. + * + * @public + * @function default + * @static + * + * @param {object} options Options + * @param {string} options.projectName Project name + * @param {string} options.inputPattern Search pattern for *.less files to be built + * @param {string} [options.librariesPattern] Search pattern for .library files + * @param {string} [options.themesPattern] Search pattern for sap.ui.core theme folders + * @param {boolean} [options.compress=true] + * @returns {object[]} The task's build steps + */ +export default function build({projectName, inputPattern, librariesPattern, themesPattern, compress}) { + compress = compress === undefined ? true : compress; + + return [{ + name: "buildThemes", + // One key per theme's library.source.less, so a delta build rebuilds only the affected theme. + keys: async ({workspace}) => workspace.byGlob(inputPattern), + each: async (themeResource, {workspace, dependencies, taskUtil}) => { + // Prioritize workspace over dependencies. Reads through this combo are attributed to this step, + // so the marker probe and import resolution become tracked inputs of this specific theme. + const combo = new ReaderCollectionPrioritized({ + name: `theme - prioritize workspace over dependencies: ${projectName}`, + readers: dependencies ? [workspace, dependencies] : [workspace], + }); + if (!(await isThemeAvailable(themeResource, combo, {librariesPattern, themesPattern}))) { + // The gating marker/theme folder is missing: write nothing. The probes above are recorded, + // so a later marker creation re-runs this step and builds the theme. + return; + } + const processedResources = await buildThemeResources([themeResource], combo, compress, taskUtil); + await Promise.all(processedResources.map((resource) => workspace.write(resource))); + }, + }]; } diff --git a/packages/builder/lib/tasks/enhanceManifest.js b/packages/builder/lib/tasks/enhanceManifest.js index e643f924f71..66ff1e34ee4 100644 --- a/packages/builder/lib/tasks/enhanceManifest.js +++ b/packages/builder/lib/tasks/enhanceManifest.js @@ -1,4 +1,3 @@ -import manifestEnhancer from "../processors/manifestEnhancer.js"; import fsInterface from "@ui5/fs/fsInterface"; /* eslint "jsdoc/check-param-names": ["error", {"disableExtraPropertyReporting":true}] */ @@ -11,26 +10,38 @@ import fsInterface from "@ui5/fs/fsInterface"; * Adds missing information based on the available project resources, * for example the locales supported by the present i18n resources. * + * A step-based task: the default export is a factory returning one map step with a key per matched + * manifest.json. The processor reads the i18n bundle files next to each manifest through the + * step's workspace, so those reads are recorded as inputs of that step: editing a manifest or adding, + * removing, or changing one of its i18n files re-runs only the owning manifest's step on a delta build and + * leaves the others served from cache. + * * @public * @function default * @static * - * @param {object} parameters Parameters - * @param {@ui5/fs/DuplexCollection} parameters.workspace DuplexCollection to read and write files - * @param {object} parameters.options Options - * @param {string} parameters.options.projectNamespace Namespace of the application - * @returns {Promise} Promise resolving with undefined once data has been written + * @param {object} options Options + * @param {string} options.projectNamespace Namespace of the application + * @returns {object[]} The task's build steps */ -export default async function({workspace, options}) { - const {projectNamespace} = options; - - // Note: all "manifest.json" files in the given namespace - const resources = await workspace.byGlob(`/resources/${projectNamespace}/**/manifest.json`); - - const processedResources = await manifestEnhancer({ - resources, - fs: fsInterface(workspace), - }); - - await Promise.all(processedResources.map((resource) => workspace.write(resource))); +export default function build({projectNamespace}) { + return [{ + name: "enhanceManifest", + // One key per manifest.json. The i18n bundle reads made by manifestEnhancer go through the step's + // workspace and are thus tracked per step. + keys: async ({workspace}) => workspace.byGlob(`/resources/${projectNamespace}/**/manifest.json`), + each: async (resource, {workspace}) => { + // Load the manifest enhancer lazily, inside the step body, so plan-time step-name discovery can + // import this factory module without evaluating that processor's module graph. A build whose + // manifest keys are all cache hits never reaches here. + const manifestEnhancer = (await import("../processors/manifestEnhancer.js")).default; + const [processed] = await manifestEnhancer({ + resources: [resource], + fs: fsInterface(workspace), + }); + if (processed) { + await workspace.write(processed); + } + }, + }]; } diff --git a/packages/builder/lib/tasks/escapeNonAsciiCharacters.js b/packages/builder/lib/tasks/escapeNonAsciiCharacters.js index 697b2425080..bfe724bd575 100644 --- a/packages/builder/lib/tasks/escapeNonAsciiCharacters.js +++ b/packages/builder/lib/tasks/escapeNonAsciiCharacters.js @@ -8,37 +8,38 @@ import nonAsciiEscaper from "../processors/nonAsciiEscaper.js"; /** * Task to escape non ascii characters in properties files resources. * + * A step-based task: the default export is a factory returning one map step with a key per matched + * resource, so a delta build re-processes only the resources whose content changed. Escaping is a step's + * only input, and a resource key is content-addressed, so a changed resource is a new key that re-runs and + * any removed resource drops its stale output. + * * @public * @function default * @static * - * @param {object} parameters Parameters - * @param {@ui5/fs/DuplexCollection} parameters.workspace DuplexCollection to read and write files - * @param {string[]} [parameters.changedProjectResourcePaths] Set of changed resource paths within the project. - * This is only set if a cache is used and changes have been detected. - * @param {object} parameters.options Options - * @param {string} parameters.options.pattern Glob pattern to locate the files to be processed - * @param {string} parameters.options.encoding source file encoding either "UTF-8" or "ISO-8859-1" - * @returns {Promise} Promise resolving with undefined once data has been written + * @param {object} options Options + * @param {string} options.pattern Glob pattern to locate the files to be processed + * @param {string} options.encoding source file encoding either "UTF-8" or "ISO-8859-1" + * @returns {object[]} The task's build steps */ -export default async function({workspace, changedProjectResourcePaths, options: {pattern, encoding}}) { +export default function build({pattern, encoding}) { if (!encoding) { throw new Error("[escapeNonAsciiCharacters] Mandatory option 'encoding' not provided"); } - let allResources; - if (changedProjectResourcePaths) { - allResources = await Promise.all(changedProjectResourcePaths.map((resource) => workspace.byPath(resource))); - } else { - allResources = await workspace.byGlob(pattern); - } - - const processedResources = await nonAsciiEscaper({ - resources: allResources, - options: { - encoding: nonAsciiEscaper.getEncodingFromAlias(encoding) - } - }); + const escaperOptions = { + encoding: nonAsciiEscaper.getEncodingFromAlias(encoding) + }; - await Promise.all(processedResources.map((resource) => resource && workspace.write(resource))); + return [{ + name: "escapeNonAsciiCharacters", + // One key per matched resource, so a delta build re-processes only the resources that changed. + keys: async ({workspace}) => workspace.byGlob(pattern), + each: async (resource, {workspace}) => { + const [processed] = await nonAsciiEscaper({resources: [resource], options: escaperOptions}); + if (processed) { + await workspace.write(processed); + } + }, + }]; } diff --git a/packages/builder/lib/tasks/generateThemeDesignerResources.js b/packages/builder/lib/tasks/generateThemeDesignerResources.js index d2f9b60bdf7..78443b3950b 100644 --- a/packages/builder/lib/tasks/generateThemeDesignerResources.js +++ b/packages/builder/lib/tasks/generateThemeDesignerResources.js @@ -1,7 +1,6 @@ import posixPath from "node:path/posix"; import {getLogger} from "@ui5/logger"; const log = getLogger("builder:tasks:generateThemeDesignerResources"); -import libraryLessGenerator from "../processors/libraryLessGenerator.js"; import {updateLibraryDotTheming} from "./utils/dotTheming.js"; import ReaderCollectionPrioritized from "@ui5/fs/ReaderCollectionPrioritized"; import Resource from "@ui5/fs/Resource"; @@ -92,32 +91,37 @@ async function generateThemeDotTheming({workspace, combo, themeFolder}) { * @module @ui5/builder/tasks/generateThemeDesignerResources */ -/* eslint "jsdoc/check-param-names": ["error", {"disableExtraPropertyReporting":true}] */ /** * Generates resources required for integration with the SAP Theme Designer. * + * A step-based task: the default export is a factory returning a scalar "scan" step (does the library + * have themes at all), an optional scalar "libraryTheming" step (the library-level .theming, + * for a project of type library), and a "themes" map step with a key per theme's + * library.source.less. Each theme step generates that theme's .theming and + * library.less, reading the core .theming and the less imports through its own + * combo so those reads are recorded per step. A delta build regenerates only the affected theme. The + * later steps consume the scan result through needs, so they stay cached while a library has + * themes even as individual themes change. Standalone invocation runs every step through runSteps. + * * @public * @function default * @static * - * @param {object} parameters Parameters - * @param {@ui5/fs/DuplexCollection} parameters.workspace DuplexCollection to read and write files - * @param {@ui5/fs/AbstractReader} parameters.dependencies Reader or Collection to read dependency files - * @param {object} parameters.options Options - * @param {string} parameters.options.projectName Project name - * @param {string} parameters.options.version Project version - * @param {string} [parameters.options.projectNamespace] If the project is of type library, - * provide its namespace. - * Omit for type theme-library - * @returns {Promise} Promise resolving with undefined once data has been written + * @param {object} options Options + * @param {string} options.projectName Project name + * @param {string} options.version Project version + * @param {string} [options.projectNamespace] If the project is of type library, provide its + * namespace. Omit for type theme-library + * @returns {object[]} The task's build steps */ -export default async function({workspace, dependencies, options}) { +export default function build(options) { const {projectName, version} = options; const namespace = options.projectNamespace; - // Skip sap.ui.documentation since it is not intended to be available in SAP Theme Designer to create custom themes + // Skip sap.ui.documentation since it is not intended to be available in SAP Theme Designer to create + // custom themes if (namespace === "sap/ui/documentation") { - return; + return []; } let librarySourceLessPattern; @@ -129,78 +133,89 @@ export default async function({workspace, dependencies, options}) { librarySourceLessPattern = `/resources/**/themes/*/library.source.less`; } - const librarySourceLessResources = await workspace.byGlob(librarySourceLessPattern); - - const hasThemes = librarySourceLessResources.length > 0; - - // library .theming file - // Only for type "library". Type "theme-library" does not provide a namespace - // Also needs to be created in case a library does not have any themes (see bIgnore flag) + const steps = [{ + // Whether the library has any themes at all. Consumed by the later steps through needs, so they + // stay cached while this holds even as individual themes change. + name: "scan", + run: async ({workspace}) => ({ + hasThemes: (await workspace.byGlob(librarySourceLessPattern)).length > 0 + }), + }]; + + // library .theming file. Only for type "library" (type "theme-library" provides no namespace). Also + // needs to be created when a library has no themes (the bIgnore flag). if (namespace) { - let libraryDotThemingResource; - - // Do not generate a .theming file for the sap.ui.core library - if (namespace === "sap/ui/core") { - // Check if the .theming file already exists - libraryDotThemingResource = await workspace.byPath(`/resources/${namespace}/.theming`); - if (libraryDotThemingResource) { - // Update the existing .theming resource - log.verbose(`Updating .theming for namespace ${namespace}`); - await updateLibraryDotTheming({ - resource: libraryDotThemingResource, - namespace, - version, - hasThemes - }); - } - } - - if (!libraryDotThemingResource) { - log.verbose(`Generating .theming for namespace ${namespace}`); - libraryDotThemingResource = generateLibraryDotTheming({ - namespace, - version, - hasThemes - }); - } - - await workspace.write(libraryDotThemingResource); - } - - if (!hasThemes) { - // Skip further processing as there are no themes - return; + steps.push({ + name: "libraryTheming", + needs: ["scan"], + run: async ({needs, workspace}) => { + const {hasThemes} = needs.scan; + let libraryDotThemingResource; + + // Do not generate a .theming file for the sap.ui.core library + if (namespace === "sap/ui/core") { + // Update the existing .theming file if present + libraryDotThemingResource = await workspace.byPath(`/resources/${namespace}/.theming`); + if (libraryDotThemingResource) { + log.verbose(`Updating .theming for namespace ${namespace}`); + await updateLibraryDotTheming({ + resource: libraryDotThemingResource, + namespace, + version, + hasThemes + }); + } + } + + if (!libraryDotThemingResource) { + log.verbose(`Generating .theming for namespace ${namespace}`); + libraryDotThemingResource = generateLibraryDotTheming({ + namespace, + version, + hasThemes + }); + } + + await workspace.write(libraryDotThemingResource); + }, + }); } - const combo = new ReaderCollectionPrioritized({ - name: `generateThemeDesignerResources - prioritize workspace over dependencies: ${projectName}`, - readers: [workspace, dependencies] - }); + steps.push({ + // One key per theme, so a delta build regenerates only the affected theme. keys() returns nothing + // when the library has no themes. + name: "themes", + needs: ["scan"], + keys: async ({needs, workspace}) => + needs.scan.hasThemes ? workspace.byGlob(librarySourceLessPattern) : [], + each: async (librarySourceLess, {workspace, dependencies}) => { + // Build the combo from the step readers so the core .theming and less-import reads are + // recorded as inputs of this theme. + const combo = new ReaderCollectionPrioritized({ + name: `generateThemeDesignerResources - prioritize workspace over dependencies: ${projectName}`, + readers: dependencies ? [workspace, dependencies] : [workspace] + }); - // theme .theming files - const themeDotThemingFiles = await Promise.all( - librarySourceLessResources.map((librarySourceLess) => { const themeFolder = posixPath.dirname(librarySourceLess.getPath()); log.verbose(`Generating .theming for theme ${themeFolder}`); - return generateThemeDotTheming({ - workspace, combo, themeFolder - }); - }) - ); - await Promise.all( - themeDotThemingFiles.map(async (resource) => { - if (resource) { - await workspace.write(resource); + + // theme .theming file + const themeDotThemingResource = await generateThemeDotTheming({workspace, combo, themeFolder}); + if (themeDotThemingResource) { + await workspace.write(themeDotThemingResource); } - }) - ); - // library.less files - const libraryLessResources = await libraryLessGenerator({ - resources: librarySourceLessResources, - fs: fsInterface(combo), + // library.less file. Load the less generator lazily, inside the step body, so plan-time step-name + // discovery can import this factory module without evaluating that processor's module graph. A + // build whose themes step is a cache hit never reaches here. + const libraryLessGenerator = (await import("../processors/libraryLessGenerator.js")).default; + const [libraryLessResource] = await libraryLessGenerator({ + resources: [librarySourceLess], + fs: fsInterface(combo), + }); + await workspace.write(libraryLessResource); + }, }); - await Promise.all( - libraryLessResources.map((resource) => workspace.write(resource)) - ); + + return steps; } diff --git a/packages/builder/lib/tasks/minify.js b/packages/builder/lib/tasks/minify.js index dc51c3c8eb0..4402569d0e5 100644 --- a/packages/builder/lib/tasks/minify.js +++ b/packages/builder/lib/tasks/minify.js @@ -1,4 +1,3 @@ -import minifier from "../processors/minifier.js"; import fsInterface from "@ui5/fs/fsInterface"; /** @@ -9,61 +8,34 @@ import fsInterface from "@ui5/fs/fsInterface"; /** * Task to minify resources. * + * A step-based task: the default export is a factory returning one map step with a key per matched + * resource, so a delta build re-minifies only the resources whose inputs changed. A resource's input + * source map (the //# sourceMappingURL= target) is read through the step's workspace and + * thus recorded as an input of that step, so changing only the .js.map re-runs its owning + * .js and regenerates a correct -dbg.js.map. + * * @public * @function default * @static * - * @param {object} parameters Parameters - * @param {@ui5/fs/DuplexCollection} parameters.workspace DuplexCollection to read and write files - * @param {@ui5/project/build/helpers/TaskUtil|object} [parameters.taskUtil] TaskUtil - * @param {string[]} [parameters.changedProjectResourcePaths] Set of changed resource paths within the project. - * This is only set if a cache is used and changes have been detected. - * @param {object} parameters.options Options - * @param {string} parameters.options.pattern Pattern to locate the files to be processed - * @param {boolean} [parameters.options.omitSourceMapResources=false] Whether source map resources shall + * @param {object} options Options + * @param {string} options.pattern Pattern to locate the files to be processed + * @param {boolean} [options.omitSourceMapResources=false] Whether source map resources shall * be tagged as "OmitFromBuildResult" and no sourceMappingURL shall be added to the minified resource - * @param {boolean} [parameters.options.useInputSourceMaps=true] Whether to make use of any existing source + * @param {boolean} [options.useInputSourceMaps=true] Whether to make use of any existing source * maps referenced in the resources to be minified. Use this option to preserve reference to the original * source files, such as TypeScript files, in the generated source map. - * @returns {Promise} Promise resolving with undefined once data has been written + * @returns {object[]} The task's build steps */ -export default async function({ - workspace, taskUtil, changedProjectResourcePaths, - options: {pattern, omitSourceMapResources = false, useInputSourceMaps = true} -}) { - let resources; - if (changedProjectResourcePaths) { - resources = await Promise.all( - changedProjectResourcePaths - // Filtering out non-JS resources such as .map files - // FIXME: A changed input source map (.js.map) does not re-minify its owning .js here, - // so the produced -dbg.js.map goes stale. Matching changed paths against "pattern" - // would not fix this: the task would need to learn the .map -> .js relation while - // processing changedProjectResourcePaths. That is likely a larger rework rather than a - // local fix (see the failing minify source-map staleness tests in @ui5/project). - .filter((resourcePath) => resourcePath.endsWith(".js")) - .map((resource) => workspace.byPath(resource)) - ); - } else { - resources = await workspace.byGlob(pattern); - } - if (resources.length === 0) { - return; - } - const processedResources = await minifier({ - resources, - fs: fsInterface(workspace), - taskUtil, - options: { - addSourceMappingUrl: !omitSourceMapResources, - readSourceMappingUrl: !!useInputSourceMaps, - useWorkers: !process.env.UI5_CLI_NO_WORKERS && !!taskUtil, - } - }); +export default function build({pattern, omitSourceMapResources = false, useInputSourceMaps = true}) { + const minifierOptions = { + addSourceMappingUrl: !omitSourceMapResources, + readSourceMappingUrl: !!useInputSourceMaps, + }; - return Promise.all(processedResources.map(async ({ - resource, dbgResource, sourceMapResource, dbgSourceMapResource - }) => { + // Applies the debug/omit tags to a minified resource and its derived resources, then writes them. + const tagAndWrite = async (processed, workspace, taskUtil) => { + const {resource, dbgResource, sourceMapResource, dbgSourceMapResource} = processed; if (taskUtil) { // Carry over OmitFromBuildResult from input resource to all derived resources if (taskUtil.getTag(resource, taskUtil.STANDARD_TAGS.OmitFromBuildResult)) { @@ -83,11 +55,34 @@ export default async function({ } } } - return Promise.all([ + await Promise.all([ workspace.write(resource), workspace.write(dbgResource), workspace.write(sourceMapResource), dbgSourceMapResource && workspace.write(dbgSourceMapResource) ]); - })); + }; + + return [{ + name: "minify", + // One key per matched resource, so a delta build re-minifies only the resources whose inputs + // changed. The input source map is read through the step's workspace and thus tracked per step. + keys: async ({workspace}) => workspace.byGlob(pattern), + each: async (inputResource, {workspace, taskUtil}) => { + // Load the minifier (and its worker-pool module graph) lazily, inside the step body, so plan-time + // step-name discovery can import this factory module without evaluating that graph. A build whose + // minify keys are all cache hits never reaches here. + const minifier = (await import("../processors/minifier.js")).default; + const [processed] = await minifier({ + resources: [inputResource], + fs: fsInterface(workspace), + taskUtil, + options: { + ...minifierOptions, + useWorkers: !process.env.UI5_CLI_NO_WORKERS && !!taskUtil + }, + }); + await tagAndWrite(processed, workspace, taskUtil); + }, + }]; } diff --git a/packages/builder/lib/tasks/replaceBuildtime.js b/packages/builder/lib/tasks/replaceBuildtime.js index 44498a09186..e8cad9174eb 100644 --- a/packages/builder/lib/tasks/replaceBuildtime.js +++ b/packages/builder/lib/tasks/replaceBuildtime.js @@ -3,8 +3,7 @@ import stringReplacer from "../processors/stringReplacer.js"; function pad(v) { return String(v).padStart(2, "0"); } -function getTimestamp() { - const date = new Date(); +function formatTimestamp(date) { const year = date.getFullYear(); const month = pad(date.getMonth() + 1); const day = pad(date.getDate()); @@ -22,36 +21,37 @@ function getTimestamp() { /** * Task to replace the buildtime ${buildtime}. * + * A step-based task: the default export is a factory returning one map step with a key per matched + * resource, so a delta build re-processes only the resources whose content changed. The buildtime comes + * from the build run's shared timestamp via + * [taskUtil.getBuildTime]{@link @ui5/project/build/helpers/TaskUtil#getBuildTime}, which is not a tracked + * cache input: a cached step keeps its previous timestamp until its resource content changes. Without a + * TaskUtil (e.g. a direct invocation) the step falls back to the wall clock. + * * @public * @function default * @static * - * @param {object} parameters Parameters - * @param {@ui5/fs/DuplexCollection} parameters.workspace DuplexCollection to read and write files - * @param {string[]} [parameters.changedProjectResourcePaths] Set of changed resource paths within the project. - * This is only set if a cache is used and changes have been detected. - * @param {object} parameters.options Options - * @param {string} parameters.options.pattern Pattern to locate the files to be processed - * @returns {Promise} Promise resolving with undefined once data has been written + * @param {object} options Options + * @param {string} options.pattern Pattern to locate the files to be processed + * @returns {object[]} The task's build steps */ -export default async function({workspace, changedProjectResourcePaths, options: {pattern}}) { - let resources; - if (changedProjectResourcePaths) { - resources = await Promise.all(changedProjectResourcePaths.map((resource) => workspace.byPath(resource))); - } else { - resources = await workspace.byGlob(pattern); - } - const timestamp = getTimestamp(); - const processedResources = await stringReplacer({ - resources, - options: { - pattern: "${buildtime}", - replacement: timestamp - } - }); - return Promise.all(processedResources.map((resource) => { - if (resource) { - return workspace.write(resource); - } - })); +export default function build({pattern}) { + return [{ + name: "replaceBuildtime", + // One key per matched resource, so a delta build re-processes only the resources that changed. + keys: async ({workspace}) => workspace.byGlob(pattern), + each: async (resource, {workspace, taskUtil}) => { + // Source the timestamp from the build run's shared clock so every project and task in the run + // agrees. Fall back to a direct Date read when the task runs without a TaskUtil. + const timestamp = formatTimestamp(taskUtil?.getBuildTime ? taskUtil.getBuildTime() : new Date()); + const [processed] = await stringReplacer({ + resources: [resource], + options: {pattern: "${buildtime}", replacement: timestamp} + }); + if (processed) { + await workspace.write(processed); + } + }, + }]; } diff --git a/packages/builder/lib/tasks/replaceCopyright.js b/packages/builder/lib/tasks/replaceCopyright.js index 90daed02fd5..a88d6ea313d 100644 --- a/packages/builder/lib/tasks/replaceCopyright.js +++ b/packages/builder/lib/tasks/replaceCopyright.js @@ -18,44 +18,46 @@ import stringReplacer from "../processors/stringReplacer.js"; * it will be replaced with the current year. * If no copyright string is given, no replacement is being done. * + * A step-based task: the default export is a factory returning one map step with a key per matched + * resource, so a delta build re-processes only the resources whose content changed. Each step reads the + * current year through [taskUtil.getTime]{@link @ui5/project/build/helpers/TaskUtil#getTime} so the + * incremental build cache tracks it: a cached result re-runs when the calendar year rolls over. Without a + * TaskUtil (e.g. a direct invocation) the step falls back to the wall clock. + * * @public * @function default * @static * - * @param {object} parameters Parameters - * @param {@ui5/fs/DuplexCollection} parameters.workspace DuplexCollection to read and write files - * @param {string[]} [parameters.changedProjectResourcePaths] Set of changed resource paths within the project. - * This is only set if a cache is used and changes have been detected. - * @param {object} parameters.options Options - * @param {string} parameters.options.copyright Replacement copyright - * @param {string} parameters.options.pattern Pattern to locate the files to be processed - * @returns {Promise} Promise resolving with undefined once data has been written + * @param {object} options Options + * @param {string} options.copyright Replacement copyright + * @param {string} options.pattern Pattern to locate the files to be processed + * @returns {object[]} The task's build steps */ -export default async function({workspace, changedProjectResourcePaths, options: {copyright, pattern}}) { +export default function build({copyright, pattern}) { if (!copyright) { - return; + return []; } - // Replace optional placeholder ${currentYear} with the current year - copyright = copyright.replace(/(?:\$\{currentYear\})/, new Date().getFullYear()); + const replacePattern = /(?:\$\{copyright\}|@copyright@)/g; - let resources; - if (changedProjectResourcePaths) { - resources = await Promise.all(changedProjectResourcePaths.map((resource) => workspace.byPath(resource))); - } else { - resources = await workspace.byGlob(pattern); - } + return [{ + name: "replaceCopyright", + // One key per matched resource, so a delta build re-processes only the resources that changed. + keys: async ({workspace}) => workspace.byGlob(pattern), + each: async (resource, {workspace, taskUtil}) => { + // Read the current year through taskUtil.getTime so the build cache tracks it as a step input: + // a cached step then re-runs when the calendar year rolls over. Fall back to a direct Date read + // when the task runs without a TaskUtil. + const currentYear = taskUtil?.getTime ? taskUtil.getTime("year") : new Date().getFullYear(); + const replacement = copyright.replace(/(?:\$\{currentYear\})/, currentYear); - const processedResources = await stringReplacer({ - resources, - options: { - pattern: /(?:\$\{copyright\}|@copyright@)/g, - replacement: copyright - } - }); - return Promise.all(processedResources.map((resource) => { - if (resource) { - return workspace.write(resource); - } - })); + const [processed] = await stringReplacer({ + resources: [resource], + options: {pattern: replacePattern, replacement} + }); + if (processed) { + await workspace.write(processed); + } + }, + }]; } diff --git a/packages/builder/lib/tasks/replaceVersion.js b/packages/builder/lib/tasks/replaceVersion.js index d30b0839dc6..47655c86564 100644 --- a/packages/builder/lib/tasks/replaceVersion.js +++ b/packages/builder/lib/tasks/replaceVersion.js @@ -8,36 +8,35 @@ import stringReplacer from "../processors/stringReplacer.js"; /** * Task to replace the version ${version}. * + * A step-based task: the default export is a factory returning one map step with a key per matched + * resource, so a delta build re-processes only the resources whose content changed. The replacement is + * a step's only input, and a resource key is content-addressed, so a changed resource is a new key that + * re-runs and any removed resource drops its stale output. + * * @public * @function default * @static * - * @param {object} parameters Parameters - * @param {@ui5/fs/DuplexCollection} parameters.workspace DuplexCollection to read and write files - * @param {string[]} [parameters.changedProjectResourcePaths] Set of changed resource paths within the project. - * This is only set if a cache is used and changes have been detected. - * @param {object} parameters.options Options - * @param {string} parameters.options.pattern Pattern to locate the files to be processed - * @param {string} parameters.options.version Replacement version - * @returns {Promise} Promise resolving with undefined once data has been written + * @param {object} options Options + * @param {string} options.pattern Pattern to locate the files to be processed + * @param {string} options.version Replacement version + * @returns {object[]} The task's build steps */ -export default async function({workspace, changedProjectResourcePaths, options: {pattern, version}}) { - let resources; - if (changedProjectResourcePaths) { - resources = await Promise.all(changedProjectResourcePaths.map((resource) => workspace.byPath(resource))); - } else { - resources = await workspace.byGlob(pattern); - } - const processedResources = await stringReplacer({ - resources, - options: { - pattern: /\$\{(?:project\.)?version\}/g, - replacement: version - } - }); - await Promise.all(processedResources.map((resource) => { - if (resource) { - return workspace.write(resource); - } - })); +export default function build({pattern, version}) { + const replacerOptions = { + pattern: /\$\{(?:project\.)?version\}/g, + replacement: version + }; + + return [{ + name: "replaceVersion", + // One key per matched resource, so a delta build re-processes only the resources that changed. + keys: async ({workspace}) => workspace.byGlob(pattern), + each: async (resource, {workspace}) => { + const [processed] = await stringReplacer({resources: [resource], options: replacerOptions}); + if (processed) { + await workspace.write(processed); + } + }, + }]; } diff --git a/packages/builder/lib/tasks/runSteps.js b/packages/builder/lib/tasks/runSteps.js new file mode 100644 index 00000000000..8f08c537b35 --- /dev/null +++ b/packages/builder/lib/tasks/runSteps.js @@ -0,0 +1,116 @@ +import AbstractReaderWriter from "@ui5/fs/AbstractReaderWriter"; +import {assertDistinctWrite, flushWriteBuffer} from "@ui5/fs/internal/stepWriteBuffer"; + +/** + * Buffers the writes of one concurrent map-step key and delegates reads to the underlying workspace, so + * a map step's writes can be flushed in key order after all keys finish. Mirrors the cached step runner's + * write buffering (minus the cache recording), including the same-path guard that keeps concurrent keys + * independent. The same-path guard, its error message and the key-order flush are shared with the cached + * runner through @ui5/fs/internal/stepWriteBuffer so the two cannot drift. + */ +class BufferedWriter extends AbstractReaderWriter { + #workspace; + #buffer; + #index; + + /** + * @param {@ui5/fs/AbstractReaderWriter} workspace Underlying workspace + * @param {Map} buffer Shared write buffer, keyed by resource path + * @param {number} index Key position, used to flush in key order and detect same-path writes + */ + constructor(workspace, buffer, index) { + super(typeof workspace.getName === "function" ? workspace.getName() : "workspace"); + this.#workspace = workspace; + this.#buffer = buffer; + this.#index = index; + } + + _byGlob(virPattern, options) { + return this.#workspace.byGlob(virPattern, options); + } + + _byPath(virPath, options) { + return this.#workspace.byPath(virPath, options); + } + + // Overrides the public write (rather than _write, unlike @ui5/project's RecordingReaderWriter) so the + // caller's exact arguments are preserved: the base class would default options to an object, which the + // key-order flush would then re-pass. The buffered entry stores those raw arguments as args, which + // flushWriteBuffer replays via write(resource, ...args). + async write(resource, ...args) { + // Real resources are keyed and deduplicated by their virtual path; a value without getPath + // (a test fake) is keyed by identity so it still buffers and flushes in insertion order. + const key = typeof resource.getPath === "function" ? resource.getPath() : resource; + assertDistinctWrite(this.#buffer, key, this.#index); + this.#buffer.set(key, {resource, args, index: this.#index}); + } +} + +/** + * Runs a step-based task's steps without a build cache. + * + * A step-based task default-exports a factory build(options) => Step[]. This runner is the + * no-cache counterpart to the cached step runner in @ui5/project (which + * @ui5/builder cannot import, the dependency direction being + * @ui5/cli -> @ui5/project -> @ui5/builder): it runs every step in order, fans out every + * map step's keys set, threads needs returns in memory, and buffers a + * concurrent map step's writes so they flush in key order. It replaces the per-task batch fallback tasks + * used to hand-write for standalone (direct or programmatic) invocation of @ui5/builder. + * + * Delta selection, CAS-backed returns, tag replay and signature computation are cache concerns and are + * absent here; every step runs. + * + * @public + * @module @ui5/builder/tasks/runSteps + * @param {Function} build Task factory build(options) => Step[] + * @param {object} parameters + * @param {@ui5/fs/DuplexCollection} parameters.workspace Workspace to read and write files + * @param {@ui5/fs/AbstractReader} [parameters.dependencies] Reader to read dependency files + * @param {@ui5/builder/tasks/TaskUtil|object} [parameters.taskUtil] TaskUtil, passed through to each step + * @param {object} [parameters.options] Task options, passed to the factory and each step + * @returns {Promise} Resolves once all steps have run and their writes are flushed + */ +export default async function runSteps(build, {workspace, dependencies, taskUtil, options} = {}) { + const steps = await build(options); + // Each step's return, so a later step's needs can consume it. A scalar step's return is its single + // value; a map step's is the array of its per-key returns in key order. + const returns = new Map(); + + for (const step of steps) { + const needs = {}; + if (step.needs) { + for (const name of step.needs) { + needs[name] = returns.get(name); + } + } + // Shared by the step's keys enumerator and all of its units, so frozen like in the cached step + // runner: a unit assigning to needs. would otherwise leak into its siblings. The freeze + // is shallow, since a producer may return resources whose own state must stay writable. + Object.freeze(needs); + const context = {needs, workspace, dependencies, taskUtil, options}; + + if (typeof step.run === "function") { + // Scalar step: run once, writes go straight to the workspace so a later step sees them. + returns.set(step.name, await step.run(context)); + continue; + } + + // Map step: enumerate keys, then run each key. + const keys = [...((await step.keys(context)) ?? [])]; + if (step.sequential) { + // Writes persist immediately, so a later key reads what an earlier key wrote. + const results = []; + for (const key of keys) { + results.push(await step.each(key, context)); + } + returns.set(step.name, results); + } else { + // Concurrent keys: buffer writes and flush them in key order once all keys finish. + const buffer = new Map(); + const results = await Promise.all(keys.map((key, index) => + step.each(key, {...context, workspace: new BufferedWriter(workspace, buffer, index)}))); + await flushWriteBuffer(buffer, workspace); + returns.set(step.name, results); + } + } +} diff --git a/packages/builder/test/lib/tasks/buildThemes.integration.js b/packages/builder/test/lib/tasks/buildThemes.integration.js index 23434f5e624..765c91d0f7f 100644 --- a/packages/builder/test/lib/tasks/buildThemes.integration.js +++ b/packages/builder/test/lib/tasks/buildThemes.integration.js @@ -1,5 +1,6 @@ import test from "ava"; import buildThemes from "../../../lib/tasks/buildThemes.js"; +import runSteps from "../../../lib/tasks/runSteps.js"; import {createAdapter, createResource} from "@ui5/fs/resourceFactory"; import DuplexCollection from "@ui5/fs/DuplexCollection"; @@ -45,7 +46,7 @@ test("integration: simple", async (t) => { string: content }); await reader.write(resource); - await buildThemes({ + await runSteps(buildThemes, { workspace: duplexCollection, dependencies: dependencies, options: { @@ -126,7 +127,7 @@ test("integration: imports", async (t) => { return reader.write(resource); })); - await buildThemes({ + await runSteps(buildThemes, { workspace: duplexCollection, dependencies: dependencies, options: { diff --git a/packages/builder/test/lib/tasks/buildThemes.js b/packages/builder/test/lib/tasks/buildThemes.js index d737ac67dad..f85cb49d4d6 100644 --- a/packages/builder/test/lib/tasks/buildThemes.js +++ b/packages/builder/test/lib/tasks/buildThemes.js @@ -2,6 +2,7 @@ import test from "ava"; import sinon from "sinon"; import esmock from "esmock"; import {deserializeResources} from "../../../lib/processors/themeBuilderWorker.js"; +import runSteps from "../../../lib/tasks/runSteps.js"; let buildThemes; test.before(async () => { @@ -18,7 +19,11 @@ test.beforeEach(async (t) => { t.context.ReaderCollectionPrioritizedStub = sinon.stub(); t.context.comboByGlob = sinon.stub().resolves([]); - t.context.ReaderCollectionPrioritizedStub.returns({byGlob: t.context.comboByGlob}); + t.context.comboByPath = sinon.stub().resolves(null); + t.context.ReaderCollectionPrioritizedStub.returns({ + byGlob: t.context.comboByGlob, + byPath: t.context.comboByPath + }); buildThemes = await esmock.p("../../../lib/tasks/buildThemes.js", { "@ui5/fs/fsInterface": t.context.fsInterfaceStub, @@ -35,7 +40,7 @@ test.afterEach.always(() => { test.serial("buildThemes", async (t) => { t.plan(6); - const lessResource = {}; + const lessResource = {getPath: () => "/resources/test/library.source.less"}; const workspace = { byGlob: async (globPattern) => { @@ -58,7 +63,7 @@ test.serial("buildThemes", async (t) => { jsonParametersResource ]); - await buildThemes({ + await runSteps(buildThemes, { workspace, options: { projectName: "sap.ui.demo.app", @@ -88,7 +93,7 @@ test.serial("buildThemes", async (t) => { test.serial("buildThemes (compress = false)", async (t) => { t.plan(6); - const lessResource = {}; + const lessResource = {getPath: () => "/resources/test/library.source.less"}; const workspace = { byGlob: async (globPattern) => { @@ -111,7 +116,7 @@ test.serial("buildThemes (compress = false)", async (t) => { jsonParametersResource ]); - await buildThemes({ + await runSteps(buildThemes, { workspace, options: { projectName: "sap.ui.demo.app", @@ -139,7 +144,7 @@ test.serial("buildThemes (compress = false)", async (t) => { }); test.serial("buildThemes (filtering libraries)", async (t) => { - t.plan(3); + t.plan(5); const lessResources = { "sap/ui/lib1/themes/theme1/library.source.less": { @@ -178,16 +183,15 @@ test.serial("buildThemes (filtering libraries)", async (t) => { lessResources["sap/ui/lib3/themes/theme1/library.source.less"] ]); - t.context.comboByGlob - .withArgs("/resources/**/(*.library|library.js)").resolves([ - dotLibraryResources["sap/ui/lib1/.library"], - dotLibraryResources["sap/ui/lib1/library.js"], - dotLibraryResources["sap/ui/lib3/library.js"] - ]); + // Per theme, isThemeAvailable probes the library markers by path. lib1 and lib3 have a marker; + // lib2 does not, so its theme is skipped. + t.context.comboByPath.callsFake(async (p) => + Object.values(dotLibraryResources).find((res) => res.getPath() === p) ?? null); - t.context.themeBuilderStub.returns([{}]); + // One step per surviving theme; a fresh result per call so concurrent writes stay independent. + t.context.themeBuilderStub.callsFake(() => [{}]); - await buildThemes({ + await runSteps(buildThemes, { workspace, options: { projectName: "sap.ui.test.lib1", @@ -196,26 +200,23 @@ test.serial("buildThemes (filtering libraries)", async (t) => { } }); - t.is(t.context.themeBuilderStub.callCount, 1, - "Processor should be called once"); + t.is(t.context.themeBuilderStub.callCount, 2, + "Processor should be called once per surviving theme"); - t.deepEqual(t.context.themeBuilderStub.getCall(0).args[0], { - resources: [ - lessResources["sap/ui/lib1/themes/theme1/library.source.less"], - lessResources["sap/ui/lib3/themes/theme1/library.source.less"] - ], - fs: {}, - options: { - compress: true, - } - }, "Processor should be called with expected arguments"); + const processed = t.context.themeBuilderStub.getCalls().map((call) => call.args[0].resources[0]); + t.true(processed.includes(lessResources["sap/ui/lib1/themes/theme1/library.source.less"]), + "lib1 theme was built"); + t.true(processed.includes(lessResources["sap/ui/lib3/themes/theme1/library.source.less"]), + "lib3 theme was built"); + t.false(processed.includes(lessResources["sap/ui/lib2/themes/theme1/library.source.less"]), + "lib2 theme was skipped (no library marker)"); - t.is(workspace.write.callCount, 1, - "workspace.write should be called once"); + t.is(workspace.write.callCount, 2, + "workspace.write should be called once per surviving theme"); }); test.serial("buildThemes (filtering themes)", async (t) => { - t.plan(3); + t.plan(5); const lessResources = { "sap/ui/lib1/themes/theme1/library.source.less": { @@ -263,9 +264,10 @@ test.serial("buildThemes (filtering themes)", async (t) => { baseThemes["sap/ui/core/themes/theme3/"] ]); - t.context.themeBuilderStub.returns([{}]); + // One step per surviving theme; a fresh result per call so concurrent writes stay independent. + t.context.themeBuilderStub.callsFake(() => [{}]); - await buildThemes({ + await runSteps(buildThemes, { workspace, options: { projectName: "sap.ui.test.lib1", @@ -274,26 +276,23 @@ test.serial("buildThemes (filtering themes)", async (t) => { } }); - t.is(t.context.themeBuilderStub.callCount, 1, - "Processor should be called once"); + t.is(t.context.themeBuilderStub.callCount, 2, + "Processor should be called once per surviving theme"); - t.deepEqual(t.context.themeBuilderStub.getCall(0).args[0], { - resources: [ - lessResources["sap/ui/lib1/themes/theme1/library.source.less"], - lessResources["sap/ui/lib1/themes/theme3/library.source.less"] - ], - fs: {}, - options: { - compress: true, - } - }, "Processor should be called with expected arguments"); + const processed = t.context.themeBuilderStub.getCalls().map((call) => call.args[0].resources[0]); + t.true(processed.includes(lessResources["sap/ui/lib1/themes/theme1/library.source.less"]), + "theme1 was built"); + t.true(processed.includes(lessResources["sap/ui/lib1/themes/theme3/library.source.less"]), + "theme3 was built"); + t.false(processed.includes(lessResources["sap/ui/lib1/themes/theme2/library.source.less"]), + "theme2 was skipped (no sap.ui.core theme folder)"); - t.is(workspace.write.callCount, 1, - "workspace.write should be called once"); + t.is(workspace.write.callCount, 2, + "workspace.write should be called once per surviving theme"); }); test.serial("buildThemes (filtering libraries + themes)", async (t) => { - t.plan(3); + t.plan(6); const lessResources = { "sap/ui/lib1/themes/theme1/library.source.less": { @@ -372,19 +371,17 @@ test.serial("buildThemes (filtering libraries + themes)", async (t) => { ]); t.context.comboByGlob - .withArgs("/resources/**/(*.library|library.js)").resolves([ - dotLibraryResources["sap/ui/lib1/.library"], - dotLibraryResources["sap/ui/lib1/library.js"], - dotLibraryResources["sap/ui/lib3/library.js"] - ]) .withArgs("/resources/sap/ui/core/themes/*", {nodir: false}).resolves([ baseThemes["sap/ui/core/themes/theme1/"], baseThemes["sap/ui/core/themes/theme3/"] ]); + t.context.comboByPath.callsFake(async (p) => + Object.values(dotLibraryResources).find((res) => res.getPath() === p) ?? null); - t.context.themeBuilderStub.returns([{}]); + // One step per surviving theme; a fresh result per call so concurrent writes stay independent. + t.context.themeBuilderStub.callsFake(() => [{}]); - await buildThemes({ + await runSteps(buildThemes, { workspace, options: { projectName: "sap.ui.test.lib1", @@ -394,24 +391,18 @@ test.serial("buildThemes (filtering libraries + themes)", async (t) => { } }); - t.is(t.context.themeBuilderStub.callCount, 1, - "Processor should be called once"); + t.is(t.context.themeBuilderStub.callCount, 4, + "Processor should be called once per surviving theme"); - t.deepEqual(t.context.themeBuilderStub.getCall(0).args[0], { - resources: [ - lessResources["sap/ui/lib1/themes/theme1/library.source.less"], - lessResources["sap/ui/lib1/themes/theme3/library.source.less"], - lessResources["sap/ui/lib3/themes/theme1/library.source.less"], - lessResources["sap/ui/lib3/themes/theme3/library.source.less"] - ], - fs: {}, - options: { - compress: true, - } - }, "Processor should be called with expected arguments"); + const processed = t.context.themeBuilderStub.getCalls().map((call) => call.args[0].resources[0]); + // Surviving: an available library (lib1, lib3) crossed with an available theme (theme1, theme3). + t.true(processed.includes(lessResources["sap/ui/lib1/themes/theme1/library.source.less"]), "lib1 theme1"); + t.true(processed.includes(lessResources["sap/ui/lib1/themes/theme3/library.source.less"]), "lib1 theme3"); + t.true(processed.includes(lessResources["sap/ui/lib3/themes/theme1/library.source.less"]), "lib3 theme1"); + t.true(processed.includes(lessResources["sap/ui/lib3/themes/theme3/library.source.less"]), "lib3 theme3"); - t.is(workspace.write.callCount, 1, - "workspace.write should be called once"); + t.is(workspace.write.callCount, 4, + "workspace.write should be called once per surviving theme"); }); test.serial("buildThemes (useWorkers = true)", async (t) => { @@ -461,7 +452,7 @@ test.serial("buildThemes (useWorkers = true)", async (t) => { jsonParametersResource ]); - await buildThemes({ + await runSteps(buildThemes, { workspace, taskUtil: taskUtilMock, options: { @@ -532,7 +523,7 @@ test.serial("buildThemes with taskUtil and unexpected termination of the workerp mkdir: (...args) => args[args.length - 1](null, {}), }); - await buildThemes({ + await runSteps(buildThemes, { workspace, taskUtil: taskUtilMock, options: { diff --git a/packages/builder/test/lib/tasks/enhanceManifest.js b/packages/builder/test/lib/tasks/enhanceManifest.js index 25d30706708..18c6edb6bb1 100644 --- a/packages/builder/test/lib/tasks/enhanceManifest.js +++ b/packages/builder/test/lib/tasks/enhanceManifest.js @@ -1,4 +1,5 @@ import test from "ava"; +import runSteps from "../../../lib/tasks/runSteps.js"; import sinonGlobal from "sinon"; import esmock from "esmock"; import {createAdapter, createResource} from "@ui5/fs/resourceFactory"; @@ -24,7 +25,7 @@ test.beforeEach(async (t) => { t.context.manifestEnhancerStub = sinon.stub(); t.context.fsInterfaceStub = sinon.stub().returns("fs interface"); - t.context.enhanceManifest = await esmock("../../../lib/tasks/enhanceManifest.js", { + t.context.enhanceManifest = await esmock.p("../../../lib/tasks/enhanceManifest.js", { "@ui5/logger": { getLogger: sinon.stub().withArgs("builder:tasks:enhanceManifest").returns(t.context.log) }, @@ -35,6 +36,7 @@ test.beforeEach(async (t) => { }); test.afterEach.always((t) => { + esmock.purge(t.context.enhanceManifest); t.context.sinon.restore(); }); @@ -70,7 +72,7 @@ test.serial("Transforms single manifest.json resource", async (t) => { t.context.manifestEnhancerStub.returns([resource]); - await enhanceManifest({ + await runSteps(enhanceManifest, { workspace, options: { projectNamespace: "sap/ui/demo/app" @@ -92,7 +94,7 @@ test.serial("Transforms single manifest.json resource", async (t) => { test.serial("Transforms all manifest.json resources", async (t) => { const {enhanceManifest, log} = t.context; - t.plan(6); + t.plan(5); const resourceLib = createResource({ path: "/resources/sap/ui/demo/lib/manifest.json", @@ -173,22 +175,19 @@ test.serial("Transforms all manifest.json resources", async (t) => { } }; - t.context.manifestEnhancerStub.returns([resourceLib, resourceReuseComp1]); + // One step per manifest.json: the unchanged comp2 manifest returns nothing and is not written. + t.context.manifestEnhancerStub.callsFake(({resources}) => + resources[0] === resourceReuseComp2 ? [] : [resources[0]]); - await enhanceManifest({ + await runSteps(enhanceManifest, { workspace, options: { projectNamespace: "sap/ui/demo/lib" } }); - t.is(t.context.manifestEnhancerStub.callCount, 1, - "Processor should be called once"); - - t.true(t.context.manifestEnhancerStub.calledWithExactly({ - resources: [resourceLib, resourceReuseComp1, resourceReuseComp2], - fs: "fs interface" - }), "Processor should be called with expected arguments"); + t.is(t.context.manifestEnhancerStub.callCount, 3, + "Processor should be called once per manifest.json"); t.true(log.warn.notCalled, "No warnings should be logged"); t.true(log.error.notCalled, "No errors should be logged"); @@ -197,7 +196,7 @@ test.serial("Transforms all manifest.json resources", async (t) => { test.serial("Transforms multiple manifest.json resources", async (t) => { const {enhanceManifest, log} = t.context; - t.plan(7); + t.plan(6); const resourceLib = createResource({ path: "/resources/sap/ui/demo/lib/manifest.json", @@ -277,22 +276,18 @@ test.serial("Transforms multiple manifest.json resources", async (t) => { } }; - t.context.manifestEnhancerStub.returns([resourceLib, resourceReuseComp1, resourceReuseComp2]); + // One step per manifest.json: each changed manifest is written back at its own path. + t.context.manifestEnhancerStub.callsFake(({resources}) => [resources[0]]); - await enhanceManifest({ + await runSteps(enhanceManifest, { workspace, options: { projectNamespace: "sap/ui/demo/lib" } }); - t.is(t.context.manifestEnhancerStub.callCount, 1, - "Processor should be called once"); - - t.true(t.context.manifestEnhancerStub.calledWithExactly({ - resources: [resourceLib, resourceReuseComp1, resourceReuseComp2], - fs: "fs interface" - }), "Processor should be called with expected arguments"); + t.is(t.context.manifestEnhancerStub.callCount, 3, + "Processor should be called once per manifest.json"); t.true(log.warn.notCalled, "No warnings should be logged"); t.true(log.error.notCalled, "No errors should be logged"); @@ -328,7 +323,7 @@ test.serial("Should not rewrite the manifest.json if no changes were made", asyn t.context.manifestEnhancerStub.returns([]); - await enhanceManifest({ + await runSteps(enhanceManifest, { workspace, options: { projectNamespace: "sap/ui/demo/app" diff --git a/packages/builder/test/lib/tasks/escapeNonAsciiCharacters.js b/packages/builder/test/lib/tasks/escapeNonAsciiCharacters.js index 4bcc2519a2d..63453884a36 100644 --- a/packages/builder/test/lib/tasks/escapeNonAsciiCharacters.js +++ b/packages/builder/test/lib/tasks/escapeNonAsciiCharacters.js @@ -1,4 +1,5 @@ import test from "ava"; +import runSteps from "../../../lib/tasks/runSteps.js"; import escapeNonAsciiCharacters from "../../../lib/tasks/escapeNonAsciiCharacters.js"; import {createAdapter, createResource} from "@ui5/fs/resourceFactory"; import DuplexCollection from "@ui5/fs/DuplexCollection"; @@ -32,7 +33,7 @@ city=Ort:`; }); await workspace.write(resource); - await escapeNonAsciiCharacters({ + await runSteps(escapeNonAsciiCharacters, { workspace, options: { encoding: "UTF-8", @@ -79,7 +80,7 @@ city=Ort:`; }); await workspace.write(resource); - await escapeNonAsciiCharacters({ + await runSteps(escapeNonAsciiCharacters, { workspace, options: { encoding: "ISO-8859-1", @@ -97,39 +98,17 @@ city=Ort:`; }); test("integration: escape non ascii characters source encoding being empty", async (t) => { - const reader = createAdapter({ - virBasePath: "/" - }); - const writer = createAdapter({ - virBasePath: "/" - }); - const workspace = new DuplexCollection({reader, writer}); - - const error = await t.throwsAsync(escapeNonAsciiCharacters({ - workspace, - options: { - encoding: "", - pattern: "/**/*.properties" - } + const error = t.throws(() => escapeNonAsciiCharacters({ + encoding: "", + pattern: "/**/*.properties" })); return t.is(error.message, "[escapeNonAsciiCharacters] Mandatory option 'encoding' not provided"); }); test("integration: escape non ascii characters source encoding being UTF-16", async (t) => { - const reader = createAdapter({ - virBasePath: "/" - }); - const writer = createAdapter({ - virBasePath: "/" - }); - const workspace = new DuplexCollection({reader, writer}); - - const error = await t.throwsAsync(escapeNonAsciiCharacters({ - workspace, - options: { - encoding: "utf16le", - pattern: "/**/*.properties" - } + const error = t.throws(() => escapeNonAsciiCharacters({ + encoding: "utf16le", + pattern: "/**/*.properties" })); return t.is(error.message, `Encoding "utf16le" is not supported. Only UTF-8, ISO-8859-1 are allowed values`); }); diff --git a/packages/builder/test/lib/tasks/generateThemeDesignerResources.js b/packages/builder/test/lib/tasks/generateThemeDesignerResources.js index 46a6f7a90a9..646b84dbdca 100644 --- a/packages/builder/test/lib/tasks/generateThemeDesignerResources.js +++ b/packages/builder/test/lib/tasks/generateThemeDesignerResources.js @@ -1,6 +1,7 @@ import test from "ava"; import sinonGlobal from "sinon"; import esmock from "esmock"; +import runSteps from "../../../lib/tasks/runSteps.js"; test.beforeEach(async (t) => { const sinon = t.context.sinon = sinonGlobal.createSandbox(); @@ -14,7 +15,7 @@ test.beforeEach(async (t) => { t.context.ResourceStub = sinon.stub(); t.context.libraryLessGeneratorStub = sinon.stub(); - t.context.generateThemeDesignerResources = await esmock("../../../lib/tasks/generateThemeDesignerResources", { + t.context.generateThemeDesignerResources = await esmock.p("../../../lib/tasks/generateThemeDesignerResources", { "../../../lib/processors/libraryLessGenerator": t.context.libraryLessGeneratorStub, "@ui5/fs/ReaderCollectionPrioritized": t.context.ReaderCollectionPrioritizedStub, "@ui5/fs/fsInterface": t.context.fsInterfaceStub, @@ -23,6 +24,7 @@ test.beforeEach(async (t) => { }); test.afterEach.always((t) => { + esmock.purge(t.context.generateThemeDesignerResources); t.context.sinon.restore(); }); @@ -72,9 +74,16 @@ test.serial("generateThemeDesignerResources: Library", async (t) => { const libraryLessResource2 = {}; const libraryLessResource3 = {}; - libraryLessGeneratorStub.resolves([libraryLessResource1, libraryLessResource2, libraryLessResource3]); + // One step per theme, so libraryLessGenerator is called once per theme; return a distinct resource + // per input theme so concurrent writes stay independent. + const lessByTheme = new Map([ + [librarySourceLessResource1, libraryLessResource1], + [librarySourceLessResource2, libraryLessResource2], + [librarySourceLessResource3, libraryLessResource3], + ]); + libraryLessGeneratorStub.callsFake(async ({resources}) => [lessByTheme.get(resources[0])]); - await generateThemeDesignerResources({ + await runSteps(generateThemeDesignerResources, { workspace, dependencies, options: { @@ -84,87 +93,69 @@ test.serial("generateThemeDesignerResources: Library", async (t) => { } }); - t.is(t.context.ReaderCollectionPrioritizedStub.callCount, 1, "ReaderCollectionPrioritized should be created once"); - t.deepEqual(t.context.ReaderCollectionPrioritizedStub.getCall(0).args, [{ - name: `generateThemeDesignerResources - prioritize workspace over dependencies: sap.ui.demo.lib`, - readers: [workspace, dependencies] - }]); + // A combo is created per theme; its first reader is the step's (recording) workspace, the second the + // dependencies reader. + t.is(t.context.ReaderCollectionPrioritizedStub.callCount, 3, "ReaderCollectionPrioritized created per theme"); + const rcpArgs = t.context.ReaderCollectionPrioritizedStub.getCall(0).args[0]; + t.is(rcpArgs.name, `generateThemeDesignerResources - prioritize workspace over dependencies: sap.ui.demo.lib`); + t.is(rcpArgs.readers.length, 2, "combo has a workspace and a dependencies reader"); + t.is(rcpArgs.readers[1], dependencies, "second combo reader is the dependencies reader"); const combo = t.context.ReaderCollectionPrioritizedStub.getCall(0).returnValue; - t.is(fsInterfaceStub.callCount, 1, "fsInterface should be created once"); - t.deepEqual(fsInterfaceStub.getCall(0).args, [combo], "fsInterface should be created for 'combo'"); + t.is(fsInterfaceStub.callCount, 3, "fsInterface created per theme"); + t.is(fsInterfaceStub.getCall(0).args[0], combo, "fsInterface should be created for 'combo'"); const fs = fsInterfaceStub.getCall(0).returnValue; - t.is(libraryLessGeneratorStub.callCount, 1); - - t.deepEqual(libraryLessGeneratorStub.getCall(0).args[0], { - resources: [librarySourceLessResource1, librarySourceLessResource2, librarySourceLessResource3], - fs, - }, "libraryLessGenerator processor should be called with expected arguments"); - + t.is(libraryLessGeneratorStub.callCount, 3, "libraryLessGenerator called per theme"); + const lessInputs = libraryLessGeneratorStub.getCalls().map((call) => call.args[0].resources[0]); + t.true(lessInputs.includes(librarySourceLessResource1), "base theme processed"); + t.true(lessInputs.includes(librarySourceLessResource2), "my_theme processed"); + t.true(lessInputs.includes(librarySourceLessResource3), "sap_fiori_3 processed"); + libraryLessGeneratorStub.getCalls().forEach((call) => + t.is(call.args[0].fs, fs, "libraryLessGenerator called with the combo's fs")); + + // new Resource is created for the library .theming and for the generated base/my_theme .theming; + // sap_fiori_3 is cloned from the core theme instead. Order across the concurrent themes is not + // guaranteed, so assert membership. t.is(ResourceStub.callCount, 3); t.true(ResourceStub.alwaysCalledWithNew()); - - t.deepEqual(ResourceStub.getCall(0).args, [{ - path: "/resources/sap/ui/demo/lib/.theming", - string: JSON.stringify({ + const resourceArgs = ResourceStub.getCalls().map((call) => call.args[0]); + t.true(resourceArgs.some((args) => + args.path === "/resources/sap/ui/demo/lib/.theming" && + args.string === JSON.stringify({ sEntity: "Library", sId: "sap/ui/demo/lib", sVersion: "1.2.3" - }, null, 2) - }]); - const libraryDotTheming = ResourceStub.getCall(0).returnValue; - - t.deepEqual(ResourceStub.getCall(1).args, [{ - path: "/resources/sap/ui/demo/lib/themes/base/.theming", - string: JSON.stringify({ + }, null, 2)), "library .theming created"); + t.true(resourceArgs.some((args) => + args.path === "/resources/sap/ui/demo/lib/themes/base/.theming" && + args.string === JSON.stringify({ sEntity: "Theme", sId: "base", sVendor: "SAP" - }, null, 2) - }]); - const baseThemeDotTheming = ResourceStub.getCall(1).returnValue; - - t.deepEqual(ResourceStub.getCall(2).args, [{ - path: "/resources/sap/ui/demo/lib/themes/my_theme/.theming", - string: JSON.stringify({ + }, null, 2)), "base theme .theming created"); + t.true(resourceArgs.some((args) => + args.path === "/resources/sap/ui/demo/lib/themes/my_theme/.theming" && + args.string === JSON.stringify({ sEntity: "Theme", sId: "my_theme", sVendor: "SAP", oExtends: "base" - }, null, 2) - }]); - const myThemeDotTheming = ResourceStub.getCall(2).returnValue; + }, null, 2)), "my_theme .theming created"); t.is(clonedCoreBaseDotThemingResourceStub.setPath.callCount, 1); t.deepEqual(clonedCoreBaseDotThemingResourceStub.setPath.getCall(0).args, ["/resources/sap/ui/demo/lib/themes/sap_fiori_3/.theming"]); - t.is(workspace.write.callCount, 7); - t.is(workspace.write.getCall(0).args.length, 1, - "workspace.write for libraryDotTheming should be called with 1 argument"); - t.is(workspace.write.getCall(0).args[0], libraryDotTheming, - "workspace.write should be called with libraryDotTheming"); - t.is(workspace.write.getCall(1).args.length, 1, - "workspace.write for baseThemeDotTheming should be called with 1 argument"); - t.is(workspace.write.getCall(1).args[0], baseThemeDotTheming, - "workspace.write should be called with baseThemeDotTheming"); - t.is(workspace.write.getCall(2).args.length, 1, - "workspace.write for myThemeDotTheming should be called with 1 argument"); - t.is(workspace.write.getCall(2).args[0], myThemeDotTheming, - "workspace.write should be called with myThemeDotTheming"); - t.is(workspace.write.getCall(3).args.length, 1, - "workspace.write for clonedCoreBaseDotThemingResourceStub should be called with 1 argument"); - t.is(workspace.write.getCall(3).args[0], clonedCoreBaseDotThemingResourceStub, - "workspace.write should be called with clonedCoreBaseDotThemingResourceStub"); - t.is(workspace.write.getCall(4).args.length, 1, - "workspace.write for libraryLessResource1 should be called with 1 argument"); - t.is(workspace.write.getCall(4).args[0], libraryLessResource1, - "workspace.write should be called with libraryLessResource1"); - t.is(workspace.write.getCall(5).args.length, 1, - "workspace.write for libraryLessResource2 should be called with 1 argument"); - t.is(workspace.write.getCall(5).args[0], libraryLessResource2, - "workspace.write should be called with libraryLessResource2"); + // Written: the library .theming, the three theme .theming (two created, sap_fiori_3 cloned), and the + // three library.less resources. + const written = workspace.write.getCalls().map((call) => call.args[0]); + t.is(written.length, 7, "workspace.write called for every produced resource"); + ResourceStub.getCalls().forEach((call) => + t.true(written.includes(call.returnValue), "each created .theming was written")); + t.true(written.includes(clonedCoreBaseDotThemingResourceStub), "the cloned sap_fiori_3 .theming was written"); + [libraryLessResource1, libraryLessResource2, libraryLessResource3].forEach((resource) => + t.true(written.includes(resource), "each library.less was written")); }); test.serial("generateThemeDesignerResources: Library sap.ui.core", async (t) => { @@ -197,7 +188,7 @@ test.serial("generateThemeDesignerResources: Library sap.ui.core", async (t) => libraryLessGeneratorStub.resolves([libraryLessResource]); - await generateThemeDesignerResources({ + await runSteps(generateThemeDesignerResources, { workspace, dependencies, options: { @@ -208,10 +199,10 @@ test.serial("generateThemeDesignerResources: Library sap.ui.core", async (t) => }); t.is(t.context.ReaderCollectionPrioritizedStub.callCount, 1, "ReaderCollectionPrioritized should be created once"); - t.deepEqual(t.context.ReaderCollectionPrioritizedStub.getCall(0).args, [{ - name: `generateThemeDesignerResources - prioritize workspace over dependencies: sap.ui.core`, - readers: [workspace, dependencies] - }]); + const rcpArgs = t.context.ReaderCollectionPrioritizedStub.getCall(0).args[0]; + t.is(rcpArgs.name, `generateThemeDesignerResources - prioritize workspace over dependencies: sap.ui.core`); + t.is(rcpArgs.readers.length, 2, "combo has a workspace and a dependencies reader"); + t.is(rcpArgs.readers[1], dependencies, "second combo reader is the dependencies reader"); const combo = t.context.ReaderCollectionPrioritizedStub.getCall(0).returnValue; t.is(fsInterfaceStub.callCount, 1, "fsInterface should be created once"); @@ -296,7 +287,7 @@ test.serial("generateThemeDesignerResources: Library sap.ui.core with existing l libraryLessGeneratorStub.resolves([libraryLessResource]); - await generateThemeDesignerResources({ + await runSteps(generateThemeDesignerResources, { workspace, dependencies, options: { @@ -307,10 +298,10 @@ test.serial("generateThemeDesignerResources: Library sap.ui.core with existing l }); t.is(t.context.ReaderCollectionPrioritizedStub.callCount, 1, "ReaderCollectionPrioritized should be created once"); - t.deepEqual(t.context.ReaderCollectionPrioritizedStub.getCall(0).args, [{ - name: `generateThemeDesignerResources - prioritize workspace over dependencies: sap.ui.core`, - readers: [workspace, dependencies] - }]); + const rcpArgs = t.context.ReaderCollectionPrioritizedStub.getCall(0).args[0]; + t.is(rcpArgs.name, `generateThemeDesignerResources - prioritize workspace over dependencies: sap.ui.core`); + t.is(rcpArgs.readers.length, 2, "combo has a workspace and a dependencies reader"); + t.is(rcpArgs.readers[1], dependencies, "second combo reader is the dependencies reader"); const combo = t.context.ReaderCollectionPrioritizedStub.getCall(0).returnValue; t.is(fsInterfaceStub.callCount, 1, "fsInterface should be created once"); @@ -387,7 +378,7 @@ test.serial("generateThemeDesignerResources: Library sap.ui.core without themes, libraryLessGeneratorStub.resolves([libraryLessResource]); - await generateThemeDesignerResources({ + await runSteps(generateThemeDesignerResources, { workspace, dependencies, options: { @@ -456,7 +447,7 @@ test.serial("generateThemeDesignerResources: Library sap.ui.core with existing i libraryLessGeneratorStub.resolves([libraryLessResource]); - await t.throwsAsync(generateThemeDesignerResources({ + await t.throwsAsync(runSteps(generateThemeDesignerResources, { workspace, dependencies, options: { @@ -489,7 +480,7 @@ test.serial("generateThemeDesignerResources: Library sap.ui.documentation is ski write: sinon.stub() }; - await generateThemeDesignerResources({ + await runSteps(generateThemeDesignerResources, { workspace: {}, dependencies: {}, options: { @@ -517,7 +508,7 @@ test.serial("generateThemeDesignerResources: Library without themes", async (t) write: sinon.stub() }; - await generateThemeDesignerResources({ + await runSteps(generateThemeDesignerResources, { workspace, dependencies: {}, options: { @@ -575,7 +566,7 @@ test.serial("generateThemeDesignerResources: Theme-Library", async (t) => { libraryLessGeneratorStub.resolves([libraryLessResource]); - await generateThemeDesignerResources({ + await runSteps(generateThemeDesignerResources, { workspace, dependencies, options: { @@ -585,10 +576,10 @@ test.serial("generateThemeDesignerResources: Theme-Library", async (t) => { }); t.is(t.context.ReaderCollectionPrioritizedStub.callCount, 1, "ReaderCollectionPrioritized should be created once"); - t.deepEqual(t.context.ReaderCollectionPrioritizedStub.getCall(0).args, [{ - name: `generateThemeDesignerResources - prioritize workspace over dependencies: sap.ui.demo.lib`, - readers: [workspace, dependencies] - }]); + const rcpArgs = t.context.ReaderCollectionPrioritizedStub.getCall(0).args[0]; + t.is(rcpArgs.name, `generateThemeDesignerResources - prioritize workspace over dependencies: sap.ui.demo.lib`); + t.is(rcpArgs.readers.length, 2, "combo has a workspace and a dependencies reader"); + t.is(rcpArgs.readers[1], dependencies, "second combo reader is the dependencies reader"); const combo = t.context.ReaderCollectionPrioritizedStub.getCall(0).returnValue; t.is(fsInterfaceStub.callCount, 1, "fsInterface should be created once"); @@ -653,7 +644,7 @@ test.serial("generateThemeDesignerResources: .theming file missing in sap.ui.cor libraryLessGeneratorStub.resolves([libraryLessResource]); - await t.throwsAsync(generateThemeDesignerResources({ + await t.throwsAsync(runSteps(generateThemeDesignerResources, { workspace, dependencies, options: { @@ -708,7 +699,7 @@ test.serial("generateThemeDesignerResources: Failed to extract library name from }; const dependencies = {}; - await t.throwsAsync(generateThemeDesignerResources({ + await t.throwsAsync(runSteps(generateThemeDesignerResources, { workspace, dependencies, options: { diff --git a/packages/builder/test/lib/tasks/minify.integration.js b/packages/builder/test/lib/tasks/minify.integration.js index ef26c411df7..2aa3136d8db 100644 --- a/packages/builder/test/lib/tasks/minify.integration.js +++ b/packages/builder/test/lib/tasks/minify.integration.js @@ -1,6 +1,7 @@ import test from "ava"; import sinonGlobal from "sinon"; import minify from "../../../lib/tasks/minify.js"; +import runSteps from "../../../lib/tasks/runSteps.js"; import * as resourceFactory from "@ui5/fs/resourceFactory"; import DuplexCollection from "@ui5/fs/DuplexCollection"; @@ -31,6 +32,7 @@ test.afterEach.always((t) => { test.serial("integration: minify omitSourceMapResources=true", async (t) => { const taskUtil = { + getEnv: (name) => process.env[name], setTag: t.context.sinon.stub(), getTag: t.context.sinon.stub().returns(false), STANDARD_TAGS: { @@ -53,7 +55,7 @@ test();`; }); await reader.write(testResource); - await minify({ + await runSteps(minify, { workspace, taskUtil, options: { @@ -110,6 +112,7 @@ test();`; test.serial("integration: minify omitSourceMapResources=false", async (t) => { const taskUtil = { + getEnv: (name) => process.env[name], setTag: t.context.sinon.stub(), getTag: t.context.sinon.stub().returns(false), STANDARD_TAGS: { @@ -132,7 +135,7 @@ test();`; }); await reader.write(testResource); - await minify({ + await runSteps(minify, { workspace, taskUtil, options: { @@ -197,7 +200,7 @@ test();`; }); await reader.write(testResource); - await minify({ + await runSteps(minify, { workspace, options: { pattern: "/test.js", @@ -244,7 +247,7 @@ test();`; }); await reader.write(testResource); - await minify({ + await runSteps(minify, { workspace, options: { pattern: "/test.js", @@ -280,6 +283,7 @@ ${SOURCE_MAPPING_URL}=test.js.map`; test.serial("integration: minify error", async (t) => { const taskUtil = { + getEnv: (name) => process.env[name], setTag: t.context.sinon.stub(), getTag: t.context.sinon.stub().returns(false), STANDARD_TAGS: { @@ -300,7 +304,7 @@ return;`; await reader.write(testResource); await t.throwsAsync(() => { - return minify({ + return runSteps(minify, { workspace, taskUtil, options: { @@ -332,7 +336,7 @@ return;`; await reader.write(testResource); await t.throwsAsync(() => { - return minify({ + return runSteps(minify, { workspace, options: { pattern: "/resources/my/namespace/test.js", @@ -367,6 +371,7 @@ test.serial("integration: minify with taskUtil and resources tagged with OmitFro await reader.write(testResource2); const taskUtil = { + getEnv: (name) => process.env[name], STANDARD_TAGS: { HasDebugVariant: "1️⃣", IsDebugVariant: "2️⃣", @@ -383,7 +388,7 @@ test.serial("integration: minify with taskUtil and resources tagged with OmitFro registerCleanupTask: t.context.sinon.stub() }; - await minify({ + await runSteps(minify, { workspace, taskUtil, options: { diff --git a/packages/builder/test/lib/tasks/minify.js b/packages/builder/test/lib/tasks/minify.js index 1edc139c0b7..e1abdccbe26 100644 --- a/packages/builder/test/lib/tasks/minify.js +++ b/packages/builder/test/lib/tasks/minify.js @@ -1,6 +1,7 @@ import test from "ava"; import sinonGlobal from "sinon"; import esmock from "esmock"; +import runSteps from "../../../lib/tasks/runSteps.js"; test.beforeEach(async (t) => { const sinon = t.context.sinon = sinonGlobal.createSandbox(); @@ -11,6 +12,7 @@ test.beforeEach(async (t) => { t.context.taskUtil = { setTag: sinon.stub(), getTag: sinon.stub(), + getEnv: sinon.stub().returns(undefined), STANDARD_TAGS: { HasDebugVariant: "has debug variant", IsDebugVariant: "is debug variant", @@ -21,7 +23,7 @@ test.beforeEach(async (t) => { t.context.fsInterfaceStub = sinon.stub().returns("fs interface"); t.context.minifierStub = sinon.stub(); - t.context.minify = await esmock("../../../lib/tasks/minify.js", { + t.context.minify = await esmock.p("../../../lib/tasks/minify.js", { "@ui5/fs/fsInterface": t.context.fsInterfaceStub, "../../../lib/processors/minifier.js": t.context.minifierStub }); @@ -35,22 +37,28 @@ test.afterEach.always(async (t) => { await cleanupTask(); } + esmock.purge(t.context.minify); t.context.sinon.restore(); }); test("minify: Default params", async (t) => { const {minify, workspace, taskUtil, minifierStub} = t.context; - minifierStub.resolves([{ - resource: "resource A", - dbgResource: "dbgResource A", - sourceMapResource: "sourceMapResource A", - dbgSourceMapResource: "dbgSourceMapResource A" // optional - }, { - resource: "resource B", - dbgResource: "dbgResource B", - sourceMapResource: "sourceMapResource B", - }]); - await minify({ + // One step per resource: the processor is called once per input resource. + const processedByInput = { + "resource A": { + resource: "resource A", + dbgResource: "dbgResource A", + sourceMapResource: "sourceMapResource A", + dbgSourceMapResource: "dbgSourceMapResource A" // optional + }, + "resource B": { + resource: "resource B", + dbgResource: "dbgResource B", + sourceMapResource: "sourceMapResource B", + } + }; + minifierStub.callsFake(({resources}) => Promise.resolve([processedByInput[resources[0]]])); + await runSteps(minify, { workspace, taskUtil, options: { @@ -58,9 +66,10 @@ test("minify: Default params", async (t) => { } }); - t.is(minifierStub.callCount, 1, "minifier got called once"); + t.is(minifierStub.callCount, 2, "minifier got called once per resource"); const minifierCallArgs = minifierStub.firstCall.firstArg; - t.deepEqual(minifierCallArgs.resources, ["resource A", "resource B"], "Correct resources provided to processor"); + t.deepEqual(minifierCallArgs.resources, ["resource A"], "First step processes the first resource"); + t.deepEqual(minifierStub.secondCall.firstArg.resources, ["resource B"], "Second step: second resource"); t.is(minifierCallArgs.fs, "fs interface", "Correct fs interface provided to processor"); t.is(minifierCallArgs.taskUtil, taskUtil, "Correct taskUtil provided to processor"); t.deepEqual(minifierCallArgs.options, { @@ -100,17 +109,22 @@ test("minify: Default params", async (t) => { test("minify: omitSourceMapResources: true, useInputSourceMaps: false", async (t) => { const {minify, workspace, taskUtil, minifierStub} = t.context; - minifierStub.resolves([{ - resource: "resource A", - dbgResource: "dbgResource A", - sourceMapResource: "sourceMapResource A", - dbgSourceMapResource: "dbgSourceMapResource A" // optional - }, { - resource: "resource B", - dbgResource: "dbgResource B", - sourceMapResource: "sourceMapResource B", - }]); - await minify({ + // One step per resource: the processor is called once per input resource. + const processedByInput = { + "resource A": { + resource: "resource A", + dbgResource: "dbgResource A", + sourceMapResource: "sourceMapResource A", + dbgSourceMapResource: "dbgSourceMapResource A" // optional + }, + "resource B": { + resource: "resource B", + dbgResource: "dbgResource B", + sourceMapResource: "sourceMapResource B", + } + }; + minifierStub.callsFake(({resources}) => Promise.resolve([processedByInput[resources[0]]])); + await runSteps(minify, { workspace, taskUtil, options: { @@ -120,9 +134,10 @@ test("minify: omitSourceMapResources: true, useInputSourceMaps: false", async (t } }); - t.is(minifierStub.callCount, 1, "minifier got called once"); + t.is(minifierStub.callCount, 2, "minifier got called once per resource"); const minifierCallArgs = minifierStub.firstCall.firstArg; - t.deepEqual(minifierCallArgs.resources, ["resource A", "resource B"], "Correct resources provided to processor"); + t.deepEqual(minifierCallArgs.resources, ["resource A"], "First step processes the first resource"); + t.deepEqual(minifierStub.secondCall.firstArg.resources, ["resource B"], "Second step: second resource"); t.is(minifierCallArgs.fs, "fs interface", "Correct fs interface provided to processor"); t.is(minifierCallArgs.taskUtil, taskUtil, "Correct taskUtil provided to processor"); t.deepEqual(minifierCallArgs.options, { @@ -174,26 +189,32 @@ test("minify: omitSourceMapResources: true, useInputSourceMaps: false", async (t test("minify: No taskUtil", async (t) => { const {minify, workspace, minifierStub} = t.context; - minifierStub.resolves([{ - resource: "resource A", - dbgResource: "dbgResource A", - sourceMapResource: "sourceMapResource A", - dbgSourceMapResource: "dbgSourceMapResource A" // optional - }, { - resource: "resource B", - dbgResource: "dbgResource B", - sourceMapResource: "sourceMapResource B", - }]); - await minify({ + // One step per resource: the processor is called once per input resource. + const processedByInput = { + "resource A": { + resource: "resource A", + dbgResource: "dbgResource A", + sourceMapResource: "sourceMapResource A", + dbgSourceMapResource: "dbgSourceMapResource A" // optional + }, + "resource B": { + resource: "resource B", + dbgResource: "dbgResource B", + sourceMapResource: "sourceMapResource B", + } + }; + minifierStub.callsFake(({resources}) => Promise.resolve([processedByInput[resources[0]]])); + await runSteps(minify, { workspace, options: { pattern: "**" } }); - t.is(minifierStub.callCount, 1, "minifier got called once"); + t.is(minifierStub.callCount, 2, "minifier got called once per resource"); const minifierCallArgs = minifierStub.firstCall.firstArg; - t.deepEqual(minifierCallArgs.resources, ["resource A", "resource B"], "Correct resources provided to processor"); + t.deepEqual(minifierCallArgs.resources, ["resource A"], "First step processes the first resource"); + t.deepEqual(minifierStub.secondCall.firstArg.resources, ["resource B"], "Second step: second resource"); t.is(minifierCallArgs.fs, "fs interface", "Correct fs interface provided to processor"); t.is(minifierCallArgs.taskUtil, undefined, "No taskUtil provided to processor"); t.deepEqual(minifierCallArgs.options, { diff --git a/packages/builder/test/lib/tasks/replaceBuildtime.js b/packages/builder/test/lib/tasks/replaceBuildtime.js index 129af3f31ad..40e772f52c7 100644 --- a/packages/builder/test/lib/tasks/replaceBuildtime.js +++ b/packages/builder/test/lib/tasks/replaceBuildtime.js @@ -1,4 +1,5 @@ import test from "ava"; +import runSteps from "../../../lib/tasks/runSteps.js"; import replaceBuildtime from "../../../lib/tasks/replaceBuildtime.js"; import {createAdapter, createResource} from "@ui5/fs/resourceFactory"; import DuplexCollection from "@ui5/fs/DuplexCollection"; @@ -22,7 +23,7 @@ test("integration: replace version", async (t) => { const workspace = new DuplexCollection({reader, writer}); await reader.write(resource); - await replaceBuildtime({ + await runSteps(replaceBuildtime, { workspace, options: { pattern: "/test.js" @@ -42,3 +43,37 @@ test("integration: replace version", async (t) => { t.regex(values[1], expectedDatePattern, "date matches the given pattern"); } }); + +test("integration: buildtime is sourced from taskUtil.getBuildTime", async (t) => { + const reader = createAdapter({ + virBasePath: "/" + }); + const writer = createAdapter({ + virBasePath: "/" + }); + const workspace = new DuplexCollection({reader, writer}); + + // A fixed build run timestamp lets the test assert the exact formatted output: 25 September 2026, + // 14:07 local -> "20260925-1407". Sourcing it from taskUtil.getBuildTime (not new Date()) keeps + // every project and task in the run on one timestamp. + const buildTime = new Date(2026, 8, 25, 14, 7, 3); + + const resource = createResource({ + path: "/test.js", + string: "// timestamp: ${buildtime}" + }); + await reader.write(resource); + + await runSteps(replaceBuildtime, { + workspace, + taskUtil: {getBuildTime: () => buildTime}, + options: { + pattern: "/test.js" + } + }); + + const transformedResource = await writer.byPath("/test.js"); + t.truthy(transformedResource, "Could find /test.js in target"); + t.is(await transformedResource.getString(), "// timestamp: 20260925-1407", + "buildtime is formatted from the injected timestamp"); +}); diff --git a/packages/builder/test/lib/tasks/replaceCopyright.js b/packages/builder/test/lib/tasks/replaceCopyright.js index 5aa7c2cd89b..d4503ef7748 100644 --- a/packages/builder/test/lib/tasks/replaceCopyright.js +++ b/packages/builder/test/lib/tasks/replaceCopyright.js @@ -1,4 +1,6 @@ import test from "ava"; +import runSteps from "../../../lib/tasks/runSteps.js"; +import sinon from "sinon"; import replaceCopyright from "../../../lib/tasks/replaceCopyright.js"; import {createAdapter, createResource} from "@ui5/fs/resourceFactory"; import DuplexCollection from "@ui5/fs/DuplexCollection"; @@ -38,7 +40,7 @@ console.log('HelloWorld');`; await workspace.write(resource); - await replaceCopyright({ + await runSteps(replaceCopyright, { workspace, options: { copyright: copyright, @@ -56,6 +58,56 @@ console.log('HelloWorld');`; }); +test("integration: replace copyright reads the current year through taskUtil.getTime", async (t) => { + const reader = createAdapter({ + virBasePath: "/" + }); + const writer = createAdapter({ + virBasePath: "/" + }); + const workspace = new DuplexCollection({reader, writer}); + + /* eslint-disable no-useless-escape */ + const content = `/*! + * $\{copyright\} + */ +console.log('HelloWorld');`; + /* eslint-enable no-useless-escape */ + + const copyright = `(c) Copyright 2009-\${currentYear} SAP SE or an SAP affiliate company.`; + + // A stubbed taskUtil.getTime lets the test assert both that the task reads the year through the + // tracked accessor (not new Date()) and that it requests the "year" granularity. + const getTime = sinon.stub().returns("1999"); + const expected = `/*! + * (c) Copyright 2009-1999 SAP SE or an SAP affiliate company. + */ +console.log('HelloWorld');`; + + const resource = createResource({ + path: "/test.js", + string: content + }); + + await workspace.write(resource); + + await runSteps(replaceCopyright, { + workspace, + taskUtil: {getTime}, + options: { + copyright: copyright, + pattern: "/**/*.js" + } + }); + + t.true(getTime.calledOnceWithExactly("year"), "taskUtil.getTime was called with the year granularity"); + + const transformedResource = await writer.byPath("/test.js"); + t.truthy(transformedResource, "Could find /test.js in target"); + t.is(await transformedResource.getString(), expected); +}); + + test("test.xml: replace @copyright@", async (t) => { const reader = createAdapter({ virBasePath: "/" @@ -87,7 +139,7 @@ test("test.xml: replace @copyright@", async (t) => { }); await reader.write(resource); - await replaceCopyright({ + await runSteps(replaceCopyright, { workspace, options: { pattern: "/**/*.xml", diff --git a/packages/builder/test/lib/tasks/replaceVersion.js b/packages/builder/test/lib/tasks/replaceVersion.js index 6d831610dd4..58ebf224340 100644 --- a/packages/builder/test/lib/tasks/replaceVersion.js +++ b/packages/builder/test/lib/tasks/replaceVersion.js @@ -1,4 +1,5 @@ import test from "ava"; +import runSteps from "../../../lib/tasks/runSteps.js"; import replaceVersion from "../../../lib/tasks/replaceVersion.js"; import {createAdapter, createResource} from "@ui5/fs/resourceFactory"; import DuplexCollection from "@ui5/fs/DuplexCollection"; @@ -21,7 +22,7 @@ test("integration: replace version", async (t) => { const workspace = new DuplexCollection({reader, writer}); await reader.write(resource); - await replaceVersion({ + await runSteps(replaceVersion, { workspace, options: { pattern: "/test.js", diff --git a/packages/builder/test/lib/tasks/runSteps.js b/packages/builder/test/lib/tasks/runSteps.js new file mode 100644 index 00000000000..9f3f05bf482 --- /dev/null +++ b/packages/builder/test/lib/tasks/runSteps.js @@ -0,0 +1,176 @@ +import test from "ava"; +import runSteps from "../../../lib/tasks/runSteps.js"; + +function createResource(resourcePath, content = resourcePath) { + return { + getPath: () => resourcePath, + getIntegrity: async () => `sha256-${content}`, + getString: async () => content, + }; +} + +// Minimal in-memory workspace: byGlob/byPath/write plus getName so a BufferedWriter can wrap it. Write +// order and the trailing write arguments are recorded so the key-order flush and the argument handling are +// observable. +function createWorkspace(initial = []) { + const store = new Map(initial.map((res) => [res.getPath(), res])); + const writeOrder = []; + const writeArgs = []; + return { + getName: () => "workspace", + byGlob: async () => [...store.values()], + byPath: async (virPath) => store.get(virPath) ?? null, + write: async (resource, ...args) => { + writeOrder.push(resource.getPath()); + writeArgs.push(args); + store.set(resource.getPath(), resource); + }, + store, + writeOrder, + writeArgs, + }; +} + +test("Runs a scalar step and threads its return into a consumer via needs", async (t) => { + const workspace = createWorkspace(); + let consumed; + await runSteps((options) => [ + {name: "scan", run: async () => ({flag: options.flag})}, + {name: "use", needs: ["scan"], run: async ({needs, workspace}) => { + consumed = needs.scan; + await workspace.write(createResource("/out")); + }}, + ], {workspace, options: {flag: 42}}); + + t.deepEqual(consumed, {flag: 42}, "The producer's return arrived as needs.scan"); + t.true(workspace.store.has("/out"), "The consumer's write persisted"); +}); + +test("Fans out a map step's keys and writes each", async (t) => { + const workspace = createWorkspace(); + const ran = []; + await runSteps(() => [ + {name: "m", keys: async () => ["a", "b", "c"], each: async (key, {workspace}) => { + ran.push(key); + await workspace.write(createResource(`/out/${key}`)); + }}, + ], {workspace}); + + t.deepEqual(ran.sort(), ["a", "b", "c"], "each ran once per key"); + t.true(workspace.store.has("/out/a") && workspace.store.has("/out/b") && workspace.store.has("/out/c"), + "Every key's write persisted"); +}); + +test("A producer return reaches a map step's keys and each", async (t) => { + const workspace = createWorkspace(); + const eachSaw = []; + await runSteps(() => [ + {name: "scan", run: async () => ({wanted: ["x", "y"]})}, + {name: "build", needs: ["scan"], keys: async ({needs}) => needs.scan.wanted, + each: async (key, {needs}) => { + eachSaw.push([key, needs.scan.wanted.length]); + }}, + ], {workspace}); + + t.deepEqual(eachSaw.sort(), [["x", 2], ["y", 2]], "each saw the producer return per key"); +}); + +test("A concurrent map step flushes its writes in key order", async (t) => { + const workspace = createWorkspace(); + await runSteps(() => [ + {name: "m", keys: async () => ["a", "b", "c"], each: async (key, {workspace}) => { + // Reverse the natural completion order so the key-order flush is observable. + if (key === "a") { + await new Promise((resolve) => setTimeout(resolve, 15)); + } + await workspace.write(createResource(`/${key}.out`)); + }}, + ], {workspace}); + + t.deepEqual(workspace.writeOrder, ["/a.out", "/b.out", "/c.out"], + "Buffered writes flushed in key order regardless of completion order"); +}); + +test("A sequential map step makes an earlier key's write visible to a later key", async (t) => { + const workspace = createWorkspace(); + let secondSawFirst = false; + await runSteps(() => [ + {name: "m", sequential: true, keys: async () => ["first", "second"], each: async (key, {workspace}) => { + if (key === "first") { + await workspace.write(createResource("/shared")); + } else { + secondSawFirst = !!(await workspace.byPath("/shared")); + } + }}, + ], {workspace}); + + t.true(secondSawFirst, "The second key read the first key's write"); +}); + +test("Concurrent map-step keys writing the same path throw", async (t) => { + const workspace = createWorkspace(); + const err = await t.throwsAsync(runSteps(() => [ + {name: "m", keys: async () => ["a", "b"], each: async (key, {workspace}) => { + await workspace.write(createResource("/same")); + }}, + ], {workspace})); + // The exact user-visible message, shared with the cached runner through @ui5/fs/internal/stepWriteBuffer. + t.is(err.message, + "Concurrent map-step keys must not write the same resource path /same. " + + "Pass {sequential: true} if a later key must build on an earlier key's writes.", + "The same-path guard surfaces the shared message verbatim"); +}); + +test("A concurrent map step preserves each key's write arguments through the flush", async (t) => { + const workspace = createWorkspace(); + await runSteps(() => [ + {name: "m", keys: async () => ["with", "without"], each: async (key, {workspace}) => { + if (key === "with") { + await workspace.write(createResource("/with"), {drain: true}); + } else { + // No options: the override must not fabricate a defaulted options object for the flush. + await workspace.write(createResource("/without")); + } + }}, + ], {workspace}); + + t.deepEqual(workspace.writeOrder, ["/with", "/without"], "Flushed in key order"); + t.deepEqual(workspace.writeArgs, [[{drain: true}], []], + "A write with options replays its options; a write without options replays no extra argument"); +}); + +test("A later step sees an earlier step's write", async (t) => { + const workspace = createWorkspace(); + let laterSaw = false; + await runSteps(() => [ + {name: "first", run: async ({workspace}) => { + await workspace.write(createResource("/from-first")); + }}, + {name: "second", run: async ({workspace}) => { + laterSaw = !!(await workspace.byPath("/from-first")); + }}, + ], {workspace}); + + t.true(laterSaw, "The second step read the first step's write"); +}); + +test("taskUtil and dependencies are passed through to steps", async (t) => { + const workspace = createWorkspace(); + const taskUtil = {marker: "taskUtil"}; + const dependencies = {marker: "dependencies"}; + let seen; + await runSteps(() => [ + {name: "s", run: async (ctx) => { + seen = {taskUtil: ctx.taskUtil, dependencies: ctx.dependencies}; + }}, + ], {workspace, taskUtil, dependencies}); + + t.is(seen.taskUtil, taskUtil, "taskUtil passed through"); + t.is(seen.dependencies, dependencies, "dependencies passed through"); +}); + +test("An empty step list is a no-op", async (t) => { + const workspace = createWorkspace(); + await runSteps(() => [], {workspace}); + t.is(workspace.store.size, 0, "Nothing written"); +}); From cfb1b4a0f7db1f7e9655604146f6ee2c72773a7b Mon Sep 17 00:00:00 2001 From: Matthias Osswald Date: Wed, 7 Oct 2026 10:54:36 +0200 Subject: [PATCH 4/8] feat(builder): Read the bundle-info preload flag through taskUtil.getEnv generateLibraryPreload is not a step factory, so it reads the experimental UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD flag directly from process.env. The incremental build cache cannot see that read, so a changed flag value does not invalidate the cached bundle. Read the flag through taskUtil.getEnv when a TaskUtil is present, which records the read as a tracked build-cache input. Fall back to the direct process.env read when the task runs without a TaskUtil. Co-authored-by: Merlin Beutlberger --- .../builder/lib/tasks/bundlers/generateLibraryPreload.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/builder/lib/tasks/bundlers/generateLibraryPreload.js b/packages/builder/lib/tasks/bundlers/generateLibraryPreload.js index 3c7766b96e8..f310a65f4a4 100644 --- a/packages/builder/lib/tasks/bundlers/generateLibraryPreload.js +++ b/packages/builder/lib/tasks/bundlers/generateLibraryPreload.js @@ -348,7 +348,12 @@ export default async function({workspace, taskUtil, options: {skipBundles = [], } const coreVersion = taskUtil?.getProject("sap.ui.core")?.getVersion(); const allowStringBundling = taskUtil?.getProject().getSpecVersion().lt("4.0"); - const createBundleInfoPreload = !!process.env.UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD; + // Read the experimental flag through taskUtil.getEnv so that its usage is tracked as a task + // input by the incremental build cache (a changed value then invalidates this task's cached + // result). Fall back to a direct process.env read when the task runs without a TaskUtil. + const createBundleInfoPreload = taskUtil?.getEnv ? + !!taskUtil.getEnv("UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD") : + !!process.env.UI5_CLI_EXPERIMENTAL_BUNDLE_INFO_PRELOAD; const execModuleBundlerIfNeeded = ({options, resources}) => { if (skipBundles.includes(options.bundleDefinition.name)) { log.verbose(`Skipping generation of bundle ${options.bundleDefinition.name}`); From b8e7160fc3285ed38852206330e3fe405b2137ad Mon Sep 17 00:00:00 2001 From: Matthias Osswald Date: Wed, 7 Oct 2026 10:54:44 +0200 Subject: [PATCH 5/8] docs: Document the step-based task API in the incremental-build skill Update the incremental-build skill for the step-based build system. architecture.md describes the component map for StepRunner, MonitoredTaskUtil, BuildStageCache, TaskInputSet, stageSignature, and quantizeTime, the step-factory contract, the one-stage-per-step model, and the per-key identity argument. performance-investigation.md adds the step pipeline to the playbook: the running-step log marker, the per-key identity tiering that avoids content hashing each enumerated key, the cheap delta-build step fold, and the step invocation sidecar re-serialization rule. Co-authored-by: Merlin Beutlberger --- .../skills/incremental-build/architecture.md | 159 +++++++++++++++--- .../performance-investigation.md | 122 +++++++++++++- 2 files changed, 260 insertions(+), 21 deletions(-) diff --git a/.claude/skills/incremental-build/architecture.md b/.claude/skills/incremental-build/architecture.md index 66e1d788eff..b0e61a1846c 100644 --- a/.claude/skills/incremental-build/architecture.md +++ b/.claude/skills/incremental-build/architecture.md @@ -37,7 +37,8 @@ Use this table to locate source files. ALWAYS read the relevant source file befo | `ProjectBuilder` | `lib/build/ProjectBuilder.js` | Builds projects in dependency order | | `BuildContext` | `lib/build/helpers/BuildContext.js` | Global build config, project context cache | | `getBuildSignature` | `lib/build/helpers/getBuildSignature.js` | Build signature computation: `BUILD_SIG_VERSION` + build config, combined with aggregated task signatures, project id + config, the `@ui5/project` version, and `@ui5/builder`/`@ui5/fs` versions | -| `ProjectBuildContext` | `lib/build/helpers/ProjectBuildContext.js` | Per-project bridge between builder, tasks, and cache | +| `ProjectBuildContext` | `lib/build/helpers/ProjectBuildContext.js` | Per-project bridge between builder, tasks, and cache. Also hosts `resolveInputValue(type, name)`, the lookup-side counterpart to input recording: it re-derives a recorded non-resource input's current value from `process.env`, the project graph, and the build run's shared timestamp (`getBuildTime()`, see "Non-Resource Task Inputs") (passed into `ProjectBuildCache` so a cache lookup reflects the current environment and graph) | +| `MonitoredTaskUtil` | `lib/build/helpers/MonitoredTaskUtil.js` | Per-task wrapper around the `TaskUtil` (or its spec-version interface) handed to a task, analogous to `MonitoredReader`. A Proxy that preserves the wrapped shape and records the non-resource inputs the task reads (`getEnv`, `isRootProject`, `getDependencies`, and the tracked `getProject(name).get*` accessors); `getInputRecording()` drains the recording for the TaskRunner. It also wraps every `getProject(name).getReader()` result in a `MonitoredReader` and records the resources read through it, split into a project bucket and a dependencies bucket; `getResourceRequests()` drains both for the TaskRunner to merge into the workspace and dependency resource requests. Reads made outside a task (build orchestration holding the raw TaskUtil) are not monitored | | `WatchHandler` | `lib/build/helpers/WatchHandler.js` | Source path watcher (subscribes via the `fileWatcher` facade, so it runs on the native or polling backend); emits `change` events to BuildServer. The watcher coalesces its own events with a 50 ms min / 500 ms max wait, so a continuous operation is delivered as batches up to 500 ms apart. Survives a `git checkout` moving source paths: drops events whose path no longer maps (`getVirtualPath` throws) and skips a path that vanished before `subscribe` resolved. The `ProjectDefinitionWatcher` then re-inits over the new graph | | `ProjectDefinitionWatcher` | `lib/graph/ProjectDefinitionWatcher.js` | Watcher (via the `fileWatcher` facade) for project-definition files (`ui5.yaml` / `--config`, `package.json`, workspace config, static dependency-definition file). Emits `definitionChanging` (leading) and `definitionChanged` (trailing, coalesced) to drive a full serving-stack re-init. A watched definition-file event starts the burst; while the burst is open, every delivered event below the subscribed definition directories resets the settle timer. Project roots below `node_modules` are watched without the `node_modules` ignore so their own definition files remain observable. Owned by `@ui5/server`'s `Supervisor`, not the BuildServer; exported via the internal `@ui5/project/internal/graph/ProjectDefinitionWatcher` subpath, which also re-exports `waitForProjectGraphSettled` and `RecoveryBudget` so the whole live re-resolution feature is reachable through one internal entry point | | `projectGraphSettleWatcher` | `lib/graph/projectGraphSettleWatcher.js` | Short-lived acceptance gate (via the `fileWatcher` facade) for degraded server recovery. Given one or more resolved graphs, watches the union of their project roots without a `node_modules` ignore and resolves once those roots have settled for `WATCHER_BURST_SETTLE_MS`. Missing roots are watched at their nearest existing ancestor so a project that is still being restored is observable. `Supervisor` drives it inside a convergence loop (`#convergeRecoveryGraph`), feeding it each re-resolved graph plus the last-good graph so a root that only the target branch introduces is observed once it surfaces in a resolve | @@ -46,10 +47,11 @@ Use this table to locate source files. ALWAYS read the relevant source file befo | `drainSubscriptions` | `lib/build/helpers/watchSubscriptions.js` | Unsubscribes a list of subscriptions in parallel (`Promise.allSettled`), returns the failures. Used by both watchers' `destroy()` and BuildServer's recovery re-subscribe | | `fileWatcher` | `lib/build/helpers/fileWatcher.js` | Watcher-backend facade. Exposes a `subscribe()` matching `@parcel/watcher`'s contract and picks a backend once per process: `UI5_WATCH_MODE=polling\|native` forces the choice, otherwise it auto-detects containers (`/.dockerenv`, `/run/.containerenv`, PID 1 cgroup) and uses polling there. Also falls back to polling if the native `@parcel/watcher` binding cannot load. `UI5_WATCH_MODE=off` disables watching entirely: `subscribe()` returns an inert subscription (callback never invoked, `unsubscribe` a no-op) so no backend loads, for CI and other environments where sources do not change while the server runs. The memoized decision is a mode string exposed via `shouldUsePolling()` and `isWatchingDisabled()`. All three watchers subscribe through this facade rather than importing `@parcel/watcher` directly | | `pollingWatcher` | `lib/build/helpers/pollingWatcher.js` | Pure-JS polling backend. Walks the tree and diffs an `{mtimeMs, size}` snapshot every 250 ms (`DEFAULT_POLL_INTERVAL_MS`), emitting the same `{type, path}` events as the native backend. Needed on bind-mounted container volumes where inotify misses writes made from outside the container | -| `TaskRunner` | `lib/build/TaskRunner.js` | Task composition, execution loop, abort handling | +| `TaskRunner` | `lib/build/TaskRunner.js` | Task composition, execution loop, abort handling. Hands every task a plain `MonitoredTaskUtil`; for a step-based task it calls the task's step factory once per build at plan time to discover step names for `setTasks`, keeps the returned step array on the task, and reuses it to drive the `StepRunner` with per-stage hooks (`#createStepStageHooks`: `prepareStage`/`getPreviousInvocationData`/`createStageContext`/`recordStage`), and reports the task skipped/executed from the runner's `anyStepExecuted`. A legacy task runs a plain body with no step runner | +| `StepRunner` | `lib/build/helpers/StepRunner.js` | Per-task driver behind the step-factory build API, driving **one pipeline stage per step** (Phase B). Runs scalar and map steps in array order; per step it calls `prepareStage` (switch + cache verdict), runs/restores the step's units against a fresh per-stage recording context and per-step `MonitoredTaskUtil`, threads `needs` returns between steps, folds the stage's own keys' reads/inputs (`#foldStageKeys`, including cached keys) and derives the stage's stale outputs (`#computeStaleOutputs`), then calls `recordStage`. No cross-step fold. See "Step-Based Build Tasks" | | `Cache` enum | `lib/build/cache/Cache.js` | Cache mode constants: `Default`, `Force`, `ReadOnly`, `Off` (CLI `--cache` option) | -| `ProjectBuildCache` | `lib/build/cache/ProjectBuildCache.js` | Cache orchestration per project: index management, stage lookup, result recording | -| `BuildTaskCache` | `lib/build/cache/BuildTaskCache.js` | Per-task resource request tracking and index management | +| `ProjectBuildCache` | `lib/build/cache/ProjectBuildCache.js` | Cache orchestration per project: index management, stage lookup, result recording. The unit of caching is a **stage**: `#stageCaches` is keyed by stage id (a legacy task's `task/{taskName}`, or a step-based task's `task/{taskName}::step/{stepName}` per step); `prepareStageExecutionAndValidateCache(taskName, stepName?)` and `recordStageResult({taskName, …, stepName?})` operate per stage; `setTasks([{taskName, stepNames?}])` creates one stage per step | +| `BuildStageCache` | `lib/build/cache/BuildStageCache.js` | Per-stage resource request tracking and index management (one instance per stage, keyed in `ProjectBuildCache` by stage id). Also holds the stage's `TaskInputSet` (non-resource inputs) and exposes `getInputSignature(resolveValue)` | | `StageCache` | `lib/build/cache/StageCache.js` | In-memory cache of stage results keyed by signature | | `BuildCacheStorage` | `lib/build/cache/BuildCacheStorage.js` | Unified SQLite storage for content (CAS) and metadata | | `CacheManager` | `lib/build/cache/CacheManager.js` | Persistent cache I/O, delegates to BuildCacheStorage; singleton per cache directory. No automatic eviction/GC; `cleanCache()` -> `dropAllRecords()` (a full wipe backing `ui5 cache clean`) is the only cleanup, with the on-disk `VACUUM` deferred to the next run via a pending marker | @@ -57,6 +59,7 @@ Use this table to locate source files. ALWAYS read the relevant source file befo | `ResourceRequestGraph` | `lib/build/cache/ResourceRequestGraph.js` | DAG of request sets with delta encoding and best-parent optimization | | `ResourceIndex` | `lib/build/cache/index/ResourceIndex.js` | Wrapper around hash trees with delta detection | | `HashTree` | `lib/build/cache/index/HashTree.js` | Directory-based Merkle tree for resource hashing | +| `TaskInputSet` | `lib/build/cache/index/TaskInputSet.js` | Flat set of a task's non-resource inputs (env vars, tracked TaskUtil reads), hashed into one signature folded into the task's project-component signature. A sibling of `HashTree` (shares the `version`/`toCacheObject`/`fromCache` conventions) but deliberately not a Merkle tree: inputs are few, unordered, and non-hierarchical. Persists only entry type/name; values are re-read on lookup. Exports `normalizeInputValue`, the shared record/lookup value normalizer | | `SharedHashTree` | `lib/build/cache/index/SharedHashTree.js` | HashTree with structural sharing via TreeRegistry | | `TreeRegistry` | `lib/build/cache/index/TreeRegistry.js` | Batch update coordinator for shared trees | | `TreeNode` | `lib/build/cache/index/TreeNode.js` | Merkle tree node (resource or directory) | @@ -252,7 +255,7 @@ On a failed re-resolve `Supervisor` flags the surviving stack degraded (last-goo | content (CAS: integrity -> gzip-compressed BLOB) | | index_cache (resource index trees, by kind="source") | | stage_metadata (cached stage results, by stage signature) | -| task_metadata (resource requests per task, by type) | +| task_metadata (resource requests per stage, by type) | | result_metadata(per-build result metadata) | +-----------------------------------------------------------------+ ``` @@ -267,8 +270,123 @@ The cache uses content-based signatures at multiple levels: |-------|-----------------|----------------| | **Build signature** | `getBaseSignature()` (`BUILD_SIG_VERSION` + build config), combined by `getProjectSignature()` with the aggregated task signatures, `project.getId()`, project config, the `@ui5/project` version (`getPackageVersion("@ui5/project")`), and the effective `@ui5/builder` / `@ui5/fs` versions (`taskRepository.getVersions()`). Task signatures come from `TaskDefinitions.getBuildSignatures()`, which calls each task's `determineBuildSignature()` (falling back to a hash of the task options/configuration). | `getBuildSignature.js`, `TaskDefinitions.getBuildSignatures()`, `ProjectBuildContext.create()` | | **Source signature** | Merkle root of all source resources | `ResourceIndex` (source index) | -| **Task stage signature** | `projectIndexSignature-dependencyIndexSignature` | `ProjectBuildCache.prepareTaskExecutionAndValidateCache()` | -| **Result signature** | Combined source signature + last task stage signature | `ProjectBuildCache.#getResultStageSignature()` | +| **Stage signature** | An explicit four-component tuple `projectIndexSignature-dependencyIndexSignature-inputSignature-rootSignature`, joined by `-` (each component a SHA-256 hex digest, so the separator never occurs inside a component and the split is lossless). The four dimensions (project resources, dependency resources, non-resource **input signature**, root resources) are independent slots rather than folded into two, so a delta pairs a changed project or dependency signature with the current input and root signatures with no reverse mapping. The join/split primitives live in `cache/stageSignature.js`; `BuildStageCache.getStageSignatures()` composes the exact-match candidates. One stage per step for a step-based task, one per legacy task | `ProjectBuildCache.prepareStageExecutionAndValidateCache()`, `BuildStageCache.getStageSignatures()` | +| **Result signature** | A four-component tuple `sourceSignature-combinedDependencySignature-aggregatedInputSignature-aggregatedRootSignature`. The dependency component is a cartesian product over per-stage candidate dependency signatures; the aggregated input and root signatures are single current values (constants across the product). Both the candidate list and the stored signature derive the per-stage dependency list from the single stage order (`#stageOrder`), so the store and lookup sides cover the same stages in the same order (a declared stage missing a signature throws rather than silently never matching). Grows by one factor per stage, bounded because only dependency-reading stages with a delta contribute a factor >1 | `ProjectBuildCache.#getResultStageSignature()` / `#getPossibleResultStageSignatures()` | + +### Non-Resource Task Inputs + +A task's output can depend on inputs that are not resources: an environment variable, or a value read through the `TaskUtil` interface (`isRootProject()`, `getDependencies()`, a dependency's version via `getProject(name).getVersion()`, `getCustomConfiguration()`, framework getters). None of these feed the resource indices or the build signature, so without tracking, changing one between builds leaves a stale cached result being served. The canonical example: `generateLibraryManifest` embeds a dependency's version as the manifest `minVersion` via `getProject(depName).getVersion()`; removing or bumping that dependency must re-run the task even though no source resource changed. + +Tracking has a record side and a lookup side, mirroring the resource-request flow: + +- **Record** (task executes): the TaskRunner hands the task a `MonitoredTaskUtil` instead of the raw `TaskUtil`. It is a Proxy that preserves the wrapped shape (a custom task's limited interface stays limited) and records every tracked read as `{type, name, value}`, normalizing the value via `normalizeInputValue`. `getProject(name)` returns a wrapped project whose tracked accessors record under the project's name; its `getReader()` result is wrapped in a `MonitoredReader` so the resources the task reads through it are recorded as resource requests, and, for the project being built, its `getRootReader()` result is wrapped too so reads of files outside the UI5 resource model are recorded as root requests (see the reader-monitoring note below), while untracked members (`getRootPath`, `getSpecVersion`, ...) pass straight through unrecorded. After the task, the TaskRunner drains `getInputRecording()` into `ProjectBuildCache.recordStageResult`, which builds a `TaskInputSet` and folds its signature into the task's project-component signature (`combineProjectAndInputSignature`). Only entry type/name are persisted (`task_metadata` type `"input"`), never values. +- **Lookup** (later build): `BuildStageCache.getInputSignature(resolveValue)` recomputes the input signature, re-reading each recorded input's *current* value through `ProjectBuildContext.resolveInputValue(type, name)` (which reaches `process.env` and the current project graph). A value that differs from the one baked into the cached stage signature misses the cache and re-runs the task. Record and lookup normalize through the same `normalizeInputValue`, so equal values compare equal. + +Excluded from input-value tracking: mutations (`setTag`/`clearTag`, already captured as tag operations in the hash trees), constructors (`resourceFactory`), readers (`getReader`/`getRootReader`), side effects (`registerCleanupTask`), and the FS-path accessors (`getRootPath`/`getSourcePath`) whose absolute, machine-specific values would make cache entries non-portable. + +The `time` input (`taskUtil.getTime(granularity)`) is quantized through `quantizeTime(granularity, date)` (`lib/build/helpers/quantizeTime.js`). Both the record side (`TaskUtil.getTime`) and the lookup side (`resolveInputValue`) pass one timestamp fixed per build run: `BuildContext` holds it and `ProjectBuilder` calls `BuildContext.refreshBuildTime()` at the start of each `#build`/`#validate`, reached via `ProjectBuildContext.getBuildTime()`. This keeps every time read within a run consistent (all projects, plus the record and lookup within that run, agree) while still advancing across runs so a rolled-over bucket misses the cache. It cannot be fixed at `BuildContext` construction: a single `BuildContext` is reused for the whole `ui5 serve` lifetime, so a frozen bucket would keep agreeing with itself and silently serve stale output (e.g. last year's copyright). The `date` argument of `quantizeTime` is mandatory to stop a caller from falling back to a fresh `new Date()`. + +`taskUtil.getBuildTime()` returns that same shared per-run timestamp as a raw `Date`, but is deliberately **untracked**: `MonitoredTaskUtil` passes it through without recording an input (it is absent from `TRACKED_TASK_UTIL_METHODS`), so its value never folds into a task's cache signature. This is the opposite of the tracked, quantized `getTime`: a raw timestamp advances every build, so tracking it would miss the cache every time. A cached result keeps the timestamp it embedded rather than re-running. `replaceBuildtime` consumes it (formatting the `${buildtime}` placeholder), so a cached step keeps its previous timestamp until its resource content changes. The interface exposes `getBuildTime` to custom tasks from Specification Version 5.0. + +Reads through a `getProject(name).getReader()` are not input values but resources, so they are tracked on the resource-request side instead: `MonitoredTaskUtil` wraps that reader in a `MonitoredReader` and records the reads into a project bucket (the project being built) or a dependencies bucket (any other project). `getResourceRequests()` drains both, and the TaskRunner merges each bucket into the workspace and dependency resource requests it already collects, so the read resources get content-hashed like any other request. The project being built's `getRootReader()` is wrapped too: its reads (files outside the UI5 resource model, such as a root `tsconfig.json` or packages under `node_modules`) are recorded into a root bucket, split by the `useGitignore` flag because a glob resolves differently with the flag on versus off, and resolved against a dedicated root reader rather than the dependency reader collection. Those root requests use full-refresh signatures, not differential deltas, so a changed root file re-runs the whole stage (the per-file delta tracking the project and dependency paths have is a possible future extension, deferred because the current use case tracks only a few root config files). A *dependency's* `getRootReader()` stays an unwrapped pass-through: it exposes a project root outside the reader collection those requests are later resolved against, so a request registered through it could not be content-hashed. + +Contract for task authors: read an env var through `taskUtil.getEnv(name)` (not `process.env` directly), and read graph-derived values through the `taskUtil`/`getProject` interface, so the monitor observes the read. A direct `process.env` read or a value obtained outside the monitored `taskUtil` is untracked and can serve stale. Conditional reads are handled correctly as long as the branching input is itself read through `taskUtil`: the recorded set changes with the branch, and the branching input's own value invalidates the cache when it changes. + +### Step-Based Build Tasks + +A step-based task default-exports a factory `build(options) => Step[]` instead of a task body. The factory receives `options` only (never readers or `taskUtil`), so it cannot close over build state; every input a step reads arrives through the step's own arguments. This replaced the old differential mode where a task received `changedProjectResourcePaths` and did its own delta bookkeeping. Step-based tasks: `minify` (one map step over resources), `buildThemes` (one map step over themes), the three `replace*` tasks, `escapeNonAsciiCharacters` and `enhanceManifest` (one map step each), and `generateThemeDesignerResources` (two scalar steps plus a themes map step, wired by `needs`). + +**Lazy processor imports.** `TaskRunner` imports a step-based task's module and calls its factory at plan time, before the cache decides whether any step runs (see "Driving one stage per step"). So a step-based task module imports its heavy processors lazily, inside the step bodies (`const p = (await import("../processors/...")).default`), not at module top level. The factory stays cheap: it only returns step descriptors. A build whose steps are all cache hits never runs a step body, so the processor module graphs stay unloaded, in particular the `minify` and `enhanceManifest` and `generateThemeDesignerResources` processors (`minifier`'s worker pool, `manifestEnhancer`'s `semver`, the less generator) and `buildThemes`' `less-openui5` graph. For `buildThemes` the deferral is in `themeBuilderWorker.js`: it imports `themeBuilder` (and `less-openui5`) lazily inside its worker entry `execThemeBuild`, so the main thread importing it for the `FsMainThreadInterface` and (de)serialize helpers, and plan-time discovery, do not evaluate that graph; its light `workerpool` import stays static because the worker registration (`workerpool.worker`) runs at module load. A custom step-based task should follow the same contract. The step bodies are the only place a processor is needed, so a cold build loads each processor once, when its first unit runs. Measured on the library step-based task set (`minify`, `buildThemes`, the three `replace*`, `escapeNonAsciiCharacters`, `enhanceManifest`, `generateThemeDesignerResources`), plan-time module evaluation dropped from ~58 ms to ~40 ms (min-of-5 fresh-process import of all task modules; see performance-investigation.md item 18). + +**One stage per step.** Each step is promoted to its own pipeline stage: a scalar step is a stage; a map step is a single stage carrying an internal per-key delta (keys never become stages — they are discovered at runtime by `keys()` and must stay out of the static stage set and out of the result-stage cartesian product). A step-based task therefore contributes N stages (one per step) rather than one task stage; a legacy task still contributes one stage. Stage ids are `task/{taskName}::step/{stepName}` for a step-based task's steps and `task/{taskName}` for a legacy task (or a step-based task that returns no steps). This unifies steps and stages: per-step caching runs through the same stage-cache, tag-replay, and CAS-return machinery as any stage, rather than a parallel sub-cache folded inside one task stage. The result-stage signature grows by one factor per step, but only dependency-reading steps carrying a delta contribute a factor >1, so the growth is bounded (measured at 1.00× on the realistic framework-library task set, since the scalar steps read no dependencies — see the measurement note below). + +Two step shapes, run in array order by `StepRunner.runSteps()`: +- scalar `{name, needs?, run}` where `run: async ({needs, workspace, dependencies, taskUtil, options}) => value?` runs once. +- map `{name, needs?, sequential?, keys, each}` where `keys: async ({needs, workspace, dependencies, taskUtil, options}) => keySet` enumerates the key set and `each: async (key, {needs, workspace, dependencies, taskUtil, options}) => value?` runs once per key. + +A scalar step is a one-key group (a single implicit unit); a map step is a multi-key group. Both go through the same per-key machinery within their stage, so delta selection, stale-output derivation, and per-stage request/input folding are uniform across shapes. + +**`needs` wiring.** A step lists earlier step names in `needs`; those steps' returns arrive as `needs.` in its context (both `keys` and `each` for a map step). A step may reference only earlier steps, so array order is always a valid execution order. One `needs` object is shared by a step's `keys` enumerator and all of its units, and is frozen (shallowly): a unit assigning to `needs.` would otherwise leak into its siblings and into the `needsInputs` recorded for whichever unit ran next, making a delta build's per-unit selection depend on execution order. `@ui5/builder`'s standalone `runSteps` freezes it too, so a step behaves the same under both runners. Because steps share `needs` state in memory, `StepRunner` remains the per-task driver that runs all of a task's steps in order — it just drives one stage per step (see below) instead of folding them into one. + +**Declaration validation.** The whole step list is validated **before any stage is created**, by `validateSteps(steps, {taskName?})` (exported from `StepRunner.js`). `TaskRunner` calls it at discovery, right after the factory returns and before it derives the step names for `setTasks`: a stage id is `task/{taskName}::step/{stepName}`, so a duplicate step name would create two stages sharing one id — the error must be raised before `setTasks`, not when the step later runs. `StepRunner.runSteps()` calls it again at its entry, so a directly or standalone instantiated runner validates identically. The rules are static over the descriptors (no reader, taskUtil, options or cache state): the factory must return an array; each element must be an object with a non-empty string `name`; names must be unique; a step is either scalar (`run`) or map (`keys` **and** `each`), never both nor neither (a half-defined map names the missing half); `needs`, when present, must be an array of strings naming only **earlier** steps (so a forward or self reference throws, which is what makes a `needs` cycle impossible). Scope is `@ui5/project`; `@ui5/builder`'s standalone `runSteps` does not yet share this validator. + +**Authoring a step-based task (the DSL).** A task module default-exports the `build(options)` factory and returns the step array. The reference shape is `generateThemeDesignerResources` (`packages/builder/lib/tasks/generateThemeDesignerResources.js`), a scalar producer feeding a scalar consumer and a map step: + +```js +export default function build(options) { + const {version} = options; + const namespace = options.projectNamespace; + if (namespace === "sap/ui/documentation") { + return []; // not offered in Theme Designer, so the factory emits no steps + } + const pattern = namespace ? + `/resources/${namespace}/themes/*/library.source.less` : + `/resources/**/themes/*/library.source.less`; + + const steps = [{ + name: "scan", + run: async ({workspace}) => ({hasThemes: (await workspace.byGlob(pattern)).length > 0}), + }]; + + if (namespace) { // only a library (which has a namespace) gets a library .theming file + steps.push({ + name: "libraryTheming", + needs: ["scan"], + run: async ({needs, workspace}) => { + // write /resources//.theming from needs.scan.hasThemes, namespace, version + }, + }); + } + + steps.push({ + name: "themes", + needs: ["scan"], + keys: async ({needs, workspace}) => needs.scan.hasThemes ? workspace.byGlob(pattern) : [], + each: async (librarySourceLess, {workspace, dependencies}) => { + // build one theme's .theming and library.less + }, + }); + + return steps; +} +``` + +Two DSL rules make the per-step recording honest, and both are visible here. The factory is pure over `options`: it may branch on `namespace`, precompute `pattern`, and include or omit steps, all from values that are part of the build signature. And no `run`/`keys`/`each` body closes over a reader or `taskUtil`; every input arrives as a callback argument (`workspace`, `dependencies`, `taskUtil`, `needs`, `options`), so a read cannot escape the step it belongs to. A matching contract holds for writes: every write a unit makes goes through its own `workspace`, the only writable handle a step receives (the factory gets `options` only, and `taskUtil.getProject().getReader()` is read-only), so a unit's recorded `writes` are the complete set `#computeStaleOutputs` needs to drop a path the unit stops producing. A custom step that reaches a writer another way leaves that write unrecorded, and its stale output would survive a delta build. + +**How these steps become stages.** The factory above produces up to three stages, one per returned step: `task/generateThemeDesignerResources::step/scan`, `.../step/libraryTheming` (present only when `namespace` is set, so the stage set follows the factory's branching and is known before execution), and `.../step/themes` (one stage carrying the per-theme key delta, since keys are discovered at runtime and never become stages). `libraryTheming` is keyed on `scan`'s return through `needs`, so it stays cached while `hasThemes` holds even as individual `library.source.less` files change; the `themes` map step regenerates only the themes whose keys changed. `TaskRunner` calls the factory once per build at plan time (pure over `options`) to collect the step names for `setTasks`, so `ProjectBuildCache` creates these stages in step order before the build runs. It keeps the returned step array on the task and reuses it when the step runs, so the factory is not called a second time. + +**Returns.** A step may return resources (stored in the CAS by integrity, as before) or a JSON-serializable value (persisted inline with the unit's invocation data). Either is injected into a consumer via `needs.`, and the return's signature (resource integrities, or the serialized value) folds into the consumer's per-unit selection so a changed producer return re-runs the consumer. A map producer's return signature is order-independent: it hashes a `keyId -> signature` map sorted by `keyId`, not a positional list in `keys()` order, so a key reordering that changes nothing semantically (an adapter change, filesystem ordering, a reader-collection reshuffle) does not move the signature and re-run every consumer for nothing. A step returning nothing has a `null` return descriptor. A returned value that is neither resources nor JSON-serializable throws. + +**The `stepBased` opt-in.** A task is step-based only when it declares it; absent, the default export is a legacy task body and runs unchanged. A standard task sets `stepBased: true` in its build-definition entry (`definitions/*.js`), read by `_addTask`. A custom task declares a static `stepBased` export on its task module, surfaced by `Task#getStepBased` and honored from Specification Version 5.0 (`_addCustomTask` computes `specVersion.gte("5.0") && (await getStepBased()) === true`); a `stepBased` export below 5.0 is ignored and the task runs as a legacy body. There is no shape-sniffing: both legacy and factory are default-export functions, disambiguated by the flag. + +**Driving one stage per step.** Before the build, `TaskRunner.runTasks` calls each step-based task's factory once (pure over `options`) to discover its step names and hands `ProjectBuildCache.setTasks` a `[{taskName, stepNames?}]` list, which creates one stage per step in step order. The returned step array is kept on the task (frozen, so a custom task cannot mutate the shared value) and reused when the step runs, so the factory is called once per build rather than once to discover and once to execute. One call is the single source of truth: discovery and execution cannot drive different stages, so no runtime check is needed to reconcile two factory calls. The `options` object is completed at registration time (`_addTask` fills in `projectName`, `projectNamespace`) and is fixed for a `TaskRunner`'s lifetime: a changed `ui5.yaml` task configuration rebuilds the whole serving stack through `Supervisor.reinitialize`, giving a fresh `TaskRunner`, while a plain source-change rebuild reuses the `TaskRunner` and re-derives the step array. Re-deriving per build keeps the steps in step with the options without any explicit invalidation. A factory body that reads the environment or the clock untracked is still impure, but a single call means its one output drives both the stage list and execution consistently. For each step, the `StepRunner` drives that step's own stage via hooks the `TaskRunner` supplies (`#createStepStageHooks`): `prepareStage(stepName)` switches the project to the step's stage and returns its cache verdict (`prepareStageExecutionAndValidateCache(taskName, stepName)`); `createStageContext()` builds fresh monitored `workspace`/`dependencies` readers and a `MonitoredTaskUtil` bound to that stage (so the reads reflect the cumulative output of all earlier stages with this stage's writer on top); `recordStage(stepName, …)` records the step's stage (`recordStageResult({taskName, …, stepName})`). A fully-cached stage (`prepareStage` returns `true`) does not run: its return is rebuilt from the persisted per-key data (in key order) so later steps' `needs` resolve, and its tag operations are replayed. A `needs` return is excluded from the stage signature, so a full-hit consumer whose producer re-ran with a changed return would otherwise serve stale output. The `StepRunner` checks this on the full-hit path (`#needsReturnChanged`) and, when it fires, reopens the stage with a fresh writer and re-runs it as a full execution (`reopenStage` returns a falsy verdict, not a delta). Reopen installs a fresh empty writable stage, so pruning the re-run to only the affected units would need re-seeding it with the cached outputs first, so the whole stage re-runs instead. No shipped multi-step task pays for this: only `generateThemeDesignerResources` wires `needs`, and a changed `scan` return regenerates its themes anyway. The verdict (`true` / delta object / falsy) is read three ways in `runSteps`. An explicit verdict shape would read more clearly but is deferred, since that shape is shared with the stage-signature code. A task is reported to the `ProjectBuildLogger` as skipped (`skipTask`) when every step was served from cache, else as executed. Its skip verdict is only known once every stage has been driven, so it cannot be announced up front like a legacy task: the `StepRunner` instead calls `notifyStepExecution` from the first stage that stops being a pure cache hit, before that stage runs anything, and `TaskRunner.#createTaskExecutionReport` turns that into `startTask`. `task-start` therefore precedes the work it announces, and the matching `endTask` (with the union of the stages' written paths) follows it, so a `project-build-status` consumer sees the task running while it runs. A step-based task that returns **no** steps still has one stage (`task/{taskName}`) that is prepared and recorded (empty), so its empty result caches and a later build reports it skipped. + +**Standalone (uncached) runner.** `@ui5/builder` exposes `runSteps(factory, {workspace, dependencies, taskUtil, options})` (`lib/tasks/runSteps.js`) for standalone task invocation with no build cache: it runs every step in order, fans out each `keys` set, threads `needs` in memory, and buffers map writes in key order, without delta selection, persistence, or tag replay. It is an independent implementation (it does not use the `@ui5/project` `StepRunner`); the full cached engine stays in `@ui5/project` because `@ui5/builder` cannot import it. The two runners stay separate but share one definition of the write-buffer contract: the same-path guard (with its user-visible error message) and the key-order flush live in `@ui5/fs/internal/stepWriteBuffer` (`assertDistinctWrite`, `flushWriteBuffer`), reached by both because both already depend on `@ui5/fs`. The two writers differ only in which method they override: `@ui5/project`'s `RecordingReaderWriter` overrides `_write` (it also records writes for the cache, and the base `write` has already defaulted options), while `@ui5/builder`'s `BufferedWriter` overrides the public `write` to preserve the caller's exact arguments; both buffer `{resource, args, index}` entries and flush them identically. + +Per unit within a stage, the callback receives its own recording readers (`{workspace, dependencies}`) that delegate to the stage's monitored readers and additionally attribute each read to the unit, plus a per-unit `MonitoredTaskUtil` wrapping the stage's one. The per-unit monitor attributes the unit's non-resource inputs (`getEnv`, `getTime`, `getProject(name).getVersion()`, `isRootProject`, `getDependencies`) and its `getTag`/`setTag`/`clearTag` operations to the unit, while reads still delegate through the stage monitor so a full build's stage-level recording stays the union that keys the stage. Key identity (`StepRunner.#keyId`) is content-based: a resource key by its path and a content discriminator (path distinguishes resources that share content but produce different output; the discriminator makes a content change a new key, so the unit re-runs and its previous output is dropped rather than served stale), a string key by its value. Compound keys are the caller's responsibility to express as a stable string. The discriminator is **tiered like `isResourceUnchanged`** (see §"HashTree") rather than always the SSRI integrity: `lastModified` + `size` when both are statically available (`getLastModified()` returns a number and `hasSize()` is true, so no content read), falling back to the integrity when either is missing — a memory-backed or generated resource with no `lastModified`, or one whose size is not statically known. A resource restored from a stage cache carries its `integrity`, so the fallback `getIntegrity()` resolves without reading content and that path stays cheap. This avoids hashing every enumerated key's full content before delta selection; it is why `#resolveEntries` is cheap for a broad `workspace.byGlob` step (`replaceCopyright`, `replaceVersion`) on a stale-cache build (see performance-investigation.md §12). The residual staleness risk is identical to `isResourceUnchanged`'s and accepted for the same reason: a content change that preserves **both** `lastModified` and `size` keeps the same key and does not re-run the unit — a real edit moves mtime, so the gap is only for mtime-preserving replacements (`cp -p`, `tar -x`, atomic rename) that also hold size constant, and the changed-path delta does not independently cover this case because it is derived through the same tiered comparison. Because the stat tier and the integrity tier use distinct prefixes (`\0m…\0s…` vs `\0i…`), a key never aliases across the two tiers. + +> This is a signature-shape change to the per-key invocation data keys, not a persisted-cache-format change requiring a `CACHE_VERSION`/`BUILD_SIG_VERSION` bump: the per-key invocation data is re-derived each build and a changed key identity simply looks like a new key (the unit re-runs), which is the correct and safe outcome. The feature is unreleased, so no backward-compat shim is added. + +Execution and writes: +- `sequential: true` (always set for a scalar step) persists each unit's writes immediately, so a later unit reads what an earlier one wrote. +- The default (`sequential: false`) runs a map step's keys concurrently: writes are buffered and flushed in key order after all keys finish; two concurrent keys writing the same path throw, since concurrent keys must be independent. +- Steps run in array order, and the abort signal is checked between units; a single in-flight unit is not interrupted. + +Delta selection and cache integration (per stage): +- On a delta build, a unit re-runs when its key is new, its recorded reads intersect the changed project/dependency paths (the reverse mapping that re-runs the owner of a changed cross-resource input, e.g. a `.js` whose `.js.map` changed, or a theme whose gating marker was added), one of its recorded non-resource inputs no longer resolves to its stored value (an env var flipped, a dependency version bumped, or a time bucket rolled over between runs, since the bucket derives from the run's shared `BuildContext` timestamp), or a `needs` return it consumed changed. The input is re-derived through `ProjectBuildCache.getResolveInputValue()`, the same resolver the stage-level input lookup uses; without a resolver (standalone use) a unit is selected on its resource reads alone. A scalar step's single implicit unit is the exception: it re-runs whenever its stage takes the delta path (not a full cache hit), without consulting the per-unit reads delta. A scalar step is one unit, so there is nothing finer to prune, and the reads delta cannot catch a file newly matching a glob the step evaluated: the recorder stores resolved paths, not patterns (`#foldStageKeys`), so a file that did not exist on the previous build is in no recorded read, yet the stage-level monitor did record the glob, so an added match moves the stage signature and `prepareStage` returns a delta verdict. Re-running the one unit on that verdict is what makes `generateThemeDesignerResources`'s `scan` step observe a first theme being added (map steps are already covered: `keys()` re-enumerates and a new key has no previous entry). A full stage-cache hit still keeps the scalar step cached. +- `StepRunner.#computeStaleOutputs` reports paths a unit produced before but no longer produces (a re-run unit that writes fewer paths, or a removed key). The dropped-path comparison runs over the units that actually executed this build, while the set of paths still claimed is taken from the stage's complete per-key data, so a path a cached unit owns is never dropped because a sibling key disappeared. It is scoped to the one stage (a stage owns its outputs), and the `recordStage` hook appends the stale paths to the stage's delta `changedProjectResourcePaths` so `recordStageResult`'s stage merge drops them. +- A map step's `keys` enumerator owns no key, so it runs against the stage's own context rather than a per-unit recording one: it runs whenever the stage runs, so its reads and non-resource inputs are captured by the stage's monitored readers and `MonitoredTaskUtil` and fold into the stage signature, and the tags it sets are recorded as the stage's tag operations, which a fully cached stage restores along with its writer. +- `StepRunner.#foldStageKeys` folds every one of a stage's keys' reads and non-resource inputs — including keys served from cache on a delta build, whose reads and inputs the stage-level monitor never observed — into one request set and one input set, which the `recordStage` hook merges into the stage-level monitored requests before `recordStageResult` re-keys the stage. The recorder stores resolved paths and a path is commonly read by more than one key, so `#foldStageKeys` accumulates into `Set`s (deduped per read bucket) and the `recordStage` hook's `foldReadsInto` adds only the fold paths the stage monitor did not already request; the duplicates would otherwise collapse downstream in `ResourceRequestGraph.findExactMatch` (which keys on a `Set`), but carrying them inflated the recording that graph rebuilds. Deduplication does not move the stage signature — the set of reads is unchanged, only its representation. This is the map step's internal key-delta fold (kept in Phase B); there is no cross-**step** fold — each step's stage records only its own keys. The consumed `needs` returns are deliberately excluded from this fold (tracked in the separate per-key `needsInputs` field, see below): they drive per-unit selection only and are re-derived from producer reads/inputs that are themselves tracked, so `resolveInputValue` has no resolver for them and folding one into the stage signature would permanently miss the stage cache. Omitting this per-key fold makes a stage whose keys read `taskUtil.getTime` (`replaceCopyright`, `replaceBuildtime`) fail to reconverge to a cached signature across builds, because a delta build that restores every key records no inputs at the stage level. +- A unit served from cache replays its recorded `set`/`clear` tag operations via `ProjectResources.replayTagOperations`, routed by tag to the monitored project or build tag collection and applied by path, so its tags reappear in this build's tag operations (captured by `recordStageResult` like a unit that ran). A unit whose key is gone is not replayed, so a removed unit's tags do not linger. `get` operations carry no persistent effect and are skipped. +- Per-key invocation data (`Map`) is persisted **inside the step's own `stage_metadata` row** (`metadata.stepInvocationData`, as `[[keyId, entry], ...]` pairs since JSON has no Map), keyed by the stage signature the stage output is keyed under, so the per-key map and the stage output can never pair with a different run's data. It is embedded by `ProjectBuildCache.#prepareStageCache` and extracted by `#processStageCacheMetadata`; a cache lookup (`prepareStageExecutionAndValidateCache`) stashes the matched stage's map into `#stepInvocationData` so `getStepInvocationData(stageId)` returns the signature-matched "previous" data, and a running stage overwrites it via `setStepInvocationData`. `needsInputs` holds the consumed `needs` return signatures for per-unit selection and is intentionally kept out of the stage input fold. Because a `needs` return is excluded from the stage signature, a cached consumer's correctness rests on two checks rather than the signature: `#needsReturnChanged` on the full-hit path and the `needsInputs` comparison in `#selectStepsToRun` on the delta path. These are the only two paths that reach a cached stage through the `StepRunner`. A whole project restored from the project-level result cache skips the `StepRunner` entirely (and `ProjectBuildCache.#importStages` installs its stages blindly by signature), yet that is safe without a needs check: the result signature aggregates every stage's inputs, producer stages included, so any change that could alter a producer's return perturbs the result signature, misses the result cache, and defers to the `StepRunner` where the two checks run. The map rides on the stage's own `stage_metadata` write, which happens only for a stage recorded this build, so a full cache hit (not re-recorded) never rewrites it; a map step whose key set dropped to zero re-records under a new signature with an embedded `[]`, so the next build that matches it sees no stale keys. +- Return values are first-class: `StepRunner` stores a resource return's content in the CAS via `ProjectBuildCache.getStepReturnValueStore()` (`store` buffers the compressed content deduped against stage rows; `flush`, called by the driver once per step after its units have returned, writes that step's buffer in a single transaction) and records per-unit descriptors in the `returns` field of the invocation entry; a serializable value is recorded inline. On a delta build a unit served from cache has its return rebuilt from the CAS (or read back from the inline value), so a map step's result array is reassembled in key order from a mix of freshly-returned and restored entries and handed to consumers via `needs`. A returned resource whose path collides with a written output is stored once by integrity and rebuilt independently of that output. + +Removing an input resource yields a per-unit delta: `ResourceRequestManager.getDeltas` includes removed paths in the delta's `changedPaths`, so a unit that read the removed input re-runs (or, for a gone key, drops out) and its stale output is dropped via the stale-output merge above, while the other units stay cached. Known gap (parked): a processor-library version (e.g. terser, less-openui5) is not yet a tracked per-unit input, so a task whose output depends on a processor version can serve stale until a follow-up routes that version through the per-unit `taskUtil` (open-gaps §1). + +**Result-stage-signature growth (Phase B measurement).** The result-stage signature is a cartesian product over per-**stage** candidate dependency signatures (`ProjectBuildCache.#getPossibleResultStageSignatures`). Promoting each scalar step to its own stage adds a factor per stage, but a stage contributes a factor >1 only when it read dependencies AND carries a dependency delta. In the realistic framework-library task set (`minify` + `buildThemes` + `generateThemeDesignerResources`), the scalar steps (`scan`, `libraryTheming`) read no dependencies, so they add factor-1 stages; the dependency reads stay with the single `themes`/`buildThemes` map stage that owns them. Measured growth under a broad dependency change: **1.00×** (both a real-build probe over the integration corpus and a forced worst-case harness with two dependency-reading stages each carrying a 2-node delta — current product 4, projected product 4). A dependency-reading scalar step WOULD multiply the product; the realistic set avoids it. + +**Stage-count cost, and why single-step tasks are not collapsed.** The design keeps one stage per step. A single-step task already is exactly one stage, so the only step-based task that adds stages is `generateThemeDesignerResources` (three steps), and it is not in the default library task set: `ui5 build` and even `ui5 build --all` across sap.m, sap.ui.core, sap.ui.layout and sap.ui.unified run **zero** multi-step tasks. A library therefore builds with step count equal to task count, the same number of stages as the pre-redesign per-task model, and the stage growth the one-stage-per-step design could cause only appears for a build that runs `generateThemeDesignerResources`. Collapsing a single-step task to one `task/{taskName}` stage is a pure stage-id rename: same stage count, same `ReaderCollectionPrioritized` depth, same result-stage cartesian product, still driven by `StepRunner` with full per-key recording, so it buys nothing measurable. Any warm or stale build-time delta against the per-task model is not a stage-count effect (per-stage `updateProjectIndices` cost is unchanged); it comes from the per-key map-step machinery on a delta build and from startup and module loading on a warm build. Measured numbers live in the benchmark results repository, not here, since they drift with the code. + ### Source File CAS Storage (Frozen Sources) @@ -293,15 +411,15 @@ validateCache({prepareForBuild: true}) -> State is INITIAL -> return false (no cache to validate) For each task: - prepareTaskExecutionAndValidateCache(taskName) - -> No task cache exists (#taskCache empty) + prepareStageExecutionAndValidateCache(taskName) + -> No task cache exists (#stageCaches empty) -> return false (task must execute) [task executes] - recordTaskResult(taskName, workspace, dependencies, cacheInfo) + recordStageResult(taskName, workspace, dependencies, cacheInfo) -> Records resource requests -> creates hash trees (ResourceRequestManager.addRequests) - -> Creates BuildTaskCache with request patterns and indices + -> Creates BuildStageCache with request patterns and indices -> Reads stage writer for produced resources -> Gets tag operations from MonitoredResourceTagCollection -> Computes stage signature @@ -331,8 +449,8 @@ validateCache({prepareForBuild: true}) -> If result cache valid: return true (skip entire project build) For each task: - prepareTaskExecutionAndValidateCache(taskName) - -> Task cache EXISTS (from previous build's recordTaskResult) + prepareStageExecutionAndValidateCache(taskName) + -> Task cache EXISTS (from previous build's recordStageResult) -> updateProjectIndices(reader, writtenResultResourcePaths) -> ResourceRequestManager.updateIndices(): Match changed paths against request graph @@ -395,10 +513,10 @@ Manages the request graph for a task -- delegates to `ResourceRequestGraph` for At runtime, each materialized request set references a `SharedHashTree` representing the resources currently matching that set. -- `addRequests(recording, reader)`: Records path/glob requests, creates or reuses a request set in the graph, builds a resource index (SharedHashTree), returns signature +- `addRequests(recording, reader)`: Records path/glob requests, creates or reuses a request set in the graph, builds a resource index (SharedHashTree), returns signature. Reusing an existing set (an exact `findExactMatch` hit) is a no-op for persistence and leaves `hasNewOrModifiedCacheEntries()` untouched, so a stage that records the same request set every delta build (the common case for a step-based stage) does not force the whole request graph to be re-serialized; only creating a new set, or a tree update in `updateIndices` that moves a signature, marks the manager dirty - `updateIndices(reader, changedPaths)`: Traverses graph breadth-first, matches changed paths against request patterns per node, batch-fetches resources, upserts into affected resource indices via TreeRegistry - `getIndexSignatures()`: Returns current signatures for all request sets -- `getDeltas()`: Returns map of original -> new signature for changed request sets +- `getDeltas()`: Returns map of original -> new signature for changed request sets. Both ends are the node's exposed (composite) signature (tree hash folded with any unresolved-request keys, matching `getIndexSignatures`), so a delta on a task that probed an absent path keys on the same signature the stage was stored under. A delta is not emitted once a resource is removed from a set (the shared path cannot express a removal), so a removal falls back to a full re-execution ## Resource Tags @@ -450,7 +568,7 @@ This ensures that tag-only changes (e.g., a resource gaining `IsDebugVariant` af ## Stage Pipeline -Each task has its own **stage** with a writer. Resources written by a task go into that stage's writer. +Each stage has its own writer; resources written during that stage go into it. A legacy task is one stage; a step-based task is one stage per step (Phase B). The pipeline itself is stage-id-agnostic (`ProjectResources.initStages`/`useStage`/`setStage` take arbitrary ordered stage ids); the stage set and naming come from `ProjectBuildCache` (`task/{taskName}` or `task/{taskName}::step/{stepName}`). More stages means a deeper prioritized reader stack, so a later step reads the cumulative output of all earlier steps (and earlier tasks) exactly as a later task read all earlier tasks before. ### Reader Construction @@ -470,6 +588,8 @@ project.getProjectResources().setStage(stageName, stageCache.stage, stageCache.projectTagOperations, stageCache.buildTagOperations); ``` +`ProjectBuildCache` keeps the stage ids `setTasks` created in execution order (`#stageOrder`), which the dependency component of a stage signature is composed over. Looking up a stage reads directly in `#findStageCache`: it checks the in-memory `StageCache` first, then reads the matching row from SQLite. There is no lookahead prefetch. A one-stage-ahead prefetch existed on this branch but was removed: every `CacheManager` read is synchronous better-sqlite3, so moving the read earlier cannot overlap the current stage's execution, and the prefetched signatures were computed before the next stage's `updateProjectIndices`, so on a delta build they did not match the signatures the lookup then asked for. Measured on `sap.m` it never helped (see `performance-investigation.md`). + ## Persistent Cache Format ### On Disk (CacheManager) @@ -481,7 +601,7 @@ project.getProjectResources().setStage(stageName, stageCache.stage, - content(integrity TEXT PK, data BLOB) # CAS: gzip above ~128 bytes - index_cache(project_id, build_signature, kind, data) # kind: "source" - stage_metadata(project_id, build_signature, stage_id, stage_signature, data) - - task_metadata(project_id, build_signature, task_name, type, data) # type: "project" | "dependencies" + - task_metadata(project_id, build_signature, stage_id, type, data) # stage_id is the STAGE id (task/{taskName} or task/{taskName}::step/{stepName}); type: "project" | "dependencies" | "input" | "root" | "root-no-gitignore" - result_metadata(project_id, build_signature, stage_signature, data) ``` @@ -492,7 +612,7 @@ Note: Both CAS content and metadata BLOBs are gzip-compressed via thresholds (`C The index cache (one row per `(project_id, build_signature, kind="source")`) contains: - `indexTimestamp`: creation timestamp (used for racy-git detection) - `root`: serialized Merkle tree (TreeNode hierarchy) -- `tasks`: array of `[taskName, supportsDifferentialBuilds ? 1 : 0]` recording the task execution order and differential build capability +- `tasks`: array of `[stageId, stepBased ? 1 : 0]` recording the stage execution order (one entry per stage: a legacy task's `task/{taskName}`, or a step-based task's per-step `task/{taskName}::step/{stepName}`) and whether the stage ran the step runner (step-based), which drives delta tracking. The stage id is the `task_metadata` key everything else for that stage is stored under #### Stage Metadata Format @@ -500,6 +620,7 @@ Stage metadata stored on disk includes: - `resourceMetadata`: resource paths mapped to `{integrity, lastModified, size, inode}` - `resourceMapping` (optional, for WriterCollection stages): virtual path prefixes mapped to indices in the `resourceMetadata` array, supporting project types where multiple virtual paths map to the same physical path - `projectTagOperations` / `buildTagOperations`: tag operations to apply when restoring the cached stage +- `stepInvocationData` (optional, step-based stages only): the step's per-key invocation map as `[[keyId, entry], ...]` pairs, embedded here (rather than in a separate row) so it is keyed by the same stage signature as the output and the two can never pair with a different run's data ## Key Architectural Patterns @@ -508,7 +629,7 @@ Stage metadata stored on disk includes: 3. **Abort/retry**: File changes abort running builds; projects re-queued automatically 4. **Structural sharing**: Derived hash trees share unchanged subtrees, reducing memory 5. **Content-addressed storage**: Resources deduplicated via integrity hashes in custom CAS (synchronous path resolution, gzip-compressed) -6. **Differential caching**: Tasks track resource requests; delta builds only re-process changed resources +6. **Differential caching**: Stages track resource requests; delta builds only re-process changed resources. A task participates by being step-based (the `stepBased` flag, see "Step-Based Build Tasks"), which promotes each step to its own stage; this replaced the older `changedProjectResourcePaths` parameter 7. **Tag propagation**: Resource tags flow through stages via cached tag operations, included in hash signatures 8. **Two-tier cache**: Fast in-memory StageCache + persistent filesystem cache via CacheManager 9. **Two-phase invalidation**: Changes queued via `projectSourcesChanged()` / `dependencyResourcesChanged()` (state -> `REQUIRES_UPDATE`), applied only during `#flushPendingChanges()` at next build start. "Definitely invalidated" only after content comparison confirms actual differences. diff --git a/.claude/skills/incremental-build/performance-investigation.md b/.claude/skills/incremental-build/performance-investigation.md index 4b122e8dde1..d1f91f183c2 100644 --- a/.claude/skills/incremental-build/performance-investigation.md +++ b/.claude/skills/incremental-build/performance-investigation.md @@ -171,7 +171,7 @@ info › Running task generateLibraryPreload... ← Full re-execution ``` - `✔ Skipping` — exact cache match for this task's signature. -- `◇ Running` — differential execution (using `changedProjectResourcePaths`). +- `◇ Running` — a step-based task where at least one step re-ran (`StepRunner` selects the changed steps/keys), or a legacy task running a delta (`cacheInfo`). - `› Running` — full execution (no cache match, no delta available). After task execution, `recordTaskResult` runs (logged per task): @@ -352,6 +352,124 @@ When cache writes are deferred (CLI mode), `#writeTaskStageCache` + `#writeSourc When diagnosing slow `writeStageResources`, check the `CAS skipped` vs `CAS written` counts in the log. If most resources are being written (not skipped), `#knownCasIntegrities` is not being populated from one of these sources — trace which source is missing for the scenario. +### 11. Do not parallelize `isResourceUnchanged` with `Promise.all` + +`HashTree.upsertResources` and `TreeRegistry.flush` call `isResourceUnchanged` (`utils.js`) per resource, which checks `lastModified`/`size` (sync) first and only reads and hashes the file (`getIntegrity()`) when that fast path fails (see the tiered comparison in `architecture.md`). On an incremental build most resources are unchanged, so the common path is synchronous. Wrapping these calls in `Promise.all` either forces `getIntegrity()` for every resource (a regression) or adds promise overhead to hundreds of synchronously-resolving checks (no gain). Initial builds already parallelize through `createResourceIndex`. Before parallelizing I/O here, confirm the short-circuit does not already make the common path synchronous, and benchmark before and after. + +### 12. `StepRunner.#keyId` must not force the content hash per enumerated key + +`StepRunner.#resolveEntries` computes a key identity (`#keyId`) for every key a map step enumerates, via `Promise.all`, **before** delta selection runs. If `#keyId` derives that identity from the SSRI integrity, it reads and hashes every enumerated key's full content on every build — the same force-the-hash trap as §11, but one tier up (per enumerated key rather than per index resource). This bites hardest on a step whose keys come from a broad `workspace.byGlob` resolving straight to the project source reader (`replaceCopyright`, `replaceVersion`), where the key set is the whole source tree and the resources carry `lastModified`+`size` from statInfo but no `#integrity` (the `FileSystem` adapter never sets it), so `getIntegrity()` reads the file. A step whose keys come from a stage cache (`minify`) is already cheap because those resources carry `integrity`. + +`#keyId` therefore tiers the identity like `isResourceUnchanged`: `lastModified`+`size` when both are statically available (`getLastModified()` is a number and `hasSize()` is true, no content read), integrity only as the fallback (memory/generated resources with no `lastModified`, or no static size; and stage-cache resources, whose `getIntegrity()` is cached and so stays cheap). See the key-identity paragraph under "Step-Based Build Tasks" in `architecture.md` for the correctness argument and the residual mtime+size-preserving risk. + +**Measured (2026-10, stale-cache sap.m, one source file edited, instrumenting `#resolveEntries` per step):** + +| Step | keys | integrity tier (before) | lastModified+size tier (after) | +|------|------|-------------------------|--------------------------------| +| `replaceCopyright` | 4411 | ~210–375 ms | ~4 ms | +| `replaceVersion` | 5292 | ~196–227 ms | ~2 ms | +| `minify` | 737 | ~0.5 ms (already cheap, stage-cache keys carry integrity) | ~0.2 ms | + +`#resolveEntries` across the two broad-glob steps dropped from ~420–600 ms (first-run fs-cache-cold spike at the top of the range) to ~6 ms — the per-key `ssri.fromData` disappears from the stale-cache critical path. Total stale-cache sap.m build time: ~3.2 s → ~2.4 s. Warm cache is unaffected (the result cache is valid, so `#resolveEntries` is never reached), and cold cache is unaffected (integrity is already computed during indexing). + +### 13. The delta-build step fold is already cheap; dedup is hygiene, not a speedup + +The step fold on the delta path (`StepRunner.#foldStageKeys` → `recordStage`'s `foldReadsInto` → `ProjectBuildCache.#foldStepReads` → `BuildStageCache.recordRequests` → `ResourceRequestManager.addRequests`) was reviewed as a suspected delta-path cost: a `minify` fold of ~4,400 duplicated per-key paths, and a `findExactMatch` iterating every node and re-resolving every path on a miss. **Measured at HEAD (2026-10, stale-cache sap.m), that cost does not materialize**, because an earlier branch commit stopped the broad map steps from recording per-key reads: `minify`/`replaceCopyright`/`replaceVersion` operate on the key resource `keys()` hands them, not via `workspace.byPath`, so each key records **zero** reads and the fold is empty. The only standard step that folds real per-key reads is `buildThemes` (its `each` reads dependencies per theme), and only when a theme source changes. + +Instrumenting `#foldStageKeys` (raw vs unique paths), `#foldStepReads` (per-stage timing), `findExactMatch` (calls, request keys built, reuse vs miss) and `#prepareStageRequestCache` (dirty vs clean stages, spurious reuse-dirty flags): + +| Scenario (one file edited) | fold per stage | `findExactMatch` | request keys built | stages re-serialized | +|---|---|---|---|---| +| JS file (`Button.js`) | ~0.1 ms, 0 fold paths | 5 calls, 5 iterations, **0 misses** | 7 → 7 (no change) | unchanged | +| Theme file (`base/Bar.less`, drives `buildThemes`) | ~0.4 ms, 2 dup / 443 paths | 5 calls, 5 iterations, **0 misses** | **647 → 454** | **1 fewer** | + +Findings: +- **Every `findExactMatch` is a reused hit (0 misses).** The feared miss path (`#getResourcesForRequests` resolving each recorded path through the reader stack and rebuilding the hash tree) is never reached on this corpus, so there is no path-resolution cost to shrink today. Dedup only shrinks the input *to* a miss, which does not occur here. +- **Dedup win is request-key string building only.** `#foldStageKeys` deduping into `Set`s plus `foldReadsInto` dropping fold paths the stage monitor already requested cut the keys `findExactMatch` rebuilds from 647 to 454 on a theme change (`buildThemes`' ~2 duplicate reads plus the per-key paths double-counted against the stage monitor). This is microseconds; it does not move wall-clock on a ~2.4 s build. +- **The dirty-flag fix (W5) is the one with latent value.** `ResourceRequestManager.#addRequestSet` previously flagged the manager dirty on *every* call, so a request set reused byte-identical still forced the whole request graph + resource indices to be re-serialized. The common case for a step-based stage is recording the same set every delta build; measured `spuriousReuseDirty=1` on the theme scenario (one stage's request cache needlessly rewritten), 0 after the fix. In CLI mode this is a deferred background write off the critical path; it matters more in **BuildServer mode, where cache writes are awaited** (Phase 4), so cutting re-serialized stages directly shortens the awaited write. + +**Verdict: no measurable build-time speedup on the common path.** The value is correctness/hygiene (reused-unchanged no longer marks dirty; the fold is a clean deduped `Set` union, not an array concat the review's "no-op union" comment misdescribed) plus headroom for custom step-based tasks whose `each` reads per key (where the fold would otherwise carry key-count × reads-per-key duplicates) and for the awaited-write BuildServer path. The scope was deliberately kept to dedup + the dirty-flag fix (C8 answered: not worth an incrementally-maintained union); `foldReadsInto` dedups exact paths only and does not reason about pattern coverage (a dropped-pattern-coverage variant was prototyped — ~629 → ~442 on the theme change — then dropped as unjustified complexity for a sub-ms win). The larger latent lever (avoiding the per-path `byPath` on a `findExactMatch` miss) is untouched and out of scope. + +### 14. One stage per step costs no extra stages for an OpenUI5 library + +The one-stage-per-step design was reviewed as the suspected structural cause of the branch building slower than `main`. Measured, it is not. Count stages from the perf log (`importStages ... with N stages`, or the unique `task/...::step/...` ids) and compare `main` to this branch for a `ui5 build` and a `ui5 build --all`: a library builds with the same stage count on both. Every shipped step-based task (`minify`, `buildThemes`, the three `replace*`, `escapeNonAsciiCharacters`, `enhanceManifest`) is single-step, and `generateThemeDesignerResources` (the only multi-step task, three steps) is not in the default library task set, so step count equals task count. A single-step task already is exactly one stage, so collapsing it to `task/{taskName}` removes no stage, leaves `StepRunner`'s per-key machinery in place, and was measured and rejected as a pure stage-id rename. Per-stage `updateProjectIndices` cost is the same on both revisions. + +The branch's warm and stale deltas against `main` are real but are not a stage-count effect. On a stale build they are the per-key map-step machinery (`#keyId` content hashing, item 12; the delta fold, item 13; per-unit allocation). On a warm build the project is served from cache and tasks are skipped, so the delta is startup and module loading, not the step pipeline; the plan-time module-loading part is item 18. Precise numbers live in the benchmark results repository alongside the config that produced them, not here, since they drift with the code. + +### 15. The step invocation sidecar is re-serialized only for stages that re-recorded + +The per-stage step invocation data (`task_metadata` type `"steps"`) was re-serialized and rewritten in `writeCache` on every build that writes cache, for every step-based stage, whether or not its map changed. `getStepInvocationData` loads and memoizes a stage's map (the `getPreviousInvocationData` hook calls it for every step, including full cache hits that never re-record), and the old `#prepareStageRequestCache` loop walked the whole memoized map and emitted a row for each non-empty entry. A full-hit stage therefore paid a `JSON.stringify` plus a SQLite write of its unchanged map for nothing. + +**Measured (2026-10, cold-built `sap.m` + its three built dependencies, scratch cache):** the sidecar is tens of KB per step stage, not the multi-megabyte an earlier estimate assumed, because the per-key id was shortened to `path` + `lastModified` + `size` (item 12). `SELECT project_id, count(*), sum(length(data)) FROM task_metadata WHERE type='steps' GROUP BY project_id`: + +| Project | step rows | sidecar bytes | +|---|---|---| +| `sap.ui.core` | 7 | ~239 KB | +| `sap.m` | 6 | ~226 KB | +| `sap.ui.layout` | 6 | ~29 KB | +| `sap.ui.unified` | 6 | ~22 KB | + +Largest single row ~94 KB (`replaceVersion`), ~515 KB across the four projects per build. + +The fix tracks which stages re-recorded (a dirty `Set` added to by `setStepInvocationData`, the only mutation path; `getStepInvocationData` loads without marking) and emits only those. On a one-file delta build (`Button.js` edited), `sap.m` re-records 3 of its 7 loaded step stages; the other 4 (and every step stage of the three dependencies, all full hits on this delta) are no longer rewritten. `writeCache` for `sap.m` was ~150 ms on this scenario; the sidecar serialization it now skips is a small fraction of that, so the direct wall-clock win is minor in CLI mode where the write is deferred off the critical path. As with the request-graph dirty-flag fix (§13), it matters more in BuildServer mode (Phase 4), where cache writes are awaited and each skipped row shortens the awaited write. + +Alongside this, `#storeStepReturns` previously opened one SQLite transaction per returning unit; it now buffers compressed rows and the driver flushes one transaction per step (`StepRunner.#runGroup` calls the store's `flush` after a step's units run). No shipped builder task returns resources from `each`, so this path is latent, but the first task that does would otherwise pay a transaction per key. The return descriptors are also built from the metadata `#prepareStageResources` already computed rather than re-reading each resource's `getIntegrity()`/`getSize()`. + +### 16. The stage prefetch never paid off and was removed + +A one-stage-ahead prefetch (`#prefetchNextStageCache`, `prefetchStageCache`, `#prefetchedStageReads`) read the next stage's cache rows while the current stage was prepared, on the premise that the read would overlap the current stage's execution. Every `CacheManager` read is synchronous better-sqlite3, so there is no overlap to win: the prefetch moves the same blocking read earlier in the same thread. It also computed the next stage's signatures before that stage's `updateProjectIndices` ran, so on a delta build the prefetched signatures did not match the ones the lookup then asked for, and it read every existing signature's `resourceMetadata` while `#findStageCache` needs only the first match. + +**Measured (2026-10, `sap.m` with its three built dependencies, working-tree CLI via `UI5_CLI_NO_LOCAL`):** counters around the prefetch over a whole build. + +| Scenario | signatures read | lookups served from prefetch | hit rate | bytes deserialized | stages that read disk anyway | +|---|---|---|---|---|---| +| Cold (empty cache) | 0 | 0 | n/a | 0 | 0 | +| Warm (no change) | 0 | 0 | n/a | 0 | 0 | +| One file changed (delta) | 8 | 3 | 37.5% | ~736 KB | 5 | + +Cold has nothing cached to prefetch. The warm no-change build is served by the project-level result cache before any per-stage `prepareStageExecutionAndValidateCache` runs, so the prefetch never fires. The delta path is the only one that prefetches, and 5 of its 8 prefetched maps missed the lookup (the stale-signature and over-read effects above). A hyperfine A/B (prefetch on vs a `UI5_NO_PREFETCH` early return, warmup 3 / runs 10) found no win: cold 21.91 s vs 21.88 s and warm 2.43 s vs 2.52 s are within noise, and on the delta path prefetch-off was marginally faster (4.654 s vs 4.753 s, 1.02x). The mechanism was removed; `#findStageCache` keeps the in-memory `StageCache` fast path and the single-row disk read. + +### 17. Per-stage hot-path micro-costs: one real cold-build win, the rest within noise + +A set of small inefficiencies in paths that now run once per stage was reviewed as a possible contributor to the branch building slower than `main`. Measured on `sap.m` (with its three built dependencies, working-tree CLI via `UI5_CLI_NO_LOCAL`, isolated `UI5_DATA_DIR`), only one moves wall-clock, and only on a cold build. + +Applied unconditionally, all behavior-preserving: +- **`#writtenResultResourcePaths` accumulation.** Three sites appended to this ordered list behind an `Array.includes` membership test. The list grows to the project's full written-resource count and is appended to once per written resource per stage, so the membership scan is O(n squared) per stage. A parallel `Set` now backs the membership check; the ordered list stays for `updateProjectIndices`. This is largest on a **cold** build, where every one of `sap.m`'s ~12k written resources is checked against a growing array across every stage. +- **Empty input and root signatures.** `TaskInputSet.#computeSignature` and `BuildStageCache.getRootSignature` each built a sha256 over an empty list on every call, producing a known constant. Both lists are empty for every stage of a standard build (no shipped task records inputs at the stage level or reads through `getRootReader`). Each now returns a module-level constant for the empty case, equal to the digest the loop produced (covered by unit tests). +- **Input-set sort.** `TaskInputSet.getEntries()` sorted by `(type + "\0" + name).localeCompare(...)`, running ICU collation on ASCII identifiers on every `getInputSignature()` call. It now uses a plain code-point comparison on the composite `type\0name` key and memoizes the sorted array (the map is populated only in the constructor). A unit test asserts the code-point order equals the previous `localeCompare` order for the recorded input shapes, so no stage signature moves. + +Left as-is after analysis: +- **`updateProjectIndices` input.** The list passed per stage is the paths accumulated so far this build (source changes plus earlier stages' writes), not the whole build's final set, and it grows as stages run, so stage N already receives only the changes from stages 0..N-1. A finer per-stage delta is not safely derivable: this stage's cached index baseline is the previous build's final state, so it must see every change since then. `updateIndices` also early-exits when the stage recorded no requests. Documented at the call site. +- **Linear stage lookup.** The per-stage `#stageOrder.indexOf(stageId)` the review flagged lived in the one-stage-ahead prefetch, which was already removed (§16). No lookup remains. + +Skipped after measurement (no win in any scenario): +- **Write-buffer flush.** `StepRunner.#flushWriteBuffer` drains a map step's buffered writes with a sequential `await` loop. The duplicate-path rejection is enforced at buffer insertion, not flush, so the flush is a pure replay of distinct-path writes to the in-memory stage workspace, and the buffer already holds every write before the flush, so parallelizing does not cut peak memory. Instrumented totals: on a **stale** one-file build no flush batch exceeds 50 writes (sub-millisecond); on a **cold** build the flush totals ~147 ms across the whole ~21 s build (largest single batch 2565 writes in ~40 ms, ~15 µs/write, consistent with memory-adapter writes). Parallelizing would save a fraction of that against a real write-ordering risk, so the sequential loop stays. +- **Per-unit `MonitoredTaskUtil` proxy.** The proxy allocates a wrapper closure per tracked-method access. Counting `get`-trap invocations across a whole build: ~4,973 on a stale one-file build, ~183,261 on a cold build (every unit runs). At tens of nanoseconds per short-lived closure this is a few milliseconds on a ~21 s cold build and negligible on stale, so caching bound methods on the per-unit proxy (against the deliberate per-unit isolation) was not worth it. + +**Aggregate before/after (hyperfine, isolated `UI5_DATA_DIR`):** + +| Scenario | Before | After | +|---|---|---| +| Stale (one file appended to `Button.js`, warmup 2 / runs 8) | 4.630 s ± 0.067 | 4.723 s ± 0.086 | +| Cold (empty cache, warmup 1 / runs 3) | 21.438 s ± 0.165 | 20.598 s ± 0.195 | + +Stale is within noise (the ranges overlap; these edits barely execute when few units run). Cold is ~0.8 s faster (~4%), consistent with the O(n squared) accumulation removal being largest where ~12k resources are written. The headline is correctness and hygiene, with a measurable cold-build improvement and no stale regression. + +### 18. Step-based task modules must not import their processors at plan time + +`TaskRunner.runTasks` imports every step-based task's module and calls its factory at plan time, before the cache decides whether any step runs, to discover step names for `setTasks`. When a task module imported its processor at module top level, planning evaluated that processor's whole graph for every step-based task, including tasks that turn out to be full cache hits and never run a step body. The heavy graphs are `buildThemes`' `less-openui5` (pulled in through `themeBuilderWorker.js` -> `themeBuilder.js`), `minify`'s `minifier`, `enhanceManifest`'s `manifestEnhancer` (`semver`), and `generateThemeDesignerResources`' less generator. On `main` these loaded lazily inside the task body, after the cache check; the step-factory refactor moved them to module top level because the factory module is imported eagerly for discovery. + +The fix keeps the factory cheap and defers each processor to its step body (`const p = (await import("../processors/...")).default`), so a cache-hit build never loads it. `buildThemes` is the exception: its worker module `themeBuilderWorker.js` is both the main-thread fs-bridge helper source and the workerpool worker entry, so dynamically importing it from the main thread breaks the worker tests (a dangling, unterminated pool). Instead `themeBuilderWorker.js` defers `themeBuilder` (the `less-openui5` graph) inside its worker entry `execThemeBuild`; its static `workerpool` import stays, since the worker registration runs at module load. `workerpool` is light (~4 ms). + +**Measured (2026-10, min-of-5 fresh-process import of the eight library step-based task modules, `node --input-type=module`):** + +| Task module set | Before | After | +|---|---|---| +| All eight imported in one process | ~58 ms | ~40 ms | +| `buildThemes` alone (fresh process) | ~64 ms | ~37 ms | + +The aggregate drop is bounded by dependency sharing: the light factory modules still load `@ui5/fs` and `@ui5/logger`, and `buildThemes` still loads `workerpool`. The win is the heavy processor graphs (`less-openui5`, `minifier`, `semver`, the less generator) no longer evaluating at plan time, which also keeps them out of memory on a warm build where no step runs. This is the "module loading" part of the warm-build delta noted in item 14. Measure it directly (import the task modules and time module evaluation, or `NODE_OPTIONS=--cpu-prof` on a warm `sap.m` build and read module-eval time before the first `_executeTask`) rather than through the end-to-end build, where ~18 ms sits below the stale/warm noise floor. + ## Investigation Workflow 1. **Establish a baseline.** Run the build 2-3 times to get stable warm-cache timings. Note the total time and per-phase breakdown. @@ -361,7 +479,7 @@ When diagnosing slow `writeStageResources`, check the `CAS skipped` vs `CAS writ 3. **Find the dominant phase.** In the perf log, look for the largest times: - Source index init? → Check `fromCacheWithDelta` vs total to see if it's I/O or hash-bound - Dependency index flush? → Check "changed paths" count and "cache misses" - - Task execution? → Check which tasks run and whether they support differential builds (◇ vs ›) + - Task execution? → Check which tasks run and whether they run as a delta (◇, a step-based task with a re-run step or a legacy delta) or a full re-execution (›) - `allTasksCompleted`? → Check `#revalidateSourceIndex` and `#freezeUntransformedSources` sub-timings - Cache write? → Check the sub-operation breakdown From 010e0a887bc2223da1fa0eb2601300ffe0bf5f26 Mon Sep 17 00:00:00 2001 From: Matthias Osswald Date: Wed, 7 Oct 2026 10:54:57 +0200 Subject: [PATCH 6/8] docs: Rework the cache-aware custom task section Replace the supportsDifferentialBuilds and changedProjectResourcePaths documentation in CustomTasks.md with the step-based task API: a task opts in through a static stepBased flag and a default build(options) factory that returns scalar and map step descriptors, and the build cache re-runs only the steps and keys whose inputs changed on a delta build. Rewrite the renderMarkdownFiles example as a step-based task with a single map step and a lazily imported processor. Co-authored-by: Merlin Beutlberger --- .../docs/pages/extensibility/CustomTasks.md | 195 ++++++++---------- 1 file changed, 86 insertions(+), 109 deletions(-) diff --git a/internal/documentation/docs/pages/extensibility/CustomTasks.md b/internal/documentation/docs/pages/extensibility/CustomTasks.md index 2fe7db198ed..63812dde9f8 100644 --- a/internal/documentation/docs/pages/extensibility/CustomTasks.md +++ b/internal/documentation/docs/pages/extensibility/CustomTasks.md @@ -128,14 +128,6 @@ A custom task implementation needs to return a function with the following signa * Namespace of the project currently being built * @param {string} parameters.options.configuration * Custom task configuration, as defined in the project's ui5.yaml - * @param {string[] | undefined} parameters.changedProjectResourcePaths - * List of changed resource paths since last execution. - * Only used if the task supports differential builds (supportsDifferentialBuilds=true). - * Returns undefined if unsupported or no cache is available. - * @param {string[] | undefined} parameters.changedDependencyResourcePaths - * List of changed dependency resource paths since last execution. - * Only used if the task supports differential builds (supportsDifferentialBuilds=true). - * Returns undefined if unsupported or no cache is available. * @param {string} parameters.options.taskName * Name of the custom task. * This parameter is only provided to custom task extensions @@ -174,14 +166,6 @@ export default async function({dependencies, log, options, taskUtil, workspace}) * Namespace of the project currently being built * @param {string} parameters.options.configuration * Custom task configuration, as defined in the project's ui5.yaml - * @param {string[] | undefined} parameters.changedProjectResourcePaths - * List of changed resource paths since last execution. - * Only used if the task supports differential builds (supportsDifferentialBuilds=true). - * Returns undefined if unsupported or no cache is available. - * @param {string[] | undefined} parameters.changedDependencyResourcePaths - * List of changed dependency resource paths since last execution. - * Only used if the task supports differential builds (supportsDifferentialBuilds=true). - * Returns undefined if unsupported or no cache is available. * @param {string} parameters.options.taskName * Name of the custom task. * This parameter is only provided to custom task extensions @@ -302,48 +286,47 @@ module.exports.determineRequiredDependencies = async function({availableDependen ``` ::: -### "Cache-aware" Tasks +### "Cache-aware" Tasks -Due to UI5 Builder and UI5 Server supporting **build caches** of task data, custom tasks can opt into this behavior to improve performance. To do this, export optional callback functions in your task implementation: +Due to UI5 Builder and UI5 Server supporting **build caches** of task data, custom tasks can opt into this behavior to improve performance. A cache-aware task is a **step-based task**: instead of a task body, it default-exports a factory `build(options)` and declares a static `stepBased` flag. The factory returns an array of steps that describe the work. The build cache tracks each step's inputs and, on a delta build, re-runs only the steps (and, within a step, only the keys) whose inputs changed, restoring the rest from cache. A task that is not step-based, runs on a Specification Version below 5.0, or has no cache available, processes all resources from scratch. -#### `supportsDifferentialBuilds()` +Step-based custom tasks are available from Specification Version 5.0. A task opts in with a static `stepBased` export set to `true`: ::: code-group + ```js [ESM] -/** - * Indicates whether the task supports differential builds - * - * Tasks that support differential builds can use incremental cache invalidation, - * processing only changed resources rather than rebuilding from scratch. - * - * @public - * @returns {boolean} True if differential builds are supported - */ -export function supportsDifferentialBuilds() { - return true; -} +export const stepBased = true; +export default function build(options) { + return [ /* steps */ ]; +}; ``` ```js [CommonJS] -/** - * Indicates whether the task supports differential builds - * - * Tasks that support differential builds can use incremental cache invalidation, - * processing only changed resources rather than rebuilding from scratch. - * - * @public - * @returns {boolean} True if differential builds are supported - */ -module.exports.supportsDifferentialBuilds = function() { - return true; -} +module.exports = function build(options) { + return [ /* steps */ ]; +}; +module.exports.stepBased = true; ``` +::: -When this returns `true`, your task's main function receives an additional parameter `changedProjectResourcePaths`. This parameter provides an array of changed resource paths (strings) since its last execution. The task then processes only those resources instead of all resources. If this callback isn't provided or returns a falsy value, your task can't use incremental cache invalidation and processes all resources from scratch. +The factory receives the task `options` only (the same `options` object the standard task function receives); it never receives readers or a `taskUtil`, so it cannot close over build state. It must be pure over `options`: it may branch on `options.projectNamespace`, precompute glob patterns, or include or omit steps, but it must not read or write resources. Every input a step reads arrives through the step's own callback arguments. + +There are two kinds of steps, run in array order: + +* **Scalar step** `{name, needs?, run}` — runs once. `run: async ({needs, workspace, dependencies, taskUtil, options}) => value?` +* **Map step** `{name, needs?, keys, each}` — runs once per key. `keys: async ({needs, workspace, dependencies, taskUtil, options}) => keySet` enumerates the keys, and `each: async (key, {needs, workspace, dependencies, taskUtil, options}) => value?` processes one key. A key is a resource or a stable string. + +Each step gets its own `name` (a non-empty string, unique within the task). A map step is the usual cache-aware shape: the build cache treats every key as its own cached unit, so a delta build re-processes only the keys whose inputs changed. + +A step may list earlier step names in `needs`; those steps' return values then arrive as `needs.` in the step's callbacks. A step may reference only steps that come before it in the array, so there can be no cycle. A step's return value (resources, or any JSON-serializable value) folds into the cache signature of every step that consumes it, so a changed producer re-runs its consumers. ::: info Best Practices for Cache-aware Tasks -1. **Keep tasks deterministic**: Given the same inputs, always produce the same outputs -2. **Opt into differential builds carefully**: Only set `supportsDifferentialBuilds = true` if your task can safely process files independently +1. **Keep tasks deterministic**: Given the same inputs, always produce the same outputs. +2. **Keep the factory pure**: The `build(options)` factory must only return step descriptors from `options`. Never read or write resources in the factory; do that in the step callbacks. +3. **Read and write only through the callback arguments**: Every input a step reads must arrive through its `workspace`, `dependencies`, `taskUtil`, or `needs` arguments, and every write must go through the step's own `workspace`. A read or write that bypasses these is not recorded, so the cache cannot track it and a delta build can serve stale output. +4. **Import heavy processors lazily**: UI5 CLI calls the factory on every build to discover the steps, even when every step is a cache hit. Import heavy modules inside the step callbacks (for example `const p = (await import("./processor.js")).default`), not at the top of the module, so a fully cached build does not load them. +5. **Split work carefully**: Only use a map step if each key can be processed independently. Two keys that run concurrently must not write the same path. +6. **Name steps stably**: Reuse the same step `name` across builds so the cached data is found. ::: ### Examples @@ -355,88 +338,82 @@ The following code snippets show examples for custom task implementations. This example is making use of the `resourceFactory` [TaskUtil](../../api/@ui5_project_build_helpers_TaskUtil.html) API to create new resources based on the output of a third-party module for rendering Markdown files. The created resources are added to the build result by writing them into the provided `workspace`. -In addition, this task supports differential builds, which re-process only changed resources. +This task is a step-based, cache-aware task: a single map step renders one Markdown resource per key, so a delta build re-renders only the files that changed. The `renderMarkdown` processor is imported lazily inside the `each` callback, so a fully cached build never loads it. ::: code-group ```js [ESM] import path from "node:path"; -import renderMarkdown from "./renderMarkdown.js"; +import {getLogger} from "@ui5/logger"; + +const log = getLogger("builder:tasks:renderMarkdownFiles"); /* * Render all .md (Markdown) files in the project to HTML */ -export default async function({dependencies, log, options, taskUtil, workspace, changedProjectResourcePaths}) { - const {createResource} = taskUtil.resourceFactory; - let textResources; - - if (changedProjectResourcePaths) { - textResources = await Promise.all(changedProjectResourcePaths.map((resource) => workspace.byPath(resource))); - } else { - textResources = await workspace.byGlob("**/*.md"); - } - - await Promise.all(textResources.map(async (resource) => { - const markdownResourcePath = resource.getPath(); - - log.info(`Rendering markdown file ${markdownResourcePath}...`); - const htmlString = await renderMarkdown(await resource.getString(), options.configuration); - - // Note: @ui5/fs virtual paths are always (on *all* platforms) POSIX. Therefore using path.posix here - const newResourceName = path.posix.basename(markdownResourcePath, ".md") + ".html"; - const newResourcePath = path.posix.join(path.posix.dirname(markdownResourcePath), newResourceName); - - const markdownResource = createResource({ - path: newResourcePath, - string: htmlString - }); - await workspace.write(markdownResource); - })); +export const stepBased = true; +export default function build(options) { + return [{ + name: "render", + // One cached unit per Markdown file, so a delta build re-renders only the files that changed. + keys: async ({workspace}) => workspace.byGlob("**/*.md"), + each: async (resource, {workspace, taskUtil}) => { + // Import the processor lazily so a fully cached build does not load it + const renderMarkdown = (await import("./renderMarkdown.js")).default; + const {createResource} = taskUtil.resourceFactory; + const markdownResourcePath = resource.getPath(); + + log.info(`Rendering markdown file ${markdownResourcePath}...`); + const htmlString = await renderMarkdown(await resource.getString(), options.configuration); + + // Note: @ui5/fs virtual paths are always (on *all* platforms) POSIX. Therefore using path.posix here + const newResourceName = path.posix.basename(markdownResourcePath, ".md") + ".html"; + const newResourcePath = path.posix.join(path.posix.dirname(markdownResourcePath), newResourceName); + + await workspace.write(createResource({ + path: newResourcePath, + string: htmlString + })); + } + }]; }; - -export function supportsDifferentialBuilds() { - return true; -} ``` ```js [CommonJS] const path = require("node:path"); -const renderMarkdown = require("./renderMarkdown.js"); +const {getLogger} = require("@ui5/logger"); + +const log = getLogger("builder:tasks:renderMarkdownFiles"); /* * Render all .md (Markdown) files in the project to HTML */ -module.exports = async function({dependencies, log, options, taskUtil, workspace, changedProjectResourcePaths}) { - const {createResource} = taskUtil.resourceFactory; - let textResources; - - if (changedProjectResourcePaths) { - textResources = await Promise.all(changedProjectResourcePaths.map((resource) => workspace.byPath(resource))); - } else { - textResources = await workspace.byGlob("**/*.md"); - } - - await Promise.all(textResources.map(async (resource) => { - const markdownResourcePath = resource.getPath(); - - log.info(`Rendering markdown file ${markdownResourcePath}...`); - const htmlString = await renderMarkdown(await resource.getString(), options.configuration); - - // Note: @ui5/fs virtual paths are always (on *all* platforms) POSIX. Therefore using path.posix here - const newResourceName = path.posix.basename(markdownResourcePath, ".md") + ".html"; - const newResourcePath = path.posix.join(path.posix.dirname(markdownResourcePath), newResourceName); - - const markdownResource = createResource({ - path: newResourcePath, - string: htmlString - }); - await workspace.write(markdownResource); - })); +module.exports = function build(options) { + return [{ + name: "render", + // One cached unit per Markdown file, so a delta build re-renders only the files that changed. + keys: async ({workspace}) => workspace.byGlob("**/*.md"), + each: async (resource, {workspace, taskUtil}) => { + // Import the processor lazily so a fully cached build does not load it + const renderMarkdown = require("./renderMarkdown.js"); + const {createResource} = taskUtil.resourceFactory; + const markdownResourcePath = resource.getPath(); + + log.info(`Rendering markdown file ${markdownResourcePath}...`); + const htmlString = await renderMarkdown(await resource.getString(), options.configuration); + + // Note: @ui5/fs virtual paths are always (on *all* platforms) POSIX. Therefore using path.posix here + const newResourceName = path.posix.basename(markdownResourcePath, ".md") + ".html"; + const newResourcePath = path.posix.join(path.posix.dirname(markdownResourcePath), newResourceName); + + await workspace.write(createResource({ + path: newResourcePath, + string: htmlString + })); + } + }]; }; - -module.exports.supportsDifferentialBuilds = function() { - return true; -} +module.exports.stepBased = true; ``` ::: From 965a674377656cfca8450e5e2319c129679f6663 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Thu, 8 Oct 2026 09:55:49 +0200 Subject: [PATCH 7/8] refactor(project): Rename leftover task terminology to stage in build cache The step-based build API renamed the build-time unit from "task" to "stage" (one stage per legacy task, one per step of a step-based task), but several cache identifiers kept the old "task" name while holding stage data. Align them with what the code actually stores. - Index cache: the stage-execution-order field `tasks` -> `stages`. It holds one entry per stage, keyed by stage id, not per task. - Persistent storage: the `task_metadata` table -> `stage_request_metadata` (it holds a stage's resource request graphs and non-resource inputs, not task output), and its accessors readTaskMetadata/writeTaskMetadata -> readStageRequestMetadata/writeStageRequestMetadata on BuildCacheStorage and CacheManager. - In-code identifiers: tasksWithDepRequests -> stagesWithDepRequests, taskDependencySignatures -> stageDependencySignatures, #anyTaskHasRootRequests -> #anyStageHasRootRequests, and cache-layer comments/JSDoc that called a stage a "task". Genuine task terms are kept: taskName/stepName, the task/{taskName} stage-id prefix, setTasks, allTasksCompleted, and references to step-based and legacy build tasks. The table and field renames change the on-disk cache format. The incremental build cache is unreleased and obsolete versioned cache directories are ignored and rebuilt, so no migration or CACHE_VERSION bump is required. Also updates the incremental-build skill architecture doc to match. --- .../skills/incremental-build/architecture.md | 8 +- .../lib/build/cache/BuildCacheStorage.js | 22 ++--- .../lib/build/cache/BuildStageCache.js | 36 +++---- .../project/lib/build/cache/CacheManager.js | 14 +-- .../lib/build/cache/ProjectBuildCache.js | 96 +++++++++---------- .../lib/build/cache/ResourceRequestManager.js | 22 ++--- .../test/lib/build/cache/BuildCacheStorage.js | 26 ++--- .../test/lib/build/cache/CacheManager.js | 6 +- .../test/lib/build/cache/ProjectBuildCache.js | 46 ++++----- 9 files changed, 138 insertions(+), 138 deletions(-) diff --git a/.claude/skills/incremental-build/architecture.md b/.claude/skills/incremental-build/architecture.md index b0e61a1846c..0896c27c8f9 100644 --- a/.claude/skills/incremental-build/architecture.md +++ b/.claude/skills/incremental-build/architecture.md @@ -255,7 +255,7 @@ On a failed re-resolve `Supervisor` flags the surviving stack degraded (last-goo | content (CAS: integrity -> gzip-compressed BLOB) | | index_cache (resource index trees, by kind="source") | | stage_metadata (cached stage results, by stage signature) | -| task_metadata (resource requests per stage, by type) | +| stage_request_metadata (resource requests per stage, by type)| | result_metadata(per-build result metadata) | +-----------------------------------------------------------------+ ``` @@ -279,7 +279,7 @@ A task's output can depend on inputs that are not resources: an environment vari Tracking has a record side and a lookup side, mirroring the resource-request flow: -- **Record** (task executes): the TaskRunner hands the task a `MonitoredTaskUtil` instead of the raw `TaskUtil`. It is a Proxy that preserves the wrapped shape (a custom task's limited interface stays limited) and records every tracked read as `{type, name, value}`, normalizing the value via `normalizeInputValue`. `getProject(name)` returns a wrapped project whose tracked accessors record under the project's name; its `getReader()` result is wrapped in a `MonitoredReader` so the resources the task reads through it are recorded as resource requests, and, for the project being built, its `getRootReader()` result is wrapped too so reads of files outside the UI5 resource model are recorded as root requests (see the reader-monitoring note below), while untracked members (`getRootPath`, `getSpecVersion`, ...) pass straight through unrecorded. After the task, the TaskRunner drains `getInputRecording()` into `ProjectBuildCache.recordStageResult`, which builds a `TaskInputSet` and folds its signature into the task's project-component signature (`combineProjectAndInputSignature`). Only entry type/name are persisted (`task_metadata` type `"input"`), never values. +- **Record** (task executes): the TaskRunner hands the task a `MonitoredTaskUtil` instead of the raw `TaskUtil`. It is a Proxy that preserves the wrapped shape (a custom task's limited interface stays limited) and records every tracked read as `{type, name, value}`, normalizing the value via `normalizeInputValue`. `getProject(name)` returns a wrapped project whose tracked accessors record under the project's name; its `getReader()` result is wrapped in a `MonitoredReader` so the resources the task reads through it are recorded as resource requests, and, for the project being built, its `getRootReader()` result is wrapped too so reads of files outside the UI5 resource model are recorded as root requests (see the reader-monitoring note below), while untracked members (`getRootPath`, `getSpecVersion`, ...) pass straight through unrecorded. After the task, the TaskRunner drains `getInputRecording()` into `ProjectBuildCache.recordStageResult`, which builds a `TaskInputSet` and folds its signature into the task's project-component signature (`combineProjectAndInputSignature`). Only entry type/name are persisted (`stage_request_metadata` type `"input"`), never values. - **Lookup** (later build): `BuildStageCache.getInputSignature(resolveValue)` recomputes the input signature, re-reading each recorded input's *current* value through `ProjectBuildContext.resolveInputValue(type, name)` (which reaches `process.env` and the current project graph). A value that differs from the one baked into the cached stage signature misses the cache and re-runs the task. Record and lookup normalize through the same `normalizeInputValue`, so equal values compare equal. Excluded from input-value tracking: mutations (`setTag`/`clearTag`, already captured as tag operations in the hash trees), constructors (`resourceFactory`), readers (`getReader`/`getRootReader`), side effects (`registerCleanupTask`), and the FS-path accessors (`getRootPath`/`getSourcePath`) whose absolute, machine-specific values would make cache entries non-portable. @@ -601,7 +601,7 @@ project.getProjectResources().setStage(stageName, stageCache.stage, - content(integrity TEXT PK, data BLOB) # CAS: gzip above ~128 bytes - index_cache(project_id, build_signature, kind, data) # kind: "source" - stage_metadata(project_id, build_signature, stage_id, stage_signature, data) - - task_metadata(project_id, build_signature, stage_id, type, data) # stage_id is the STAGE id (task/{taskName} or task/{taskName}::step/{stepName}); type: "project" | "dependencies" | "input" | "root" | "root-no-gitignore" + - stage_request_metadata(project_id, build_signature, stage_id, type, data) # stage_id is the STAGE id (task/{taskName} or task/{taskName}::step/{stepName}); type: "project" | "dependencies" | "input" | "root" | "root-no-gitignore" - result_metadata(project_id, build_signature, stage_signature, data) ``` @@ -612,7 +612,7 @@ Note: Both CAS content and metadata BLOBs are gzip-compressed via thresholds (`C The index cache (one row per `(project_id, build_signature, kind="source")`) contains: - `indexTimestamp`: creation timestamp (used for racy-git detection) - `root`: serialized Merkle tree (TreeNode hierarchy) -- `tasks`: array of `[stageId, stepBased ? 1 : 0]` recording the stage execution order (one entry per stage: a legacy task's `task/{taskName}`, or a step-based task's per-step `task/{taskName}::step/{stepName}`) and whether the stage ran the step runner (step-based), which drives delta tracking. The stage id is the `task_metadata` key everything else for that stage is stored under +- `stages`: array of `[stageId, stepBased ? 1 : 0]` recording the stage execution order (one entry per stage: a legacy task's `task/{taskName}`, or a step-based task's per-step `task/{taskName}::step/{stepName}`) and whether the stage ran the step runner (step-based), which drives delta tracking. The stage id is the `stage_request_metadata` key everything else for that stage is stored under #### Stage Metadata Format diff --git a/packages/project/lib/build/cache/BuildCacheStorage.js b/packages/project/lib/build/cache/BuildCacheStorage.js index 35ecf32d89a..bd1bb046bc5 100644 --- a/packages/project/lib/build/cache/BuildCacheStorage.js +++ b/packages/project/lib/build/cache/BuildCacheStorage.js @@ -10,12 +10,12 @@ const METADATA_COMPRESSION_THRESHOLD = 4096; const CONTENT_COMPRESSION_THRESHOLD = 128; /** All live data table names */ -const DATA_TABLES = ["content", "index_cache", "stage_metadata", "task_metadata", "result_metadata"]; +const DATA_TABLES = ["content", "index_cache", "stage_metadata", "stage_request_metadata", "result_metadata"]; /** * Unified SQLite-backed storage for the build cache * - * Stores both metadata (index caches, stage metadata, task metadata, result metadata) + * Stores both metadata (index caches, stage metadata, stage request metadata, result metadata) * and content-addressable resource content (gzip-compressed BLOBs) in a single database. * * @class @@ -70,7 +70,7 @@ export default class BuildCacheStorage { PRIMARY KEY (project_id, build_signature, stage_id, stage_signature) ) WITHOUT ROWID; - CREATE TABLE IF NOT EXISTS task_metadata ( + CREATE TABLE IF NOT EXISTS stage_request_metadata ( project_id TEXT NOT NULL, build_signature TEXT NOT NULL, stage_id TEXT NOT NULL, @@ -127,12 +127,12 @@ export default class BuildCacheStorage { ), // Stage request metadata - readTaskMetadata: this.#db.prepare( - `SELECT data FROM task_metadata + readStageRequestMetadata: this.#db.prepare( + `SELECT data FROM stage_request_metadata WHERE project_id = ? AND build_signature = ? AND stage_id = ? AND type = ?` ), - writeTaskMetadata: this.#db.prepare( - `INSERT OR REPLACE INTO task_metadata + writeStageRequestMetadata: this.#db.prepare( + `INSERT OR REPLACE INTO stage_request_metadata (project_id, build_signature, stage_id, type, data) VALUES (?, ?, ?, ?, ?)` ), @@ -335,9 +335,9 @@ export default class BuildCacheStorage { * @param {string} type "project" or "dependency" * @returns {object|null} Parsed stage metadata or null if not found */ - readTaskMetadata(projectId, buildSignature, stageId, type) { + readStageRequestMetadata(projectId, buildSignature, stageId, type) { try { - const row = this.#stmts.readTaskMetadata.get( + const row = this.#stmts.readStageRequestMetadata.get( projectId, buildSignature, stageId, type ); return row ? this.#deserializeMetadata(row.data) : null; @@ -359,8 +359,8 @@ export default class BuildCacheStorage { * @param {string} type "project" or "dependency" * @param {object} metadata Stage metadata object to serialize */ - writeTaskMetadata(projectId, buildSignature, stageId, type, metadata) { - this.#stmts.writeTaskMetadata.run( + writeStageRequestMetadata(projectId, buildSignature, stageId, type, metadata) { + this.#stmts.writeStageRequestMetadata.run( projectId, buildSignature, stageId, type, this.#serializeMetadata(metadata) ); } diff --git a/packages/project/lib/build/cache/BuildStageCache.js b/packages/project/lib/build/cache/BuildStageCache.js index 2d49a729818..cd61a89661c 100644 --- a/packages/project/lib/build/cache/BuildStageCache.js +++ b/packages/project/lib/build/cache/BuildStageCache.js @@ -55,11 +55,11 @@ export default class BuildStageCache { // managers because getRootReader's useGitignore flag changes which resources a glob matches, so a // request recorded with the flag on must re-materialize against a root reader with the flag on. // Both are resolved against a dedicated root reader (not the stage-pipeline project reader) and - // their signatures fold into the task's stage signature. + // their signatures fold into the stage signature. #rootRequestManagers; // Tracks non-resource inputs (environment variables and TaskUtil interface reads) recorded during - // the last execution of this task. Its signature is folded into the task's stage signature so that + // the last execution of this stage. Its signature is folded into the stage signature so that // a changed input invalidates the cached result. Only entry names are persisted; values are // re-read on lookup (see #inputSet.getSignatureWithCurrentValues). #inputSet; @@ -120,7 +120,7 @@ export default class BuildStageCache { * @param {boolean} options.stepBased Whether the stage ran the step runner, driving per-step delta tracking * @param {object} options.projectRequests Cached project request manager data * @param {object} options.dependencyRequests Cached dependency request manager data - * @param {object} [options.inputSet] Cached task input set data + * @param {object} [options.inputSet] Cached stage input set data * @param {object} [options.rootRequests] Cached useGitignore:true root request manager data * @param {object} [options.rootNoGitignoreRequests] Cached useGitignore:false root request manager data * @returns {BuildStageCache} Restored stage cache instance @@ -190,12 +190,12 @@ export default class BuildStageCache { } /** - * Returns the signature of this task's recorded non-resource inputs, computed against the current + * Returns the signature of this stage's recorded non-resource inputs, computed against the current * environment and project graph. * * Used on cache lookup: each recorded input name is re-evaluated via the given resolver, so the * returned signature reflects the environment and graph of the build performing the lookup. When - * the task recorded no inputs, a stable empty-input digest is returned. + * the stage recorded no inputs, a stable empty-input digest is returned. * * @public * @param {function(string, string, (string|undefined)): (string|undefined)} [resolveValue] @@ -208,7 +208,7 @@ export default class BuildStageCache { } /** - * Returns whether this task recorded any root resource requests + * Returns whether this stage recorded any root resource requests * * @public * @returns {boolean} @@ -242,9 +242,9 @@ export default class BuildStageCache { /** * Returns a single signature aggregating the current signatures of both root request sets. * - * Folded into the task's stage signature so a changed root file misses the cached stage. Unlike + * Folded into the stage signature so a changed root file misses the cached stage. Unlike * the project and dependency components, root requests are not delta-tracked: the aggregate is one - * value, so any root change re-runs the whole task. A task with no recorded root requests yields a + * value, so any root change re-runs the whole stage. A stage with no recorded root requests yields a * stable digest that stays constant across builds. * * @public @@ -292,7 +292,7 @@ export default class BuildStageCache { } /** - * Returns whether this task has any recorded dependency resource requests + * Returns whether this stage has any recorded dependency resource requests * * @public * @returns {boolean} @@ -306,7 +306,7 @@ export default class BuildStageCache { * * Since dependency resources may change independently from this project's cache, a full * refresh of the dependency index is required at the beginning of every build from cache. - * This ensures all dependency resources are current before task execution. + * This ensures all dependency resources are current before stage execution. * * @public * @param {module:@ui5/fs.AbstractReader} dependencyReader Reader for accessing dependency resources @@ -317,11 +317,11 @@ export default class BuildStageCache { } /** - * Gets all project index signatures for this task + * Gets all project index signatures for this stage * * Returns signatures from all recorded project-request sets. Each signature represents * a unique combination of resources, belonging to the current project, that were accessed - * during task execution. These can be used as cache keys for restoring cached task results. + * during stage execution. These can be used as cache keys for restoring cached stage results. * * @public * @returns {string[]} Array of signature strings @@ -332,12 +332,12 @@ export default class BuildStageCache { } /** - * Gets all dependency index signatures for this task + * Gets all dependency index signatures for this stage * * Returns signatures from all recorded dependency-request sets. Each signature represents * a unique combination of resources, belonging to all dependencies of the current project, - * that were accessed during task execution. These can be used as cache keys for restoring - * cached task results. + * that were accessed during stage execution. These can be used as cache keys for restoring + * cached stage results. * * @public * @returns {string[]} Array of signature strings @@ -351,7 +351,7 @@ export default class BuildStageCache { * Gets all project index delta transitions for differential updates * * Returns a map of signature transitions and their associated changed resource paths - * for project resources. Used when tasks support differential updates to identify + * for project resources. Used when stages support differential updates to identify * which resources changed between cache states. * * @public @@ -366,7 +366,7 @@ export default class BuildStageCache { * Gets all dependency index delta transitions for differential updates * * Returns a map of signature transitions and their associated changed resource paths - * for dependency resources. Used when tasks support differential updates to identify + * for dependency resources. Used when stages support differential updates to identify * which dependency resources changed between cache states. * * @public @@ -493,7 +493,7 @@ export default class BuildStageCache { } /** - * Serializes the task cache to plain objects for persistence + * Serializes the stage cache to plain objects for persistence * * Exports both project and dependency resource request graphs in a format suitable * for JSON serialization. The serialized data can be passed to fromCache() to restore diff --git a/packages/project/lib/build/cache/CacheManager.js b/packages/project/lib/build/cache/CacheManager.js index 8cab156e517..4740bdba952 100644 --- a/packages/project/lib/build/cache/CacheManager.js +++ b/packages/project/lib/build/cache/CacheManager.js @@ -18,13 +18,13 @@ export const CACHE_VERSION = "v0_9"; * for both metadata and content-addressable resource content * * CacheManager delegates metadata operations (index caches, stage metadata, - * task metadata, result metadata) and binary resource content (gzip-compressed + * stage request metadata, result metadata) and binary resource content (gzip-compressed * BLOBs) to BuildCacheStorage (single SQLite database). * * The cache is organized by: * 1. Project ID (package name) * 2. Build signature (hash of build configuration) - * 3. Stage/task identifiers and signatures + * 3. Stage identifiers and signatures * * Key features: * - Content-addressable storage with deduplication @@ -119,7 +119,7 @@ export default class CacheManager { * @param {string} projectId Project identifier * @param {string} buildSignature Build signature hash * @param {string} kind "source" or "result" - * @param {object} index Index object containing resource tree and task metadata + * @param {object} index Index object containing resource tree and stage list */ writeIndexCache(projectId, buildSignature, kind, index) { this.#storage.writeIndexCache(projectId, buildSignature, kind, index); @@ -163,8 +163,8 @@ export default class CacheManager { * @param {string} type "project" or "dependency" * @returns {object|null} Parsed stage metadata or null if not found */ - readTaskMetadata(projectId, buildSignature, stageId, type) { - return this.#storage.readTaskMetadata(projectId, buildSignature, stageId, type); + readStageRequestMetadata(projectId, buildSignature, stageId, type) { + return this.#storage.readStageRequestMetadata(projectId, buildSignature, stageId, type); } /** @@ -177,8 +177,8 @@ export default class CacheManager { * @param {string} type "project" or "dependency" * @param {object} metadata Stage metadata object to serialize */ - writeTaskMetadata(projectId, buildSignature, stageId, type, metadata) { - this.#storage.writeTaskMetadata(projectId, buildSignature, stageId, type, metadata); + writeStageRequestMetadata(projectId, buildSignature, stageId, type, metadata) { + this.#storage.writeStageRequestMetadata(projectId, buildSignature, stageId, type, metadata); } /** diff --git a/packages/project/lib/build/cache/ProjectBuildCache.js b/packages/project/lib/build/cache/ProjectBuildCache.js index 0543a7c7327..f12b9beb051 100644 --- a/packages/project/lib/build/cache/ProjectBuildCache.js +++ b/packages/project/lib/build/cache/ProjectBuildCache.js @@ -46,7 +46,7 @@ export const RESULT_CACHE_STATES = Object.freeze({ * @typedef {object} StageCacheEntry * @property {string} signature Signature of the cached stage * @property {@ui5/fs/AbstractReader} stage Reader for the cached stage - * @property {string[]} writtenResourcePaths Array of resource paths written by the task + * @property {string[]} writtenResourcePaths Array of resource paths written by the stage * @property {Map>} projectTagOperations * Map of resource paths to their tags that were set or cleared during this stage's execution, for project tags * @property {Map>} buildTagOperations @@ -286,7 +286,7 @@ export default class ProjectBuildCache { // signal. Refresh their indices against the current project root and, if their aggregate // signature moved since the cache was last validated, force result-cache revalidation so a root // change is not skipped when no source or dependency resource changed. - if (this.#combinedIndexState === INDEX_STATES.FRESH && this.#anyTaskHasRootRequests()) { + if (this.#combinedIndexState === INDEX_STATES.FRESH && this.#anyStageHasRootRequests()) { const rootStart = performance.now(); await this.#refreshRootIndices(); const rootAggregate = this.#getAggregatedRootSignature(); @@ -324,7 +324,7 @@ export default class ProjectBuildCache { } /** - * Processes changed resources since last build, updating indices and invalidating tasks as needed + * Processes changed resources since last build, updating indices and invalidating stages as needed * * @param {@ui5/fs/AbstractReader} dependencyReader Reader for dependency resources * @returns {Promise} @@ -350,9 +350,9 @@ export default class ProjectBuildCache { let depIndicesChanged = false; if (this.#changedDependencyResourcePaths.length) { const depStart = performance.now(); - const tasksWithDepRequests = Array.from(this.#stageCaches.values()) + const stagesWithDepRequests = Array.from(this.#stageCaches.values()) .filter((stageCache) => stageCache.hasDependencyRequests()); - await Promise.all(tasksWithDepRequests.map(async (stageCache) => { + await Promise.all(stagesWithDepRequests.map(async (stageCache) => { const changed = await stageCache .updateDependencyIndices(dependencyReader, this.#changedDependencyResourcePaths); if (changed) { @@ -364,7 +364,7 @@ export default class ProjectBuildCache { `#flushPendingChanges updateDependencyIndices for project ${this.#project.getName()} ` + `completed in ${(performance.now() - depStart).toFixed(2)} ms ` + `(${this.#changedDependencyResourcePaths.length} changed paths, ` + - `${tasksWithDepRequests.length}/${this.#stageCaches.size} tasks, changed=${depIndicesChanged})`); + `${stagesWithDepRequests.length}/${this.#stageCaches.size} stages, changed=${depIndicesChanged})`); } } @@ -381,7 +381,7 @@ export default class ProjectBuildCache { } /** - * Initialize dependency indices for all tasks. This only needs to be called once per build. + * Initialize dependency indices for all stages. This only needs to be called once per build. * Later builds of the same project during the same overall build can reuse the existing indices * (they will be updated based on input via dependencyResourcesChanged) * @@ -389,9 +389,9 @@ export default class ProjectBuildCache { * @returns {Promise} */ async _refreshDependencyIndices(dependencyReader) { - const tasksWithDepRequests = Array.from(this.#stageCaches.values()) + const stagesWithDepRequests = Array.from(this.#stageCaches.values()) .filter((stageCache) => stageCache.hasDependencyRequests()); - await Promise.all(tasksWithDepRequests.map(async (stageCache) => { + await Promise.all(stagesWithDepRequests.map(async (stageCache) => { await stageCache.refreshDependencyIndices(dependencyReader); })); // Reset pending dependency changes since indices are fresh now anyways @@ -559,16 +559,16 @@ export default class ProjectBuildCache { // #getResultStageSignature (the store side) always walk the same stages in the same order. // createDependencySignature is positional and length-sensitive, so a divergence here would store // a result signature no later lookup could reproduce (F2). - const taskDependencySignatures = this.#stageOrder.map((stageId) => { + const stageDependencySignatures = this.#stageOrder.map((stageId) => { const stageCache = this.#stageCaches.get(stageId); if (!stageCache) { throw new Error( `Inconsistent stage state in project ${this.#project.getName()}: stage ${stageId} is ` + - `in the stage order but has no task cache`); + `in the stage order but has no stage cache`); } return stageCache.getDependencyIndexSignatures(); }); - const dependencySignaturesCombinations = cartesianProduct(taskDependencySignatures); + const dependencySignaturesCombinations = cartesianProduct(stageDependencySignatures); // The aggregated input and root signatures are single current values (not sets of cached // alternatives), so they apply to every dependency combination as constants. Each is its own slot @@ -613,7 +613,7 @@ export default class ProjectBuildCache { } /** - * Aggregates the current non-resource input signatures (e.g. recorded env-var usage) across all task + * Aggregates the current non-resource input signatures (e.g. recorded env-var usage) across all stage * caches into a single signature, re-evaluated against the current environment and graph. * * It is one slot of the result stage signature (root resources are a sibling slot via @@ -643,11 +643,11 @@ export default class ProjectBuildCache { } /** - * Whether any task cache recorded root resource requests. + * Whether any stage cache recorded root resource requests. * * @returns {boolean} */ - #anyTaskHasRootRequests() { + #anyStageHasRootRequests() { for (const stageCache of this.#stageCaches.values()) { if (stageCache.hasRootRequests()) { return true; @@ -657,8 +657,8 @@ export default class ProjectBuildCache { } /** - * Refreshes the root resource indices of every task cache that recorded root requests, resolving - * them against the current project root. Bounded by what the tasks requested. + * Refreshes the root resource indices of every stage cache that recorded root requests, resolving + * them against the current project root. Bounded by what the stages requested. * * @returns {Promise} */ @@ -670,7 +670,7 @@ export default class ProjectBuildCache { } /** - * Aggregates the current root signatures across all task caches into one signature, used to detect + * Aggregates the current root signatures across all stage caches into one signature, used to detect * whether any recorded root file changed since the cache was last validated. * * @returns {string} Aggregated root signature @@ -683,7 +683,7 @@ export default class ProjectBuildCache { return crypto.createHash("sha256").update(rootSignatures.sort().join("\0")).digest("hex"); } - // ===== TASK MANAGEMENT ===== + // ===== STAGE MANAGEMENT ===== /** * Prepares a stage for execution by switching to it and checking for cached results @@ -747,7 +747,7 @@ export default class ProjectBuildCache { // with the current input and root signatures (BuildStageCache.getStageSignatures). The input and // root signatures are re-evaluated against the current environment, graph, and project root, so a // changed input or root file misses the cached stage. Root indices were refreshed in validateCache - // before this build's tasks run. + // before this build's stages run. const inputSignature = stageCache.getInputSignature(this.#resolveInputValue); const rootSignature = stageCache.getRootSignature(); const stageSignatures = stageCache.getStageSignatures(this.#resolveInputValue); @@ -884,7 +884,7 @@ export default class ProjectBuildCache { } /** - * Attempts to find a cached stage for the given task + * Attempts to find a cached stage for the given stage * * Checks both in-memory stage cache and persistent cache storage for a matching * stage signature. Returns the first matching cached stage found. @@ -978,13 +978,13 @@ export default class ProjectBuildCache { } /** - * Records the result of a task execution and updates the cache + * Records the result of a stage execution and updates the cache * * This method: - * 1. Creates a signature for the executed task based on its resource requests + * 1. Creates a signature for the executed stage based on its resource requests * 2. Stores the resulting stage in the stage cache using that signature - * 3. Invalidates downstream tasks if they depend on written resources - * 4. Removes the task from the invalidated tasks list + * 3. Invalidates downstream stages if they depend on written resources + * 4. Removes the stage from the invalidated stages list * * @public * @param {string} taskName Name of the executed task @@ -999,7 +999,7 @@ export default class ProjectBuildCache { * @param {{gitignore: @ui5/project/build/cache/BuildStageCache~ResourceRequests, * noGitignore: @ui5/project/build/cache/BuildStageCache~ResourceRequests}} [rootResourceRequests] * Resource requests read through the project's root reader, keyed by useGitignore - * @returns {Promise} The resource paths written by the task, + * @returns {Promise} The resource paths written by the stage, * or undefined if caching is disabled */ /** @@ -1219,7 +1219,7 @@ export default class ProjectBuildCache { log.verbose(`Recording results of stage ${stageId} in project ${this.#project.getName()}...`); const stageCache = this.#stageCaches.get(stageId); - // Identify resources written by task + // Identify resources written by stage const stage = this.#project.getProjectResources().getStage(); const stageWriter = stage.getWriter(); const writtenResources = await stageWriter.byGlob("/**/*"); @@ -1246,7 +1246,7 @@ export default class ProjectBuildCache { } // Import the previous stage cache's tag operations into the tag collections so that - // subsequent tasks can access them. Delta builds only record tags set during delta + // subsequent stages can access them. Delta builds only record tags set during delta // execution, so the previous build's tags must be imported explicitly. this.#project.getProjectResources().importTagOperations( cacheInfo.previousStageCache.projectTagOperations, @@ -1263,7 +1263,7 @@ export default class ProjectBuildCache { reader = cacheInfo.previousStageCache.stage.getWriter() ?? cacheInfo.previousStageCache.stage.getCachedWriter(); } - // Paths flagged changed but not re-emitted by the delta task: their source + // Paths flagged changed but not re-emitted by the delta stage: their source // is gone or excluded, so replaying the previous stage's copy would // resurrect content that no longer belongs in the output. The caller passes the effective // list (the verdict's changed paths plus any stale outputs it derived); fall back to the @@ -1283,7 +1283,7 @@ export default class ProjectBuildCache { continue; // Delta re-emitted this path; skip } if (changedPathSet.has(path)) { - // Flagged changed but not written back by the delta task. + // Flagged changed but not written back by the delta stage. // Drop the stale copy from the merge. droppedCount++; continue; @@ -1347,7 +1347,7 @@ export default class ProjectBuildCache { writtenResourcePaths, projectTagOperations, buildTagOperations, this.#stepInvocationData.get(stageId)); - // Update task cache with new metadata + // Update stage cache with new metadata log.verbose(`Stage ${stageId} produced ${writtenResourcePaths.length} resources`); for (const resourcePath of writtenResourcePaths) { @@ -1668,7 +1668,7 @@ export default class ProjectBuildCache { } /** - * Discards the in-memory source index and task caches so the next build re-initializes + * Discards the in-memory source index and stage caches so the next build re-initializes * the source index from scratch via a full byGlob("/**\/*") re-scan (see * {@link #initSourceIndex}), diffing the live source tree against the persisted index. * Recovers from an unreliable incremental change signal (the file watcher dropping @@ -1715,7 +1715,7 @@ export default class ProjectBuildCache { this.#resultCacheState = RESULT_CACHE_STATES.PENDING_VALIDATION; // initSourceIndex does not touch this one, so reset it here. this.#changedDependencyResourcePaths = []; - // Root managers are held on the (now cleared) task caches; drop the remembered aggregate so the + // Root managers are held on the (now cleared) stage caches; drop the remembered aggregate so the // re-initialized caches re-establish it on the next validateCache. this.#cachedRootAggregateSignature = undefined; // Clear per-build state so a failed build does not leak into the next one. @@ -1853,7 +1853,7 @@ export default class ProjectBuildCache { * Initializes the resource index from cache or creates a new one * * This method attempts to load a cached resource index. If found, it validates - * the index against current source files and invalidates affected tasks if + * the index against current source files and invalidates affected stages if * resources have changed. If no cache exists, creates a fresh index. * * @returns {Promise} @@ -1896,14 +1896,14 @@ export default class ProjectBuildCache { // Import stage caches (one entry per stage: a legacy task's single stage, or a step-based // task's per-step stages). const buildStageCaches = await Promise.all( - indexCache.tasks.map(async ([stageId, stepBased]) => { - const projectRequests = this.#cacheManager.readTaskMetadata( + indexCache.stages.map(async ([stageId, stepBased]) => { + const projectRequests = this.#cacheManager.readStageRequestMetadata( this.#project.getId(), this.#buildSignature, stageId, "project"); if (!projectRequests) { throw new Error(`Failed to load project request cache for stage ` + `${stageId} in project ${this.#project.getName()}`); } - const dependencyRequests = this.#cacheManager.readTaskMetadata( + const dependencyRequests = this.#cacheManager.readStageRequestMetadata( this.#project.getId(), this.#buildSignature, stageId, "dependencies"); if (!dependencyRequests) { throw new Error(`Failed to load dependency request cache for stage ` + @@ -1912,14 +1912,14 @@ export default class ProjectBuildCache { // Input metadata (e.g. recorded env-var usage) is optional: absent for stages that // declared no non-resource inputs, and absent in caches written before input // tracking existed. - const inputTree = this.#cacheManager.readTaskMetadata( + const inputTree = this.#cacheManager.readStageRequestMetadata( this.#project.getId(), this.#buildSignature, stageId, "input"); // Root request metadata is optional too: absent for stages that made no root reads, // and absent in caches written before root tracking existed. Kept per useGitignore // flag since the flag changes which resources a recorded glob matches. - const rootRequests = this.#cacheManager.readTaskMetadata( + const rootRequests = this.#cacheManager.readStageRequestMetadata( this.#project.getId(), this.#buildSignature, stageId, "root"); - const rootNoGitignoreRequests = this.#cacheManager.readTaskMetadata( + const rootNoGitignoreRequests = this.#cacheManager.readStageRequestMetadata( this.#project.getId(), this.#buildSignature, stageId, "root-no-gitignore"); return BuildStageCache.fromCache({ projectName: this.#project.getName(), @@ -1939,7 +1939,7 @@ export default class ProjectBuildCache { } // Capture the restored stage order so the result-signature functions have the single source of // truth available before this build's setTasks runs (result-cache validation happens first). - this.#stageOrder = indexCache.tasks.map(([stageId]) => stageId); + this.#stageOrder = indexCache.stages.map(([stageId]) => stageId); // Force mode: Fail if cache is stale (source files changed OR pending changes exist) if (this.#cacheMode === Cache.Force && @@ -2018,8 +2018,8 @@ export default class ProjectBuildCache { * * This method: * 1. Stores the signatures of all stages that lead to the current build result - * 2. Writes all pending task stage caches to persistent storage - * 3. Writes task request metadata to persistent storage + * 2. Writes all pending stage caches to persistent storage + * 3. Writes stage request metadata to persistent storage * 4. Writes the source resource index to persistent storage * * @public @@ -2070,7 +2070,7 @@ export default class ProjectBuildCache { stageId, stageSignature, metadata); } for (const {stageId, type, metadata} of stageRequestPrepared) { - this.#cacheManager.writeTaskMetadata( + this.#cacheManager.writeStageRequestMetadata( this.#project.getId(), this.#buildSignature, stageId, type, metadata); } if (sourceIndexPrepared) { @@ -2117,7 +2117,7 @@ export default class ProjectBuildCache { } /** - * Prepares all pending task stage caches for persistence. + * Prepares all pending stage caches for persistence. * * Gathers resources, computes integrity, gzip-compresses payloads. * @@ -2336,9 +2336,9 @@ export default class ProjectBuildCache { const sourceIndexObject = this.#sourceIndex.toCacheObject(); // One entry per stage in execution order: a legacy task's single stage, or a step-based task's // per-step stages. The stage id is the metadata key everything else is stored under. - const tasks = []; + const stages = []; for (const [stageId, stageCache] of this.#stageCaches) { - tasks.push([stageId, stageCache.getStepBased() ? 1 : 0]); + stages.push([stageId, stageCache.getStepBased() ? 1 : 0]); } return { projectId: this.#project.getId(), @@ -2346,7 +2346,7 @@ export default class ProjectBuildCache { kind: "source", index: { ...sourceIndexObject, - tasks, + stages, availableDependencies: this.#currentDependencySetIdentity, }, }; diff --git a/packages/project/lib/build/cache/ResourceRequestManager.js b/packages/project/lib/build/cache/ResourceRequestManager.js index c9e3711193e..0795bbd013a 100644 --- a/packages/project/lib/build/cache/ResourceRequestManager.js +++ b/packages/project/lib/build/cache/ResourceRequestManager.js @@ -33,9 +33,9 @@ function serializeUnresolvedRequests(entry, unresolvedRequests) { } /** - * Manages resource requests and their associated indices for a single task + * Manages resource requests and their associated indices for a single stage * - * Tracks all resources accessed by a task during execution and maintains resource indices + * Tracks all resources accessed by a stage during execution and maintains resource indices * for cache validation and differential updates. Supports both full and delta-based caching * strategies. * @@ -94,7 +94,7 @@ class ResourceRequestManager { * @param {object} cacheData.requestSetGraph Serialized request graph * @param {Array} cacheData.rootIndices Array of root resource indices * @param {Array} [cacheData.deltaIndices] Array of delta resource indices - * @param {boolean} [cacheData.unusedAtLeastOnce] Whether the task has been unused + * @param {boolean} [cacheData.unusedAtLeastOnce] Whether the stage has been unused * @returns {ResourceRequestManager} Restored manager instance */ static fromCache(projectName, ownerId, useDifferentialUpdate, { @@ -119,7 +119,7 @@ class ResourceRequestManager { const {resourceIndex: parentResourceIndex} = requestGraph.getMetadata(node.getParentId()); const registry = registries.get(node.getParentId()); if (!registry) { - throw new Error(`Missing tree registry for parent of node ID ${nodeId} of task ` + + throw new Error(`Missing tree registry for parent of node ID ${nodeId} of stage ` + `'${ownerId}' of project '${projectName}'`); } const resourceIndex = parentResourceIndex.deriveTreeWithIndex(addedResourceIndex); @@ -133,11 +133,11 @@ class ResourceRequestManager { } /** - * Gets all project index signatures for this task + * Gets all project index signatures for this stage * * Returns signatures from all recorded project-request sets. Each signature represents * a unique combination of resources belonging to the current project that were accessed - * during task execution. This can be used to form cache keys for restoring cached task results. + * during stage execution. This can be used to form cache keys for restoring cached stage results. * * @public * @returns {string[]} Array of signature strings @@ -575,7 +575,7 @@ class ResourceRequestManager { } /** - * Records that a task made no resource requests + * Records that a stage made no resource requests * * Marks the manager as having been unused at least once and returns a special * signature indicating no requests were made. @@ -642,7 +642,7 @@ class ResourceRequestManager { let unresolvedRequests; if (setId) { // Reuse existing resource index. - // Note: This index has already been updated before the task executed, so no update is necessary + // Note: This index has already been updated before the stage executed, so no update is necessary // here, and nothing in the persisted request graph changed: the manager stays clean so the whole // request graph is not needlessly re-serialized to SQLite. (A tree update that moved the index's // signature flags the manager dirty itself in updateIndices.) Recording the same request set on @@ -671,7 +671,7 @@ class ResourceRequestManager { resourceIndex = await parentResourceIndex.deriveTree(resourcesToAdd); // Some added requests may resolve to no resource (e.g. a byPath probe for an // optional file, or every request after a branch switch deletes probed files). - // Those reads still influence task output, so record the unresolved requests to + // Those reads still influence stage output, so record the unresolved requests to // keep the exposed signature distinct from the parent's until they resolve. unresolvedRequests = this.#collectUnresolvedRequests(addedRequests, resourcesToAdd); } else { @@ -699,7 +699,7 @@ class ResourceRequestManager { * With no unresolved requests, returns the underlying tree signature. When some * recorded requests resolved to no resources (byPath probes for files that don't * exist yet, or byGlob patterns matching nothing), those reads still influence - * task output, so the exposed signature must stay distinct from the tree hash of + * stage output, so the exposed signature must stay distinct from the tree hash of * an otherwise-identical index. The signature therefore hashes the tree signature * together with the sorted unresolved keys. * @@ -886,7 +886,7 @@ class ResourceRequestManager { * @returns {object} return.requestSetGraph Serialized request graph * @returns {Array} return.rootIndices Array of root resource indices with node IDs * @returns {Array} return.deltaIndices Array of delta resource indices with node IDs - * @returns {boolean} return.unusedAtLeastOnce Whether the task has been unused + * @returns {boolean} return.unusedAtLeastOnce Whether the stage has been unused */ toCacheObject() { if (!this.#hasNewOrModifiedCacheEntries) { diff --git a/packages/project/test/lib/build/cache/BuildCacheStorage.js b/packages/project/test/lib/build/cache/BuildCacheStorage.js index 7a3447ff52f..f96ad97388f 100644 --- a/packages/project/test/lib/build/cache/BuildCacheStorage.js +++ b/packages/project/test/lib/build/cache/BuildCacheStorage.js @@ -161,29 +161,29 @@ test("Stage metadata: Stage IDs with slashes are stored correctly", (t) => { // ===== Task metadata ===== -test("readTaskMetadata: Returns null on cache miss", (t) => { - const result = t.context.storage.readTaskMetadata("project-a", "sig-1", "minify", "project"); +test("readStageRequestMetadata: Returns null on cache miss", (t) => { + const result = t.context.storage.readStageRequestMetadata("project-a", "sig-1", "minify", "project"); t.is(result, null); }); test("Task metadata: Round-trip write and read", (t) => { const data = {requestSetGraph: {nodes: [], nextId: 1}}; - t.context.storage.writeTaskMetadata("project-a", "sig-1", "minify", "project", data); - const result = t.context.storage.readTaskMetadata("project-a", "sig-1", "minify", "project"); + t.context.storage.writeStageRequestMetadata("project-a", "sig-1", "minify", "project", data); + const result = t.context.storage.readStageRequestMetadata("project-a", "sig-1", "minify", "project"); t.deepEqual(result, data); }); test("Task metadata: Different types are independent", (t) => { const projectData = {scope: "project"}; const depData = {scope: "dependency"}; - t.context.storage.writeTaskMetadata("project-a", "sig-1", "minify", "project", projectData); - t.context.storage.writeTaskMetadata("project-a", "sig-1", "minify", "dependencies", depData); + t.context.storage.writeStageRequestMetadata("project-a", "sig-1", "minify", "project", projectData); + t.context.storage.writeStageRequestMetadata("project-a", "sig-1", "minify", "dependencies", depData); t.deepEqual( - t.context.storage.readTaskMetadata("project-a", "sig-1", "minify", "project"), projectData + t.context.storage.readStageRequestMetadata("project-a", "sig-1", "minify", "project"), projectData ); t.deepEqual( - t.context.storage.readTaskMetadata("project-a", "sig-1", "minify", "dependencies"), depData + t.context.storage.readStageRequestMetadata("project-a", "sig-1", "minify", "dependencies"), depData ); }); @@ -238,12 +238,12 @@ test("transaction: Multiple writes commit atomically", (t) => { const {storage} = t.context; storage.transaction(() => { storage.writeIndexCache("project-a", "sig-1", "source", {v: 1}); - storage.writeTaskMetadata("project-a", "sig-1", "minify", "project", {v: 2}); + storage.writeStageRequestMetadata("project-a", "sig-1", "minify", "project", {v: 2}); storage.writeResultMetadata("project-a", "sig-1", "result-sig-1", {v: 3}); }); t.deepEqual(storage.readIndexCache("project-a", "sig-1", "source"), {v: 1}); - t.deepEqual(storage.readTaskMetadata("project-a", "sig-1", "minify", "project"), {v: 2}); + t.deepEqual(storage.readStageRequestMetadata("project-a", "sig-1", "minify", "project"), {v: 2}); t.deepEqual(storage.readResultMetadata("project-a", "sig-1", "result-sig-1"), {v: 3}); }); @@ -252,13 +252,13 @@ test("transaction: Throwing callback rolls back all writes", (t) => { t.throws(() => { storage.transaction(() => { storage.writeIndexCache("project-a", "sig-1", "source", {v: 1}); - storage.writeTaskMetadata("project-a", "sig-1", "minify", "project", {v: 2}); + storage.writeStageRequestMetadata("project-a", "sig-1", "minify", "project", {v: 2}); throw new Error("boom"); }); }, {message: "boom"}); t.is(storage.readIndexCache("project-a", "sig-1", "source"), null); - t.is(storage.readTaskMetadata("project-a", "sig-1", "minify", "project"), null); + t.is(storage.readStageRequestMetadata("project-a", "sig-1", "minify", "project"), null); }); test("transaction: Combined metadata and content writes commit atomically", (t) => { @@ -440,7 +440,7 @@ test("hasRecords: Returns true when stage metadata table has records", (t) => { }); test("hasRecords: Returns true when task metadata table has records", (t) => { - t.context.storage.writeTaskMetadata("project-a", "build-sig", "minify", "project", {v: 1}); + t.context.storage.writeStageRequestMetadata("project-a", "build-sig", "minify", "project", {v: 1}); t.true(t.context.storage.hasRecords()); }); diff --git a/packages/project/test/lib/build/cache/CacheManager.js b/packages/project/test/lib/build/cache/CacheManager.js index dd89c030446..319951315b8 100644 --- a/packages/project/test/lib/build/cache/CacheManager.js +++ b/packages/project/test/lib/build/cache/CacheManager.js @@ -53,8 +53,8 @@ test.serial("Task metadata: round-trip via CacheManager", async (t) => { const cm = new CacheManager(path.join(testDir, "buildCache")); const data = {requestSetGraph: {nodes: []}}; - cm.writeTaskMetadata("project-x", "build-sig", "minify", "project", data); - const result = cm.readTaskMetadata("project-x", "build-sig", "minify", "project"); + cm.writeStageRequestMetadata("project-x", "build-sig", "minify", "project", data); + const result = cm.readStageRequestMetadata("project-x", "build-sig", "minify", "project"); t.deepEqual(result, data); cm.close(); }); @@ -78,7 +78,7 @@ test.serial("Cache miss returns null for all metadata types", async (t) => { t.is(cm.readIndexCache("no-project", "no-sig", "source"), null); t.is(cm.readStageCache("no-project", "no-sig", "no-stage", "no-sig"), null); - t.is(cm.readTaskMetadata("no-project", "no-sig", "no-task", "project"), null); + t.is(cm.readStageRequestMetadata("no-project", "no-sig", "no-task", "project"), null); t.is(cm.readResultMetadata("no-project", "no-sig", "no-sig"), null); cm.close(); }); diff --git a/packages/project/test/lib/build/cache/ProjectBuildCache.js b/packages/project/test/lib/build/cache/ProjectBuildCache.js index e31a6d55416..7512bdbe03a 100644 --- a/packages/project/test/lib/build/cache/ProjectBuildCache.js +++ b/packages/project/test/lib/build/cache/ProjectBuildCache.js @@ -81,8 +81,8 @@ function createMockCacheManager() { writeStageCache: sinon.stub(), readResultMetadata: sinon.stub().returns(null), writeResultMetadata: sinon.stub(), - readTaskMetadata: sinon.stub().returns(null), - writeTaskMetadata: sinon.stub(), + readStageRequestMetadata: sinon.stub().returns(null), + writeStageRequestMetadata: sinon.stub(), hasContent: sinon.stub().returns(false), readContent: sinon.stub().returns(Buffer.from("test")), readContentRaw: sinon.stub().returns(Buffer.from("test")), @@ -165,11 +165,11 @@ test("Create with existing index cache", async (t) => { } } }, - tasks: [["task/task1", false]] + stages: [["task/task1", false]] }; // Mock task metadata responses - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { + cacheManager.readStageRequestMetadata.callsFake((projectId, buildSig, stageId, type) => { if (type === "project") { return { requestSetGraph: { @@ -319,7 +319,7 @@ test("allTasksCompleted returns changed resource paths", async (t) => { } } }, - tasks: [] + stages: [] }; cacheManager.readIndexCache.returns(indexCache); @@ -969,7 +969,7 @@ test("projectSourcesChanged: marks cache as requiring validation", async (t) => } } }, - tasks: [] + stages: [] }; cacheManager.readIndexCache.returns(indexCache); @@ -1011,7 +1011,7 @@ test("dependencyResourcesChanged: marks cache as requiring validation", async (t } } }, - tasks: [] + stages: [] }; cacheManager.readIndexCache.returns(indexCache); @@ -1074,7 +1074,7 @@ test("projectSourcesChanged after SourceChangedDuringBuildError does not corrupt } } }, - tasks: [] + stages: [] }; cacheManager.readIndexCache.returns(indexCache); @@ -1150,7 +1150,7 @@ test("Retry after SourceChangedDuringBuildError when prior build set NO_CACHE: " } } }, - tasks: [] + stages: [] }; cacheManager.readIndexCache.returns(indexCache); // readResultMetadata returns null by default → #findResultCache returns false → NO_CACHE. @@ -1226,11 +1226,11 @@ test("_refreshDependencyIndices: updates dependency indices", async (t) => { } } }, - tasks: [["task/task1", false]] + stages: [["task/task1", false]] }; // Mock task metadata responses - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { + cacheManager.readStageRequestMetadata.callsFake((projectId, buildSig, stageId, type) => { if (type === "project") { return { requestSetGraph: { @@ -1318,7 +1318,7 @@ test("writeCache: skips writing unchanged caches", async (t) => { } } }, - tasks: [] + stages: [] }; cacheManager.readIndexCache.returns(indexCache); @@ -1738,11 +1738,11 @@ async function buildCacheWithWarmCacheAndTaskResult({ children } }, - tasks: [["task/myTask", 0]] + stages: [["task/myTask", 0]] }; // Mock task metadata for the cached task - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { + cacheManager.readStageRequestMetadata.callsFake((projectId, buildSig, stageId, type) => { if (type === "input") { return null; } @@ -1958,10 +1958,10 @@ test("restoreFrozenSources: cache miss skips gracefully", async (t) => { } } }, - tasks: [["task/task1", false]] + stages: [["task/task1", false]] }; cacheManager.readIndexCache.returns(indexCache); - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { + cacheManager.readStageRequestMetadata.callsFake((projectId, buildSig, stageId, type) => { if (type === "input") { return null; } @@ -2045,10 +2045,10 @@ test("restoreFrozenSources: cache hit creates CAS reader", async (t) => { } } }, - tasks: [["task/task1", false]] + stages: [["task/task1", false]] }; cacheManager.readIndexCache.returns(indexCache); - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { + cacheManager.readStageRequestMetadata.callsFake((projectId, buildSig, stageId, type) => { if (type === "input") { return null; } @@ -2119,7 +2119,7 @@ test("restoreFrozenSources: cache hit creates CAS reader", async (t) => { // (warm cache loaded from disk with task metadata) and spies on _refreshDependencyIndices. async function createCacheInRestoringState({ resources = [createMockResource("/test.js", "hash1", 1000, 100, 1)], - tasks = [["task1", false]], + stages = [["task1", false]], cacheMode, } = {}) { const project = createMockProject(); @@ -2163,10 +2163,10 @@ async function createCacheInRestoringState({ children } }, - tasks + stages }; - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, stageId, type) => { + cacheManager.readStageRequestMetadata.callsFake((projectId, buildSig, stageId, type) => { if (type === "input") { return null; } @@ -2657,13 +2657,13 @@ async function createCacheWithDependencyGlob({ }, }, }, - tasks: [[`task/${taskName}`, false]], + stages: [[`task/${taskName}`, false]], // Persisted dependency-set identity from the previous build. validateCache compares the // identity passed at the next build against this; a mismatch forces the refresh. availableDependencies: oldDependencySetIdentity, }; cacheManager.readIndexCache.returns(indexCache); - cacheManager.readTaskMetadata.callsFake((projectId, buildSig, task, type) => { + cacheManager.readStageRequestMetadata.callsFake((projectId, buildSig, task, type) => { if (type === "dependencies") { return depCacheObject; } From f7daf1b220397b5570b7aee35c9a9e17ace4009b Mon Sep 17 00:00:00 2001 From: Matthias Osswald Date: Thu, 8 Oct 2026 17:00:03 +0200 Subject: [PATCH 8/8] test(project): Assert delta build invalidation on dependency and source-map changes Flip the previously test.failing cases to passing, as the underlying behavior now works: - Minify re-runs and writes the correct delta when only an input source map changes (ProjectBuilder.caching + BuildServer.buildSignature). - A custom task reacts to dependency content changes, so the application is rebuilt (ProjectBuilder.customTasks). The dependency-change fixture gets a determineRequiredDependencies hook so its reads are tracked. - validateCache refreshes dependency indices on a dependency-set change (ProjectBuildCache stale-index cases). Add writtenResources and skippedTasks assertions to pin down the exact delta behavior and guard against regressions. --- .../application.a/task.dependency-change.js | 4 ++ .../BuildServer.buildSignature.integration.js | 29 +++++----- .../ProjectBuilder.caching.integration.js | 35 +++++------- .../ProjectBuilder.customTasks.integration.js | 57 +++++++++++++------ .../test/lib/build/cache/ProjectBuildCache.js | 6 +- 5 files changed, 76 insertions(+), 55 deletions(-) diff --git a/packages/project/test/fixtures/application.a/task.dependency-change.js b/packages/project/test/fixtures/application.a/task.dependency-change.js index 118e74b1cb1..8007b936df7 100644 --- a/packages/project/test/fixtures/application.a/task.dependency-change.js +++ b/packages/project/test/fixtures/application.a/task.dependency-change.js @@ -34,3 +34,7 @@ module.exports = async function ({log, taskUtil, workspace}) { // Start processing dependencies of the root project await processProject(taskUtil.getProject()); }; + +module.exports.determineRequiredDependencies = function ({availableDependencies}) { + return availableDependencies; +} diff --git a/packages/project/test/lib/build/BuildServer.buildSignature.integration.js b/packages/project/test/lib/build/BuildServer.buildSignature.integration.js index 70f94f11b39..e7fe32d18b9 100644 --- a/packages/project/test/lib/build/BuildServer.buildSignature.integration.js +++ b/packages/project/test/lib/build/BuildServer.buildSignature.integration.js @@ -69,17 +69,12 @@ test.serial.failing( "Served resource no longer reflects the stale control value v1"); }); -// Served counterpart of the ProjectBuilder minify source-map staleness test (see +// Served counterpart of the ProjectBuilder minify source-map test (see // ProjectBuilder.caching.integration.js for the full mechanism). Minify reads a resource's input source // map via fsInterface and embeds its content into the `-dbg.js.map` output. That read goes through the // monitored workspace's byPath, so changing ONLY the `.js.map` (not the referencing `.js`) invalidates -// minify's cache and re-runs it in delta mode with the `.js.map` as the sole changed path. But minify -// keeps only changed `.js` paths, so the unchanged `.js` is filtered out, the task writes nothing, and -// the previously served `-dbg.js.map` is carried forward STALE. -// -// This asserts the desired behavior (the changed input map is reflected in the served debug map without -// a server restart) and is marked test.failing because the delta path does not yet achieve it. See the -// minify FIXME for why a fix needs the `.map` -> `.js` relation, not a local pattern tweak. +// minify's cache and re-runs it with the `.js.map` as the changed path. The served `-dbg.js.map` must +// then reflect the new input map content without a server restart. test.serial( "Serve application.a, changing only an input source map read via fs by minify invalidates the debug source map", async (t) => { @@ -99,7 +94,7 @@ test.serial( // Change ONLY the input source map — NOT the referencing scriptWithSourceMap.js. The minify task // read this map via fsInterface, so it is a tracked input and this change invalidates minify's - // cache. But the owning .js is unchanged, so the differential minify path has no .js to reprocess. + // cache. const jsMapContent = await fs.readFile(jsMapFilePath, {encoding: "utf8"}); await fs.writeFile( jsMapFilePath, @@ -112,8 +107,7 @@ test.serial( // #2 request: the served debug source map must reflect the changed input source map content. // The minify task is expected to re-execute here (its cache is invalidated because the changed - // .js.map is a tracked input) — proving the staleness is a differential-execution defect, not a - // missed invalidation. + // .js.map is a tracked input). const second = await fixtureTester.requestResource({ resource: dbgSourceMapResourcePath, assertions: { @@ -127,8 +121,17 @@ test.serial( "enhanceManifest", "generateFlexChangesBundle", "generateVersionInfo" - // "minify" is NOT skipped: it re-runs in differential mode for the changed .js.map - ] + // "minify" is NOT skipped: it re-runs for the changed .js.map + ], + writtenResources: { + // Only resources affected by the source map change should be written + "minify": [ + "/resources/id1/thirdparty/scriptWithSourceMap-dbg.js", + "/resources/id1/thirdparty/scriptWithSourceMap-dbg.js.map", + "/resources/id1/thirdparty/scriptWithSourceMap.js", + "/resources/id1/thirdparty/scriptWithSourceMap.js.map", + ] + } } } } diff --git a/packages/project/test/lib/build/ProjectBuilder.caching.integration.js b/packages/project/test/lib/build/ProjectBuilder.caching.integration.js index fdc4f08cd3d..f768f50631e 100644 --- a/packages/project/test/lib/build/ProjectBuilder.caching.integration.js +++ b/packages/project/test/lib/build/ProjectBuilder.caching.integration.js @@ -358,18 +358,6 @@ test.serial("Build application.a project multiple times", async (t) => { }); }); -// Minify reads a resource's input source map (the `//# sourceMappingURL=` target) via fsInterface and -// embeds its content almost verbatim into the `-dbg.js.map` output, so that debug map is a direct -// function of the input map. The read is a tracked input, so changing ONLY the `.js.map` (not the `.js` -// that references it) invalidates minify's cache and re-runs it in delta mode with the `.js.map` as the -// sole changed path. But minify keeps only changed `.js` paths and reads input maps only as a side -// effect of processing their owning `.js`; the unchanged `.js` is filtered out, so the task writes -// nothing and the previously produced `-dbg.js.map` is carried forward STALE. -// -// This asserts the desired behavior (the changed input map is reflected in the built debug map) and is -// marked test.failing because the delta path does not yet achieve it. See BuildServer.integration.js for -// the same scenario over the served build, and the minify FIXME for why a fix needs the `.map` -> `.js` -// relation, not a local pattern tweak. test.serial( "Build application.a, changing only an input source map read via fs by minify invalidates the debug source map", async (t) => { @@ -389,9 +377,7 @@ test.serial( t.true(firstContent.includes("This is a script with a source map."), "Initial debug source map reflects the original input source map content"); - // Change ONLY the input source map — NOT the referencing scriptWithSourceMap.js. The minify task - // read this map via fsInterface, so it is a tracked input and this change invalidates minify's - // cache. But the owning .js is unchanged, so the differential minify path has no .js to reprocess. + // Change ONLY the input source map — NOT the referencing scriptWithSourceMap.js. const jsMapContent = await fs.readFile(jsMapFilePath, {encoding: "utf8"}); await fs.writeFile( jsMapFilePath, @@ -402,9 +388,7 @@ test.serial( ); // #2 build (with cache, with changes): the built debug source map must reflect the changed input - // source map content. The minify task is expected to re-execute here (its cache is invalidated - // because the changed .js.map is a tracked input) — proving the staleness is a differential- - // execution defect, not a missed invalidation. + // source map content. The minify task is expected to re-execute here await fixtureTester.buildProject({ config: {destPath, cleanDest: true}, assertions: { @@ -418,9 +402,18 @@ test.serial( // replaceCopyright is skipped because no copyright is configured in the project "replaceCopyright", // replaceVersion has no work for the changed .js.map and is skipped - "replaceVersion" - // "minify" is NOT skipped: it re-runs in differential mode for the changed .js.map - ] + "replaceVersion", + // "minify" is NOT skipped: it re-runs for the changed .js.map + ], + writtenResources: { + // Only resources affected by the source map change should be written + "minify": [ + "/resources/id1/thirdparty/scriptWithSourceMap-dbg.js", + "/resources/id1/thirdparty/scriptWithSourceMap-dbg.js.map", + "/resources/id1/thirdparty/scriptWithSourceMap.js", + "/resources/id1/thirdparty/scriptWithSourceMap.js.map", + ] + } } } } diff --git a/packages/project/test/lib/build/ProjectBuilder.customTasks.integration.js b/packages/project/test/lib/build/ProjectBuilder.customTasks.integration.js index 10f2ffafce7..6fcdb5a8b36 100644 --- a/packages/project/test/lib/build/ProjectBuilder.customTasks.integration.js +++ b/packages/project/test/lib/build/ProjectBuilder.customTasks.integration.js @@ -236,7 +236,7 @@ test.serial("Build application.a (multiple custom tasks 2)", async (t) => { }); }); -test.serial.failing("Build application.a (dependency content changes)", async (t) => { +test.serial("Build application.a (dependency content changes)", async (t) => { const fixtureTester = new FixtureTester(t, "application.a"); const destPath = fixtureTester.destPath; @@ -244,14 +244,6 @@ test.serial.failing("Build application.a (dependency content changes)", async (t // modifies application resources based on what it finds. When the dependency content changes, the application // should be rebuilt so the custom task can react to the new dependency state. The assertions below encode that // desired behavior. - // - // Marked test.failing because it currently fails: the custom task accesses dependencies through - // taskUtil.getProject("library.d").getReader() rather than the monitored "dependencies" reader parameter. Reads - // through this path are not tracked by the caching system's ResourceRequestManager, so dependency changes don't - // invalidate the application's result cache. AVA reports a failing-marked test as a pass while it throws and as a - // hard error once it starts passing, so committing it keeps CI green and flips to a signal the moment the behavior - // is fixed (at which point drop the `.failing`). - // Fixing this requires tracking reads made via taskUtil.getProject().getReader() as dependency requests. // #1 build (no cache, no changes, no dependencies) await fixtureTester.buildProject({ @@ -259,7 +251,16 @@ test.serial.failing("Build application.a (dependency content changes)", async (t config: {destPath, cleanDest: true}, assertions: { projects: { - "application.a": {} + "library.d": {}, + "library.a": {}, + "library.b": {}, + "library.c": {}, + "application.a": { + writtenResources: { + // No resources are written as no newLibraryFile.js in library.d exists yet. + "dependency-change": [], + } + } } } }); @@ -285,10 +286,14 @@ test.serial.failing("Build application.a (dependency content changes)", async (t config: {destPath, cleanDest: true, dependencyIncludes: {includeAllDependencies: true}}, assertions: { projects: { - "library.d": {}, - "library.a": {}, - "library.b": {}, - "library.c": {}, + "library.d": { + skippedTasks: [ + "buildThemes", + "enhanceManifest", + "escapeNonAsciiCharacters", + "replaceBuildtime", + ] + }, } } }); @@ -307,17 +312,35 @@ test.serial.failing("Build application.a (dependency content changes)", async (t // and modifies a resource of application.a (namely "test.js"). await fixtureTester.buildProject({ graphConfig: {rootConfigPath: "ui5-customTask-dependency-change.yaml"}, - config: {destPath, cleanDest: true, dependencyIncludes: {includeAllDependencies: true}}, + config: { + destPath, cleanDest: true, dependencyIncludes: {includeAllDependencies: true}, + }, assertions: { projects: { - "library.d": {}, + "library.d": { + skippedTasks: [ + "buildThemes", + "enhanceManifest", + "escapeNonAsciiCharacters", + "replaceBuildtime", + ] + }, "application.a": { skippedTasks: [ "enhanceManifest", "escapeNonAsciiCharacters", + "generateComponentPreload", "generateFlexChangesBundle", + "generateVersionInfo", + "minify", "replaceCopyright", - ] + "replaceVersion", + // dependency-change task is expected to run + ], + writtenResources: { + // /test.js is written when a newLibraryFile.js in the dependency library.d exists. + "dependency-change": ["/test.js"], + } }, } } diff --git a/packages/project/test/lib/build/cache/ProjectBuildCache.js b/packages/project/test/lib/build/cache/ProjectBuildCache.js index 7512bdbe03a..bf8cae4aa7d 100644 --- a/packages/project/test/lib/build/cache/ProjectBuildCache.js +++ b/packages/project/test/lib/build/cache/ProjectBuildCache.js @@ -2578,10 +2578,8 @@ test("Fail-then-succeed: delta merge does not resurrect resources from a stage a // signature the cache reports afterwards is compared against the signature a full refresh against // the new set produces (computed independently via ResourceRequestManager). // -// They are marked test.failing: they assert the *desired* behavior and currently fail because the -// bug is unfixed. AVA reports a failing-marked test as a pass while it throws and as a hard error -// once it starts passing, so committing them keeps CI green and flips to a signal the moment the -// fix lands (at which point drop the `.failing`). +// validateCache now refreshes the dependency indices on a dependency-set change, so these cases +// pass and guard against a regression of the stale-index bug. // Builds a persisted "dependencies" task-metadata object by recording a single glob request // against `oldDepResources`, mirroring what a real build stores for a task that globs dependency