fix(project): Cancel pending settle timer before watcher recovery

packages/project/lib/build/BuildServer.js

@@ -4,6 +4,7 @@ import BuildReader from "./BuildReader.js";
 import WatchHandler from "./helpers/WatchHandler.js";
 import {isAbortError, isFileNotFoundError} from "./helpers/abort.js";
 import {WATCHER_BURST_SETTLE_MS} from "./helpers/watchUtil.js";
+import {trace} from "./helpers/teardownTrace.js";
 import RecoveryBudget, {WATCHER_RECOVERY_MAX_ATTEMPTS, WATCHER_RECOVERY_WINDOW_MS} from "./helpers/RecoveryBudget.js";
 import {getLogger} from "@ui5/logger";
 import ServeLogger from "@ui5/logger/internal/loggers/Serve";
@@ -368,12 +369,15 @@ class BuildServer extends EventEmitter {
 
 
 	async destroy() {
+		trace("BuildServer.destroy: enter");
 		this.#destroyed = true;
 		clearTimeout(this.#processBuildRequestsTimeout);
 		this.#pendingDeferredRestart = false;
 		clearTimeout(this.#sourcesChangedTimeout);
 		this.#pendingFinalSourcesChanged = false;
+		trace("BuildServer.destroy: watchHandler.destroy start");
 		await this.#watchHandler.destroy();
+		trace("BuildServer.destroy: watchHandler.destroy done");
 		try {
 			// Cancel any running background validation pass and wait for it to settle.
 			await this.#stopActiveValidation("Server destroyed");
@@ -385,8 +389,11 @@ class BuildServer extends EventEmitter {
 			// Always release the cache manager, even when the active build rejected
 			// (e.g. Force-mode stale-cache errors). Otherwise the SQLite handle leaks
 			// and subsequent fs.rm of the cache directory fails with EBUSY on Windows.
+			trace("BuildServer.destroy: closeCacheManager start");
 			this.#projectBuilder.closeCacheManager();
+			trace("BuildServer.destroy: closeCacheManager done");
 		}
+		trace("BuildServer.destroy: exit");
 	}
 
 	/**

packages/project/lib/build/cache/BuildCacheStorage.js

@@ -3,6 +3,7 @@ import {mkdirSync, existsSync} from "node:fs";
 import path from "node:path";
 import {gzipSync, gunzipSync} from "node:zlib";
 import {getLogger} from "@ui5/logger";
+import {trace} from "../helpers/teardownTrace.js";
 
 const log = getLogger("build:cache:BuildCacheStorage");
 
@@ -603,15 +604,19 @@ export default class BuildCacheStorage {
 	 * Closes the database connection
 	 */
 	close() {
+		trace(`BuildCacheStorage.close: enter (${this.#dbPath})`);
 		if (this.#inTransaction) {
 			try {
 				this.#db.exec("ROLLBACK");
 			} finally {
 				this.#inTransaction = false;
 			}
 		}
+		trace("BuildCacheStorage.close: wal_checkpoint(TRUNCATE) start");
 		this.#db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+		trace("BuildCacheStorage.close: wal_checkpoint done, db.close() start");
 		this.#db.close();
+		trace("BuildCacheStorage.close: db.close() done");
 	}
 
 	/**

packages/project/lib/build/cache/CacheManager.js

@@ -4,6 +4,7 @@ import Configuration from "../../config/Configuration.js";
 import {access} from "node:fs/promises";
 import {getLogger} from "@ui5/logger";
 import BuildCacheStorage from "./BuildCacheStorage.js";
+import {trace} from "../helpers/teardownTrace.js";
 
 const log = getLogger("build:cache:CacheManager");
 
@@ -335,9 +336,12 @@ export default class CacheManager {
 	 * closed only when the last consumer releases its reference.
 	 */
 	close() {
+		trace(`CacheManager.close: refCount ${this.#refCount} -> ${this.#refCount - 1} (${this.#cacheDir})`);
 		if (--this.#refCount <= 0) {
+			trace("CacheManager.close: last ref, closing storage");
 			this.#storage.close();
 			cacheManagerInstances.delete(this.#cacheDir);
+			trace("CacheManager.close: storage closed, instance removed");
 		}
 	}
 

packages/project/lib/build/helpers/WatchHandler.js

@@ -2,6 +2,7 @@ import EventEmitter from "node:events";
 import {getLogger} from "@ui5/logger";
 import {subscribe as watchSubscribe} from "./fileWatcher.js";
 import {drainSubscriptions} from "./watchUtil.js";
+import {trace} from "./teardownTrace.js";
 import {exists} from "../../utils/fsHelper.js";
 const log = getLogger("build:helpers:WatchHandler");
 
@@ -63,15 +64,18 @@ class WatchHandler extends EventEmitter {
 	}
 
 	async destroy() {
+		trace("WatchHandler.destroy: enter");
 		// Drain the subscriptions list so a second destroy() is a no-op and a partial
 		// failure cannot leave stale handles behind to be unsubscribed twice.
 		const subscriptions = this.#subscriptions;
 		this.#subscriptions = [];
 		const failures = await drainSubscriptions(subscriptions);
+		trace("WatchHandler.destroy: drained");
 		if (failures.length) {
 			const err = new AggregateError(failures, "Failed to unsubscribe one or more file watchers");
 			this.emit("error", err);
 		}
+		trace("WatchHandler.destroy: exit");
 	}
 
 	#handleWatchEvents(eventType, filePath, project) {

packages/project/lib/build/helpers/fileWatcher.js

@@ -1,5 +1,7 @@
 import {existsSync, readFileSync} from "node:fs";
+import os from "node:os";
 import {getLogger} from "@ui5/logger";
+import {trace} from "./teardownTrace.js";
 
 const log = getLogger("build:helpers:fileWatcher");
 
@@ -43,6 +45,102 @@ let usePolling = null;
 let nativeBackend = null;
 let nativeBackendLoaded = false;
 
+// Serialization chain for native subscribe/unsubscribe. @parcel/watcher mutates a process-global
+// backend registry from both the JS thread (subscribe: find/emplace/rehash) and a libuv worker
+// thread (unsubscribe of the last subscriber: erase/rehash), with no lock guarding that static map
+// (parcel-bundler/watcher#259). Funneling every native subscribe and unsubscribe through one promise
+// chain means the process never issues two overlapping registry mutations from the JS thread. This
+// pairs with the keep-alive below: the keep-alive prevents the destructive empty-transition rehash,
+// and serializing removes the remaining same-thread overlap so subscribe fan-out and teardown drain
+// cannot interleave their native calls. The chain is process-wide because the registry it protects
+// is process-global; it only orders watcher-lifecycle calls (rare, at startup/teardown), so it costs
+// nothing on the hot path.
+let nativeWatcherChain = Promise.resolve();
+
+// Runs fn after every previously-chained native watcher operation has settled, and extends the chain
+// so the next one waits for fn. Rejections are isolated so one failed operation does not wedge the
+// chain, while the returned promise still rejects for its own caller.
+function serializeNativeWatcherOp(fn) {
+	const result = nativeWatcherChain.then(fn, fn);
+	nativeWatcherChain = result.then(() => undefined, () => undefined);
+	return result;
+}
+
+// Keep-alive subscription that pins @parcel/watcher's shared backend so its global registry never
+// empties while a serving session is active. The destructive half of parcel-bundler/watcher#259 is
+// the empty transition: when the last subscriber for a backend is removed, removeShared() runs
+// erase() + rehash(0) on a libuv worker thread, and that rehash(0) races a concurrent subscribe from
+// the JS thread — the exact shape of one reinitialize() swap fully tearing down its watchers before
+// the next stack subscribes. Serializing our own subscribe/unsubscribe calls cannot fence this,
+// because the worker-thread erase can run after our unsubscribe promise has already resolved.
+//
+// Holding one keep-alive subscription across the session keeps the registry size above zero, so
+// rehash(0) never fires during the churn. It is released only when the last session ends
+// (pinCount 0): by then nothing is serving, so no concurrent subscribe can race the final
+// registry-empty, and releasing it lets the process (and AVA's test worker) drain the native handle
+// and exit cleanly. The tmpdir target always exists and the "**" ignore drops every event, so the
+// keep-alive delivers nothing and does no work beyond existing.
+let keepAlivePromise = null;
+let backendPinCount = 0;
+
+/**
+ * Pins the native watcher backend alive for the duration of a serving session, so its shared
+ * registry never empties mid-session (see keepAlivePromise). Balanced by {@link unpinBackend}. A
+ * no-op under the polling backend, which has no such registry. Awaiting the returned promise ensures
+ * the keep-alive is in place before the first real subscribe of the session.
+ *
+ * @returns {Promise<void>} Resolves once the keep-alive is established (or skipped)
+ */
+export async function pinBackend() {
+	if (shouldUsePolling()) {
+		return;
+	}
+	const native = await loadNativeBackend();
+	if (!native) {
+		return;
+	}
+	backendPinCount++;
+	keepAlivePromise ??= serializeNativeWatcherOp(() => {
+		trace("fileWatcher: backend keep-alive subscribe start");
+		return native.subscribe(os.tmpdir(), () => {}, {ignore: ["**"]});
+	}).then((subscription) => {
+		trace("fileWatcher: backend keep-alive subscribe done");
+		return subscription;
+	}, (err) => {
+		// A failed keep-alive must not fail the session: it is an optimization, not a requirement.
+		// Without it we simply fall back to the empty-transition race being possible again, so log and
+		// carry on rather than rejecting the caller.
+		log.verbose(`Watcher backend keep-alive could not start: ${err?.message ?? err}`);
+		return null;
+	});
+	await keepAlivePromise;
+}
+
+/**
+ * Releases a {@link pinBackend} reference. When the last session ends, the keep-alive subscription is
+ * torn down so the native handle no longer holds the event loop open. Safe to over-call; a release
+ * without a matching pin is a no-op.
+ *
+ * @returns {Promise<void>} Resolves once the keep-alive has been released (or on the balancing call)
+ */
+export async function unpinBackend() {
+	if (backendPinCount === 0) {
+		return;
+	}
+	if (--backendPinCount > 0) {
+		return;
+	}
+	const pending = keepAlivePromise;
+	keepAlivePromise = null;
+	const subscription = await pending;
+	if (subscription) {
+		await serializeNativeWatcherOp(() => {
+			trace("fileWatcher: backend keep-alive unsubscribe");
+			return subscription.unsubscribe();
+		});
+	}
+}
+
 /**
  * Decides whether to poll, once per process. <code>UI5_WATCH_MODE=polling|native</code> forces the
  * choice; otherwise polling is the default inside a container and the native backend is the default
@@ -114,7 +212,19 @@ export async function subscribe(dir, callback, opts = {}) {
 	if (!shouldUsePolling()) {
 		const native = await loadNativeBackend();
 		if (native) {
-			return native.subscribe(dir, callback, opts);
+			// Serialize against every other native subscribe/unsubscribe: see nativeWatcherChain. The
+			// shared backend is pinned for the session via pinBackend(), so the registry does not empty
+			// between this subscribe and a concurrent teardown.
+			const subscription = await serializeNativeWatcherOp(() => {
+				trace(`fileWatcher.subscribe: native subscribe start (${dir})`);
+				return native.subscribe(dir, callback, opts);
+			});
+			trace(`fileWatcher.subscribe: native subscribe done (${dir})`);
+			// Route unsubscribe through the same chain so a teardown never overlaps a subscribe (or
+			// another unsubscribe) on the shared registry.
+			return {
+				unsubscribe: () => serializeNativeWatcherOp(() => subscription.unsubscribe()),
+			};
 		}
 		// The native binding could not load (see loadNativeBackend). Polling needs no native code, so
 		// fall through to it rather than failing the watch.

packages/project/lib/build/helpers/teardownTrace.js

@@ -0,0 +1,28 @@
+import {writeSync} from "node:fs";
+
+// Synchronous, env-gated teardown tracer. Set UI5_TEARDOWN_TRACE=1 to enable.
+//
+// Writes straight to fd 2 (stderr) with writeSync, bypassing the async stream buffer, so
+// the last line survives a hard native crash (a 0xC0000005 access violation on Windows in
+// @parcel/watcher or node:sqlite teardown terminates the process without flushing buffered
+// output). Each line is prefixed with the high-resolution time and the pid so interleaved
+// teardown across the Supervisor, BuildServer, and watchers can be ordered after the fact.
+//
+// Off by default and a no-op unless the env var is set, so it costs nothing in normal runs.
+const ENABLED = process.env.UI5_TEARDOWN_TRACE === "1";
+
+/**
+ * Emits a teardown trace line to stderr synchronously when UI5_TEARDOWN_TRACE=1.
+ *
+ * @param {string} msg Message to trace
+ */
+export function trace(msg) {
+	if (!ENABLED) {
+		return;
+	}
+	try {
+		writeSync(2, `[teardown ${process.hrtime.bigint()} pid=${process.pid}] ${msg}\n`);
+	} catch {
+		// Never let tracing throw during teardown.
+	}
+}

packages/project/lib/build/helpers/watchUtil.js

@@ -1,3 +1,5 @@
+import {trace} from "./teardownTrace.js";
+
 /**
  * Settle window (ms) for collapsing a burst of filesystem events into one trailing action. Shared
  * by every Parcel watcher in the build layer.
@@ -13,11 +15,20 @@
  */
 export const WATCHER_BURST_SETTLE_MS = 550;
 
+
 /**
- * Unsubscribes every subscription in parallel and returns the failures. Callers drain their list to
+ * Unsubscribes every subscription and returns the failures. Callers drain their list to
  * <code>[]</code> before calling, so a second drain is a no-op and a partial failure cannot leave
- * stale handles to be unsubscribed twice. Running in parallel and collecting failures keeps one
- * misbehaving subscription from taking down the others.
+ * stale handles to be unsubscribed twice. Each <code>unsubscribe()</code> is attempted even if an
+ * earlier one rejects, so one misbehaving subscription cannot leave the others subscribed.
+ *
+ * Unsubscribes run <b>sequentially</b>, not in parallel: <code>@parcel/watcher</code> has a data
+ * race on its global shared-backend registry between subscribe and unsubscribe
+ * (parcel-bundler/watcher#259). Firing every <code>unsubscribe()</code> at once raced that registry
+ * and access-violated (<code>0xC0000005</code>) mid-teardown on Windows, where all subscriptions
+ * share one backend thread. Draining one at a time keeps each native teardown from overlapping the
+ * next. The list is small (one subscription per watched directory) and this only runs at teardown,
+ * so the lost concurrency does not matter.
  *
  * @private
  * @param {object[]} subscriptions Subscriptions to drain, each exposing an async
@@ -26,6 +37,20 @@ export const WATCHER_BURST_SETTLE_MS = 550;
  *   when all succeeded
  */
 export async function drainSubscriptions(subscriptions) {
-	const results = await Promise.allSettled(subscriptions.map((s) => s.unsubscribe()));
-	return results.filter((r) => r.status === "rejected").map((r) => r.reason);
+	trace(`drainSubscriptions: draining ${subscriptions.length} subscription(s)`);
+	const failures = [];
+	let i = 0;
+	for (const subscription of subscriptions) {
+		trace(`drainSubscriptions: unsubscribe #${i} start`);
+		try {
+			await subscription.unsubscribe();
+			trace(`drainSubscriptions: unsubscribe #${i} done`);
+		} catch (err) {
+			trace(`drainSubscriptions: unsubscribe #${i} threw: ${err?.message ?? err}`);
+			failures.push(err);
+		}
+		i++;
+	}
+	trace(`drainSubscriptions: all drained`);
+	return failures;
 }

packages/project/lib/graph/ProjectDefinitionWatcher.js

@@ -3,6 +3,7 @@ import path from "node:path";
 import {getLogger} from "@ui5/logger";
 import {subscribe as watchSubscribe} from "../build/helpers/fileWatcher.js";
 import {drainSubscriptions, WATCHER_BURST_SETTLE_MS} from "../build/helpers/watchUtil.js";
+import {trace} from "../build/helpers/teardownTrace.js";
 import RecoveryBudget, {
 	WATCHER_RECOVERY_MAX_ATTEMPTS, WATCHER_RECOVERY_WINDOW_MS,
 } from "../build/helpers/RecoveryBudget.js";
@@ -230,13 +231,13 @@ class ProjectDefinitionWatcher extends EventEmitter {
 		}
 
 		try {
+			this.#cancelSettleTimer();
+
 			// Tear down the current subscriptions and re-subscribe the same watch set. The include
 			// set (#watchedFiles / #watchDirs) is unchanged; only the OS-level handles are renewed.
 			// Teardown failures are ignored here: the handles are discarded either way, and the
 			// re-subscribe below is what decides whether recovery succeeded.
-			const subscriptions = this.#subscriptions;
-			this.#subscriptions = [];
-			await drainSubscriptions(subscriptions);
+			await this.#drainSubscriptions();
 			if (this.#destroyed) {
 				return;
 			}
@@ -258,20 +259,33 @@ class ProjectDefinitionWatcher extends EventEmitter {
 	 * @returns {Promise<void>} Resolves once every subscription has been drained
 	 */
 	async destroy() {
+		trace("ProjectDefinitionWatcher.destroy: enter");
 		this.#destroyed = true;
+		this.#cancelSettleTimer();
+		const failures = await this.#drainSubscriptions();
+		trace("ProjectDefinitionWatcher.destroy: drained");
+		if (failures.length) {
+			const err = new AggregateError(failures, "Failed to unsubscribe one or more definition watchers");
+			this.emit("error", err);
+		}
+		trace("ProjectDefinitionWatcher.destroy: exit");
+	}
+
+	// Cancels a pending settle timer, if any. Safe to call when no timer is armed.
+	#cancelSettleTimer() {
 		if (this.#settleTimer) {
 			clearTimeout(this.#settleTimer);
 			this.#settleTimer = null;
 		}
-		// Drain the subscriptions list first so a second destroy() is a no-op and a partial failure
-		// cannot leave stale handles to be unsubscribed twice.
+	}
+
+	// Snapshots and clears the subscriptions list before draining it, so a second drain (a second
+	// destroy(), or a destroy() racing recovery) is a no-op and a partial failure cannot leave stale
+	// handles to be unsubscribed twice. Returns the unsubscribe failures for callers that report them.
+	async #drainSubscriptions() {
 		const subscriptions = this.#subscriptions;
 		this.#subscriptions = [];
-		const failures = await drainSubscriptions(subscriptions);
-		if (failures.length) {
-			const err = new AggregateError(failures, "Failed to unsubscribe one or more definition watchers");
-			this.emit("error", err);
-		}
+		return drainSubscriptions(subscriptions);
 	}
 }
 

packages/project/package.json

@@ -32,6 +32,7 @@
 		"./validation/ValidationError": "./lib/validation/ValidationError.js",
 		"./graph/ProjectGraph": "./lib/graph/ProjectGraph.js",
 		"./internal/graph/ProjectDefinitionWatcher": "./lib/graph/ProjectDefinitionWatcher.js",
+		"./internal/build/helpers/fileWatcher": "./lib/build/helpers/fileWatcher.js",
 		"./graph/projectGraphBuilder": "./lib/graph/projectGraphBuilder.js",
 		"./graph": "./lib/graph/graph.js",
 		"./package.json": "./package.json"