From 753c98118c7e1266e0bdf1654cb4cc368ed531ac Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 22 Jul 2026 00:27:16 -0700 Subject: [PATCH 01/18] Add opt-in QuickJS WASM VM engine (WORKFLOW_VM=quickjs) with full event replay --- .changeset/quickjs-vm-engine.md | 6 + .github/workflows/tests.yml | 13 +- .../docs/v5/configuration/runtime-tuning.mdx | 12 + packages/core/.gitignore | 7 + packages/core/package.json | 3 +- packages/core/scripts/build-quickjs-assets.js | 73 + .../core/scripts/build-vm-serde-bundle.js | 65 + packages/core/src/runtime.ts | 31 + .../core/src/runtime/quickjs-entrypoint.ts | 814 +++++++++++ .../core/src/runtime/quickjs-runtime.test.ts | 557 ++++++++ packages/core/src/runtime/quickjs-runtime.ts | 1255 +++++++++++++++++ packages/core/src/runtime/start.ts | 9 + packages/core/src/runtime/vm-mode.test.ts | 109 ++ packages/core/src/runtime/vm-mode.ts | 75 + .../src/serialization/codec-devalue-vm.ts | 93 ++ .../core/src/serialization/compat.test.ts | 185 +++ .../src/serialization/reducers/common-vm.ts | 542 +++++++ .../core/src/serialization/vm-bundle-entry.ts | 62 + .../src/serialization/workflow-vm.test.ts | 186 +++ .../core/src/serialization/workflow-vm.ts | 73 + packages/core/src/source-map.test.ts | 47 +- packages/core/src/source-map.ts | 26 + .../src/telemetry/semantic-conventions.ts | 30 + packages/core/turbo.json | 7 +- pnpm-lock.yaml | 8 + pnpm-workspace.yaml | 1 + scripts/create-test-matrix.mjs | 17 + 27 files changed, 4302 insertions(+), 4 deletions(-) create mode 100644 .changeset/quickjs-vm-engine.md create mode 100644 packages/core/scripts/build-quickjs-assets.js create mode 100644 packages/core/scripts/build-vm-serde-bundle.js create mode 100644 packages/core/src/runtime/quickjs-entrypoint.ts create mode 100644 packages/core/src/runtime/quickjs-runtime.test.ts create mode 100644 packages/core/src/runtime/quickjs-runtime.ts create mode 100644 packages/core/src/runtime/vm-mode.test.ts create mode 100644 packages/core/src/runtime/vm-mode.ts create mode 100644 packages/core/src/serialization/codec-devalue-vm.ts create mode 100644 packages/core/src/serialization/compat.test.ts create mode 100644 packages/core/src/serialization/reducers/common-vm.ts create mode 100644 packages/core/src/serialization/vm-bundle-entry.ts create mode 100644 packages/core/src/serialization/workflow-vm.test.ts create mode 100644 packages/core/src/serialization/workflow-vm.ts diff --git a/.changeset/quickjs-vm-engine.md b/.changeset/quickjs-vm-engine.md new file mode 100644 index 0000000000..77d2fc39e4 --- /dev/null +++ b/.changeset/quickjs-vm-engine.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': minor +'workflow': minor +--- + +Add an experimental QuickJS WASM VM engine for workflow execution, opt-in via `WORKFLOW_VM=quickjs` (or per-run `executionContext.workflowVm`). The engine performs the same full event replay as the default `node:vm` engine but runs workflow code in a QuickJS VM compiled to WebAssembly, enabling platforms without `node:vm` support and laying the groundwork for VM-memory snapshotting. diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f45b94544a..070f1e8abb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -247,10 +247,16 @@ jobs: APP_NAME: "nextjs-turbopack" vitest-plugin: - name: Vitest Plugin Tests + name: Vitest Plugin Tests (${{ matrix.vm }}) runs-on: ubuntu-latest needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' }} + strategy: + fail-fast: false + matrix: + # Workflow VM engines: node:vm (default) and the opt-in QuickJS + # WASM engine (WORKFLOW_VM=quickjs). + vm: [node, quickjs] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} @@ -271,6 +277,8 @@ jobs: - name: Run Vitest Plugin Tests run: pnpm test working-directory: workbench/vitest + env: + WORKFLOW_VM: ${{ matrix.vm }} e2e-package-build: name: Build Shared E2E Packages @@ -682,6 +690,7 @@ jobs: DEV_TEST_CONFIG: ${{ toJSON(matrix.app) }} WORKFLOW_DEV_HMR_LOGS: "1" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} + WORKFLOW_VM: ${{ matrix.app.vm || '' }} - name: Generate E2E summary if: always() @@ -770,6 +779,7 @@ jobs: WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:${{ matrix.app.name == 'sveltekit' && '4173' || (matrix.app.name == 'astro' && '4321' || '3000') }}" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} + WORKFLOW_VM: ${{ matrix.app.vm || '' }} - name: Generate E2E summary if: always() @@ -878,6 +888,7 @@ jobs: WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:${{ matrix.app.name == 'sveltekit' && '4173' || (matrix.app.name == 'astro' && '4321' || '3000') }}" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} + WORKFLOW_VM: ${{ matrix.app.vm || '' }} - name: Generate E2E summary if: always() diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 1236f5e160..68f166ccd2 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -109,6 +109,18 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - How long after an inline step's latest `step_started` other invocations assume its owner may still be executing the body. Within the lease they defer the step's backstop message; past it they enqueue immediately. - Raise this on self-hosted multi-instance deployments whose inline steps run longer than the default (the default is sized for Vercel's function duration ceiling). +## Workflow VM engine + +### `WORKFLOW_VM` + +- Default: `node` +- Values: `node` or `quickjs` +- Selects the sandboxed VM engine that executes workflow functions (`"use workflow"`). Step functions are unaffected — they always run with full Node.js access. +- `node` (default) runs workflow code in a [`node:vm`](https://nodejs.org/api/vm.html) context. +- `quickjs` (experimental) runs workflow code in a [QuickJS](https://github.com/quickjs-ng/quickjs) VM compiled to WebAssembly (via [`quickjs-wasi`](https://github.com/vercel-labs/quickjs-wasi)). Both engines implement the same event-replay execution model and are interchangeable per run. The QuickJS engine is intended for platforms that do not implement `node:vm`, and is the foundation for future VM-memory snapshotting. +- The engine choice is stamped into the run's `executionContext` when the run starts, so a run keeps executing on the engine it started on even if the deployment's `WORKFLOW_VM` changes. Runs without a stamped engine use the handler's `WORKFLOW_VM` value. +- Unknown values throw at startup. + ## Compression and tracing ### `WORKFLOW_DISABLE_COMPRESSION` diff --git a/packages/core/.gitignore b/packages/core/.gitignore index 7b6d0b4576..51bcd14a90 100644 --- a/packages/core/.gitignore +++ b/packages/core/.gitignore @@ -1,2 +1,9 @@ # Auto-generated version file src/version.ts + +# Auto-generated quickjs-wasi binary assets (base64-encoded WASM + .so files) +src/runtime/quickjs-assets.generated.ts + +# Auto-generated VM serde bundle (devalue + format-prefix + reducers, +# packaged as an ES-module string for evaluation inside the QuickJS VM) +src/runtime/vm-serde-bundle.generated.ts diff --git a/packages/core/package.json b/packages/core/package.json index 7d7e0a978b..38e460033a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -80,7 +80,7 @@ "./_workflow": "./dist/workflow/index.js" }, "scripts": { - "build": "genversion --es6 src/version.ts && tsc", + "build": "genversion --es6 src/version.ts && node scripts/build-vm-serde-bundle.js && node scripts/build-quickjs-assets.js && tsc", "dev": "genversion --es6 src/version.ts && tsc --watch", "clean": "tsc --build --clean && rm -rf dist src/version.ts docs ||:", "test": "cross-env WORKFLOW_TARGET_WORLD=local vitest run src", @@ -105,6 +105,7 @@ "devalue": "5.8.1", "ms": "2.1.3", "nanoid": "5.1.6", + "quickjs-wasi": "3.1.0", "seedrandom": "3.0.5", "semver": "catalog:", "ulid": "catalog:", diff --git a/packages/core/scripts/build-quickjs-assets.js b/packages/core/scripts/build-quickjs-assets.js new file mode 100644 index 0000000000..807589fbb6 --- /dev/null +++ b/packages/core/scripts/build-quickjs-assets.js @@ -0,0 +1,73 @@ +/** + * Build script: generates quickjs-assets.generated.ts + * + * Reads the quickjs-wasi WASM binary and native C extension .so files, + * base64-encodes them, and writes a TypeScript module that exports the + * decoded Buffer/Uint8Array values. This embeds the binaries directly + * in JavaScript, bypassing all bundler/framework/deployment issues with + * import.meta.url, require.resolve, and file tracing. + * + * quickjs-wasi >= 3.0.0 exposes the binaries via package subpath exports + * (`quickjs-wasi/quickjs.wasm`, `quickjs-wasi/.so`), which is what + * we resolve here. Note: `btoa`/`atob` and the TC39 Uint8Array base64/hex + * methods are built into the core runtime since 3.x, so there is no + * `base64` extension anymore. + */ + +import { readFileSync, writeFileSync } from 'fs'; +import { createRequire } from 'module'; +import { dirname, resolve } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const srcDir = resolve(__dirname, '../src'); + +const require_ = createRequire(import.meta.url); + +const files = { + quickjsWasm: require_.resolve('quickjs-wasi/quickjs.wasm'), + encodingSo: require_.resolve('quickjs-wasi/encoding.so'), + headersSo: require_.resolve('quickjs-wasi/headers.so'), + urlSo: require_.resolve('quickjs-wasi/url.so'), + structuredCloneSo: require_.resolve('quickjs-wasi/structured-clone.so'), +}; + +let output = `/** + * Auto-generated by scripts/build-quickjs-assets.js + * Do not edit manually. + * + * Contains base64-encoded quickjs-wasi WASM binary and native C extension + * .so files. Decoded at import time so they can be passed directly to + * QuickJS.create() and QuickJS.restore() without any filesystem access, + * import.meta.url resolution, or require.resolve calls. + */ +import type { ExtensionDescriptor } from 'quickjs-wasi'; + +`; + +let totalSize = 0; + +for (const [name, filePath] of Object.entries(files)) { + const buf = readFileSync(filePath); + const b64 = buf.toString('base64'); + totalSize += buf.length; + output += `const ${name} = Buffer.from('${b64}', 'base64');\n\n`; +} + +output += `export { quickjsWasm };\n\n`; + +output += `export const quickjsExtensions: ExtensionDescriptor[] = [ + { name: 'encoding', wasm: encodingSo }, + { name: 'headers', wasm: headersSo }, + { name: 'url', wasm: urlSo }, + { name: 'structured-clone', wasm: structuredCloneSo, initFn: 'qjs_ext_structured_clone_init' }, +];\n`; + +const outPath = resolve(srcDir, 'runtime/quickjs-assets.generated.ts'); +writeFileSync(outPath, output); + +const sizeKB = (totalSize / 1024).toFixed(0); +const b64SizeKB = (Buffer.byteLength(output) / 1024).toFixed(0); +console.log( + `Generated quickjs-assets.generated.ts (${sizeKB} KB binary → ${b64SizeKB} KB base64)` +); diff --git a/packages/core/scripts/build-vm-serde-bundle.js b/packages/core/scripts/build-vm-serde-bundle.js new file mode 100644 index 0000000000..f738853b9a --- /dev/null +++ b/packages/core/scripts/build-vm-serde-bundle.js @@ -0,0 +1,65 @@ +/** + * Build script: generates the VM serialization bundle. + * + * Uses esbuild to bundle workflow-vm.ts into a self-contained IIFE. + * The output is written as a TypeScript file containing the bundle as + * a string constant, which can be imported by the snapshot runtime. + * + * TextEncoder, TextDecoder, and Headers are provided by native C + * extensions in quickjs-wasi, so no JS polyfills are needed. + */ + +import { buildSync } from 'esbuild'; +import { writeFileSync } from 'fs'; +import { dirname, resolve } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const srcDir = resolve(__dirname, '../src'); + +const result = buildSync({ + entryPoints: [resolve(srcDir, 'serialization/vm-bundle-entry.ts')], + // NOTE: TextEncoder, TextDecoder, and Headers are provided by native + // C extensions (encoding, headers) in quickjs-wasi, so the polyfill + // injection that was previously here has been removed. + bundle: true, + format: 'iife', + platform: 'neutral', + target: 'es2020', + write: false, + minify: true, +}); + +const bundleCode = result.outputFiles[0].text; + +// Write as a TS module using a template literal. Template literals avoid +// the escaping issues that occur with regular string literals — esbuild's +// minifier produces patterns like `typeof x<"u"` whose escaped quotes +// inside a JSON-stringified string break when downstream esbuild (e.g., +// Nitro) re-processes the compiled JS output. Template literals don't +// have this problem since backticks don't conflict with inner quotes. +const escaped = bundleCode + .replace(/\\/g, '\\\\') + .replace(/`/g, '\\`') + .replace(/\$\{/g, '\\${'); + +const outPath = resolve(srcDir, 'runtime/vm-serde-bundle.generated.ts'); +writeFileSync( + outPath, + `/** + * Auto-generated by scripts/build-vm-serde-bundle.js + * Do not edit manually. + * + * This is the VM serialization bundle — a self-contained IIFE that sets up + * serialize/deserialize + TextEncoder/TextDecoder polyfills inside the + * QuickJS WASM VM. It includes devalue and all workflow-mode reducers. + * + * Size: ${(bundleCode.length / 1024).toFixed(1)} KB minified + */ +export const VM_SERDE_BUNDLE: string = \`${escaped}\`; +` +); + +console.log( + `Generated vm-serde-bundle.generated.ts (${(bundleCode.length / 1024).toFixed(1)} KB)` +); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 53f5ea80ab..dd06ca975a 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -70,6 +70,7 @@ import { queueMessage, withHealthCheck, } from './runtime/helpers.js'; +import { runWorkflowWithQuickJS } from './runtime/quickjs-entrypoint.js'; import { handleReplayBudgetExhausted, ReplayBudget, @@ -80,6 +81,7 @@ import { DEFAULT_STEP_MAX_RETRIES, executeStep, } from './runtime/step-executor.js'; +import { useQuickJSVm } from './runtime/vm-mode.js'; import { computeStepLatencyTracking } from './runtime/step-latency.js'; import { backstopIdempotencyKey, @@ -1605,6 +1607,35 @@ export function workflowEntrypoint( } // end else (non-turbo run_started) } // end if (!workflowRun) + // --- QuickJS VM engine dispatch --- + // The QuickJS engine (opt-in via WORKFLOW_VM=quickjs or + // executionContext.workflowVm) is a self-contained + // alternative to the node:vm inline-replay loop below. + // It performs the same full event replay, but runs the + // workflow code in a QuickJS WASM VM, queues steps via + // the same combined route (so they hit executeStep below + // on re-entry), and manages its own run_completed / + // run_failed lifecycle. When the QuickJS engine is in + // effect, return immediately after dispatch. + if (useQuickJSVm(workflowRun)) { + runtimeLogger.debug('Using QuickJS VM engine', { + workflowRunId: runId, + loopIteration, + }); + const quickjsResult = await runWorkflowWithQuickJS({ + workflowCode, + workflowName, + workflowRun, + preloadedEvents, + runInput, + parentSpan: span, + }); + if (quickjsResult?.timeoutSeconds !== undefined) { + return { timeoutSeconds: quickjsResult.timeoutSeconds }; + } + return; + } + // Resolve the encryption key for this run's deployment. // Used eagerly here since both runWorkflow (input // hydration / hook payload decryption) and the run_failed diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts new file mode 100644 index 0000000000..ea538a13b2 --- /dev/null +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -0,0 +1,814 @@ +/** + * QuickJS VM integration with the Workflow DevKit. + * + * This module provides the entry point for running workflows in the + * QuickJS WASM VM engine instead of the `node:vm` engine. Both engines + * implement the same event-replay execution model — every invocation: + * + * 1. Loads the full event log for the run + * 2. Runs the workflow function from the top in a fresh QuickJS VM, + * replaying the event log to resolve awaited primitives + * 3. On suspension: creates events + queues steps for new pending ops + * 4. On completion: creates run_completed + * 5. On failure: creates run_failed + */ + +import type { Span } from '@opentelemetry/api'; +import { + EntityConflictError, + RunExpiredError, + WorkflowNotRegisteredError, +} from '@workflow/errors'; +import { parseWorkflowName } from '@workflow/utils/parse-name'; +import { + type Event, + type RunInput, + SPEC_VERSION_CURRENT, + type WorkflowRun, +} from '@workflow/world'; +import { classifyRunError } from '../classify-error.js'; +import { importKey } from '../encryption.js'; +import { runtimeLogger } from '../logger.js'; +import { + dehydrateRunError, + hydrateRunError, + maybeEncrypt, +} from '../serialization.js'; +import { encrypt as encryptSerializedData } from '../serialization/encryption.js'; +import { remapErrorStack, stripInlineSourceMap } from '../source-map.js'; +import * as Attribute from '../telemetry/semantic-conventions.js'; +import { serializeTraceCarrier } from '../telemetry.js'; +import { getPortLazy } from './get-port-lazy.js'; +import { getWorkflowQueueName, queueMessage } from './helpers.js'; +import { + type PendingAttribute, + type PendingHook, + type PendingStep, + type PendingWait, + runQuickJSWorkflow, +} from './quickjs-runtime.js'; +import { getWorld } from './world.js'; + +/** Tiny ms timer using performance.now() — already monotonic on Node. */ +function tick(): number { + return performance.now(); +} + +/** + * Returns true when the supplied preloaded events indicate this is the + * first workflow handler invocation for the run — i.e. the log contains + * nothing beyond `run_created` / `run_started`. In that case the + * preloaded events ARE the complete event log and the `events.list` + * round-trips can be skipped entirely. + * + * Crucially, if the world backfilled a missing `run_created` via the + * resilient start path, `preloadedEvents` contains it even when a fresh + * `events.list` might not (eventual consistency), so preferring the + * preloaded events on first invocation is also the more correct choice. + * + * Returns false when `preloadedEvents` is missing/empty so the caller + * falls back to the normal fetch path. + * + * Exported for unit testing. + */ +export function isFirstInvocation( + preloadedEvents: readonly Event[] | undefined +): boolean { + if (!Array.isArray(preloadedEvents) || preloadedEvents.length === 0) { + return false; + } + return preloadedEvents.every( + (e) => e.eventType === 'run_created' || e.eventType === 'run_started' + ); +} + +/** + * Run a workflow using the QuickJS WASM VM engine. + * + * This replaces the `node:vm` replay path (runWorkflow + EventsConsumer) + * with a QuickJS VM invocation that performs the same full event replay. + */ +export async function runWorkflowWithQuickJS(params: { + workflowCode: string; + workflowName: string; + workflowRun: WorkflowRun; + /** + * Events returned inline by `events.create('run_started', ...)`. When + * they indicate a first invocation, they are used as the event log + * instead of fetching via `events.list`, matching the node:vm engine's + * fast path. + */ + preloadedEvents?: Event[]; + /** + * Run input carried through the queue message on first delivery. Used + * as a last-resort fallback for `run_created.eventData.input` when + * the event log is incomplete. + */ + runInput?: RunInput; + /** + * The parent OTel span (the outer `WORKFLOW {workflowName}` span from + * `runtime.ts`). When supplied, VM lifecycle attributes are attached + * to it for end-to-end visibility. + */ + parentSpan?: Span; +}): Promise<{ timeoutSeconds?: number } | void> { + const { + workflowCode, + workflowName, + workflowRun, + preloadedEvents, + runInput, + parentSpan, + } = params; + const world = await getWorld(); + const runId = workflowRun.runId; + const invocationStart = tick(); + + // Strip the inline source map comment before evaluating the bundle in + // the QuickJS VM. The map is purely host-side metadata for + // `remapErrorStack` (called below on workflow failures, against the + // ORIGINAL `workflowCode`). QuickJS retains source text for + // stack-trace line lookups, so the few-MB base64 comment would bloat + // the VM heap for no benefit. + const workflowCodeForVM = stripInlineSourceMap(workflowCode); + + // Per-invocation diagnostic id so debug logs can be correlated even if + // the same runId is processed by overlapping invocations on different + // function instances. + const invocationId = `inv_${Math.random().toString(36).slice(2, 10)}`; + + // Structured per-checkpoint diagnostic helper, grep-friendly by runId. + const wfdiag = (checkpoint: string, fields: Record) => { + runtimeLogger.debug('QUICKJS_VM_DIAG', { + checkpoint, + runId, + invocationId, + tElapsedMs: Math.round(tick() - invocationStart), + ...fields, + }); + }; + + parentSpan?.setAttributes({ + ...Attribute.WorkflowVm('quickjs'), + }); + + wfdiag('enter', { + workflowName, + hasPreloadedEvents: + Array.isArray(preloadedEvents) && preloadedEvents.length > 0, + preloadedEventCount: preloadedEvents?.length ?? 0, + hasRunInput: !!runInput, + }); + + // The workflowName from the queue topic is already the full workflow ID + // (e.g. "workflow//./workflows/1_simple//simple") + const workflowId = workflowName; + + // Resolve the encryption key up front — needed to decrypt event + // payloads inside the VM and to encrypt event payloads written below. + const rawKey = await world.getEncryptionKeyForRun?.(workflowRun); + const encryptionKey = rawKey ? await importKey(rawKey) : undefined; + + // Load the FULL event log for the run. On first invocation the + // preloaded events from the run_started response are the complete log + // and save the events.list round-trips. + let events: Event[]; + let eventsFetchedPages = 0; + const usePreloaded = isFirstInvocation(preloadedEvents); + if (usePreloaded && preloadedEvents) { + events = preloadedEvents; + } else { + const allEvents: Event[] = []; + let cursor: string | null = null; + let hasMore = true; + + while (hasMore) { + const response = await world.events.list({ + runId, + pagination: { + sortOrder: 'asc', + cursor: cursor ?? undefined, + limit: 1000, + }, + }); + eventsFetchedPages++; + allEvents.push(...response.data); + // Update the cursor to the last successfully fetched page's cursor. + // Only update when we got results — the final empty-page response + // returns cursor=null which we must NOT use (it would reset the cursor). + if (response.cursor) { + cursor = response.cursor; + } + hasMore = response.data.length > 0 && response.cursor != null; + } + + events = allEvents; + } + + parentSpan?.setAttributes({ + ...Attribute.QuickJSEventsPreloaded(usePreloaded), + ...Attribute.QuickJSEventsFetchedCount(events.length), + ...Attribute.QuickJSEventsFetchedPages(eventsFetchedPages), + }); + + wfdiag('events_fetched', { + eventCount: events.length, + eventsFetchedPages, + usePreloaded, + eventTypes: events.reduce>((acc, e) => { + acc[e.eventType] = (acc[e.eventType] ?? 0) + 1; + return acc; + }, {}), + }); + + // Check for elapsed waits + const now = Date.now(); + const completedWaitIds = new Set( + events + .filter((e) => e.eventType === 'wait_completed') + .map((e) => e.correlationId) + ); + for (const event of events) { + if ( + event.eventType === 'wait_created' && + event.correlationId && + !completedWaitIds.has(event.correlationId) + ) { + const eventData = + 'eventData' in event + ? (event.eventData as Record) + : undefined; + const resumeAt = eventData?.resumeAt; + if (resumeAt && now >= new Date(resumeAt as string).getTime()) { + try { + const result = await world.events.create(runId, { + eventType: 'wait_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: event.correlationId, + }); + if (result.event) events.push(result.event); + } catch (err) { + if (EntityConflictError.is(err)) continue; + throw err; + } + } + } + } + + // Resolve the workflow server port so `getWorkflowMetadata().url` inside + // the VM matches what the step-side handler reports. Skipped on Vercel — + // the VM reads VERCEL_URL directly in that environment. + const isVercel = process.env.VERCEL_URL !== undefined; + const port = isVercel ? undefined : await getPortLazy(); + + // Run the workflow in the QuickJS VM + runtimeLogger.debug('QuickJS runtime: invoking VM', { + workflowRunId: runId, + workflowId, + eventCount: events.length, + }); + + const result = await runQuickJSWorkflow({ + // Pass the STRIPPED bundle to the VM so the inline source map + // doesn't end up in the QuickJS heap. The original (unstripped) + // `workflowCode` is still kept in this host-side scope and is used + // by `remapErrorStack` on workflow failures below. + workflowCode: workflowCodeForVM, + workflowId, + workflowRun, + events, + encryptionKey, + port, + runInput, + }); + + runtimeLogger.debug('QuickJS runtime: VM returned', { + workflowRunId: runId, + completed: !!result.completed, + suspended: !!result.suspended, + failed: !!result.failed, + pendingOpsCount: result.suspended?.pendingOperations?.length, + }); + + wfdiag('vm_returned', { + outcome: result.completed + ? 'completed' + : result.suspended + ? 'suspended' + : result.failed + ? 'failed' + : 'unknown', + pendingOpsCount: result.suspended?.pendingOperations?.length ?? 0, + pendingOpSummary: result.suspended?.pendingOperations?.map((p) => ({ + type: p.type, + correlationId: p.correlationId, + hasCreatedEvent: p.hasCreatedEvent, + ...(p.type === 'step' ? { stepId: (p as PendingStep).stepId } : {}), + })), + failureMessage: result.failed?.message, + failureName: result.failed?.name, + }); + + if (result.completed) { + // Workflow completed + runtimeLogger.info('QuickJS runtime: workflow completed', { + workflowRunId: runId, + }); + parentSpan?.setAttributes({ + ...Attribute.QuickJSOutcome('completed'), + }); + + // Create run_completed event. + // The VM serializes the workflow result as format-prefixed devalue bytes + // ("devl" + devalue) with no encryption (the VM has no access to the + // CryptoKey). Host-side encryption is applied here so that `run_completed` + // events have the same `encr`-prefixed payload shape that the node:vm + // engine's `dehydrateWorkflowReturnValue` produces. + try { + await world.events.create(runId, { + eventType: 'run_completed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + output: await encryptSerializedData( + result.completed.result, + encryptionKey + ), + }, + }); + wfdiag('exit_completed', { result: 'run_completed_written' }); + } catch (err) { + if (EntityConflictError.is(err) || RunExpiredError.is(err)) { + runtimeLogger.warn( + 'Workflow already finished, skipping run_completed', + { workflowRunId: runId } + ); + wfdiag('exit_completed', { result: 'already_finished' }); + return; + } + wfdiag('exit_completed_error', { + errorName: (err as Error)?.name, + errorMessage: (err as Error)?.message, + }); + throw err; + } + } else if (result.suspended) { + // Workflow suspended + const { pendingOperations } = result.suspended; + + runtimeLogger.info('QuickJS runtime: workflow suspended', { + workflowRunId: runId, + pendingSteps: pendingOperations.filter((p) => p.type === 'step').length, + pendingWaits: pendingOperations.filter((p) => p.type === 'wait').length, + pendingOps: pendingOperations.map((p) => ({ + type: p.type, + correlationId: p.correlationId, + hasCreatedEvent: p.hasCreatedEvent, + ...(p.type === 'step' + ? { + stepId: (p as PendingStep).stepId, + inputType: typeof (p as PendingStep).input, + inputIsUint8Array: (p as PendingStep).input instanceof Uint8Array, + } + : {}), + })), + }); + + parentSpan?.setAttributes({ + ...Attribute.QuickJSOutcome('suspended'), + ...Attribute.QuickJSPendingOpsCount(pendingOperations.length), + }); + + // Build per-pending-op promises so events.create + queueMessage + // calls fan out in parallel rather than serially. This mirrors + // the node:vm engine's `Promise.all(ops)` pattern in + // suspension-handler.ts and significantly reduces wall-clock time + // on cloud worlds (e.g. Vercel) where each storage call is a + // network round-trip. + let minTimeoutSeconds: number | undefined; + // Set when a new attr_set event is written this invocation. The + // workflow must be re-invoked to consume it (resolving the pending + // setAttributes() promise), so the entrypoint requeues immediately — + // same pattern as an elapsed wait. + let createdAttributeEvent = false; + const opsPromises: Promise[] = []; + + for (const op of pendingOperations) { + if (op.type === 'step' && !op.hasCreatedEvent) { + const step = op as PendingStep; + opsPromises.push( + (async () => { + // Create step_created event. `step.input` is the + // format-prefixed devalue bytes ("devl" + devalue) produced + // by `globalThis[Symbol.for('workflow-serialize')]({args, + // closureVars, thisVal})` inside the VM. The VM has no + // access to the CryptoKey, so encryption is applied here + // on the host side — matching what + // `dehydrateStepArguments` does in the node:vm engine. + try { + await world.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: step.correlationId, + eventData: { + stepName: step.stepId, + input: await encryptSerializedData(step.input, encryptionKey), + }, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + + // Queue the step execution via the unified workflow queue + // (V2 architecture). The combined handler in runtime.ts + // dispatches messages with `stepId` to executeStep, which + // works for both VM engines — so the QuickJS engine reuses + // the same step execution path as the node:vm engine + // instead of needing a separate step route. + const traceCarrier = await serializeTraceCarrier(); + await queueMessage( + world, + getWorkflowQueueName(workflowRun.workflowName), + { + runId, + stepId: step.correlationId, + stepName: step.stepId, + traceCarrier, + requestedAt: new Date(), + }, + { + idempotencyKey: step.correlationId, + } + ); + wfdiag('step_queued', { + stepId: step.stepId, + correlationId: step.correlationId, + }); + })() + ); + } else if (op.type === 'hook' && !op.hasCreatedEvent) { + const hook = op as PendingHook; + runtimeLogger.debug('QuickJS runtime: creating hook_created event', { + workflowRunId: runId, + correlationId: hook.correlationId, + token: hook.token, + tokenType: typeof hook.token, + isWebhook: hook.isWebhook, + }); + + opsPromises.push( + (async () => { + // `hook.metadata` is the format-prefixed devalue bytes + // produced by `globalThis[Symbol.for('workflow-serialize')] + // (options.metadata)` inside the VM. Encrypt on the host + // side before writing — matches the node:vm engine's + // `dehydrateStepArguments` flow. + // + // No pre-check via hooks.list: with deterministic correlationIds + // (same VM seed across replays) and per-(runId, correlationId) + // uniqueness in worlds, the storage layer rejects duplicates as + // EntityConflictError, which we swallow below. This drops one + // network round-trip per pending hook. + try { + const encryptedMetadata = + typeof hook.metadata === 'undefined' + ? undefined + : await encryptSerializedData(hook.metadata, encryptionKey); + const result = await world.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: hook.correlationId, + eventData: { + token: hook.token, + metadata: encryptedMetadata, + // Always include isWebhook explicitly. Worlds default it to + // `true` when absent, which would break the public webhook + // endpoint's 404 guard for hooks created via createHook(). + isWebhook: hook.isWebhook, + } as any, + }); + + // If storage detected a real token conflict with another + // workflow's hook, re-queue so the workflow handler can + // process the conflict event and fail gracefully. + if (result.event?.eventType === 'hook_conflict') { + await queueMessage( + world, + `__wkf_workflow_${workflowRun.workflowName}`, + { + runId, + }, + { idempotencyKey: `hook_conflict_${hook.correlationId}` } + ); + } + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } else if (op.type === 'attribute' && !op.hasCreatedEvent) { + const attr = op as PendingAttribute; + opsPromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'attr_set', + specVersion: SPEC_VERSION_CURRENT, + correlationId: attr.correlationId, + eventData: { + changes: attr.changes, + writer: { type: 'workflow' }, + ...(attr.allowReservedAttributes + ? { allowReservedAttributes: true } + : {}), + } as any, + }); + createdAttributeEvent = true; + } catch (err) { + if (EntityConflictError.is(err)) { + // Event already exists (concurrent invocation) — the + // replay still needs to consume it, so requeue. + createdAttributeEvent = true; + return; + } + throw err; + } + })() + ); + } else if (op.type === 'hook_dispose' && !op.hasCreatedEvent) { + opsPromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'hook_disposed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: op.correlationId, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } else if (op.type === 'wait' && !op.hasCreatedEvent) { + const wait = op as PendingWait; + opsPromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'wait_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: wait.correlationId, + eventData: { + resumeAt: new Date(wait.resumeAt), + }, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } + } + + // Per-op dispatch runs in parallel. + await Promise.all(opsPromises); + + // Handle pending waits — both newly created and still-pending from + // earlier invocations. For each wait, either create a wait_completed + // event (if elapsed) or schedule a timeout for re-queuing. + let needsRequeue = false; + const waitCompletePromises: Promise[] = []; + for (const op of pendingOperations) { + if (op.type !== 'wait') continue; + const wait = op as PendingWait; + const resumeMs = new Date(wait.resumeAt).getTime() - Date.now(); + + if (resumeMs <= 0) { + // Wait has elapsed — create wait_completed and re-queue. + waitCompletePromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'wait_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: wait.correlationId, + }); + needsRequeue = true; + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } else { + // Wait hasn't elapsed yet — schedule a timeout + const timeoutSeconds = Math.max(1, Math.ceil(resumeMs / 1000)); + if ( + minTimeoutSeconds === undefined || + timeoutSeconds < minTimeoutSeconds + ) { + minTimeoutSeconds = timeoutSeconds; + } + } + } + if (waitCompletePromises.length > 0) { + await Promise.all(waitCompletePromises); + } + + if (needsRequeue || createdAttributeEvent) { + // An elapsed wait was completed or a new attr_set event was + // written — re-queue immediately so the next invocation can + // process the new event. + wfdiag('exit_suspended', { + action: needsRequeue ? 'wait_elapsed_requeue' : 'attr_set_requeue', + timeoutSeconds: 0, + }); + return { timeoutSeconds: 0 }; + } + + if (minTimeoutSeconds !== undefined) { + wfdiag('exit_suspended', { + action: 'schedule_wait_timeout', + timeoutSeconds: minTimeoutSeconds, + }); + return { timeoutSeconds: minTimeoutSeconds }; + } + + wfdiag('exit_suspended', { + action: 'awaiting_external', + pendingOpsCount: pendingOperations.length, + }); + } else if (result.failed) { + // Workflow failed — remap stack trace using inline source maps + let errorStack = result.failed.stack; + if (errorStack) { + const parsedName = parseWorkflowName(workflowName); + const filename = parsedName?.moduleSpecifier || workflowName; + errorStack = remapErrorStack(errorStack, filename, workflowCode); + } + + // Classify the error so consumers (`run.returnValue`, observability) + // get `USER_ERROR` / `RUNTIME_ERROR` on `error.cause.code`, matching + // what the node:vm engine already does in runtime.ts. + // + // The VM serializes errors as `{ name, message, stack }`, so we + // reconstruct a host-side Error of the correct class based on the + // VM-side `name` — specific WorkflowRuntimeError subclasses need + // to be preserved so classifyRunError() tags them as RUNTIME_ERROR. + const reconstructed: Error = + result.failed.name === 'WorkflowNotRegisteredError' + ? new WorkflowNotRegisteredError(workflowName) + : result.failed.name === 'Error' + ? new Error(result.failed.message) + : Object.assign(new Error(result.failed.message), { + name: result.failed.name, + }); + const errorCode = classifyRunError(reconstructed); + + runtimeLogger.error('QuickJS runtime: workflow failed', { + workflowRunId: runId, + errorName: result.failed.name, + errorMessage: result.failed.message, + errorStack, + errorCode, + }); + parentSpan?.setAttributes({ + ...Attribute.QuickJSOutcome('failed'), + }); + + // Create run_failed event. Serialize the error through the + // first-class dehydration pipeline so consumers (CLI, observability, + // run.returnValue) get the same hydrated value shape as the node:vm + // engine emits. Two paths: + // * Modern (valueBytes present): the VM-side rejection handler + // serialized the original thrown value (Error subclass with + // cause chain, plain object, primitive, etc.) using the VM's + // workflow-serialize. Pass those bytes through directly so + // type identity, cause chains, and non-Error throws survive. + // We just need to apply encryption if configured (the VM's + // serializer doesn't have access to the encryption key). + // * Legacy fallback: reconstruct an Error from the host-visible + // {name, message, stack} fields and run it through + // `dehydrateRunError`. Used when valueBytes is absent (e.g. + // extractError pseudo-failures from VM bootstrap). + let dehydratedError: Uint8Array; + if (result.failed.valueBytes) { + // Hydrate the VM-side bytes, remap the error stack with the + // host-side source map (the VM can't do this — it lacks both the + // source map and `remapErrorStack`), and re-dehydrate. This + // preserves the original value's type identity / cause chain + // while fixing up frames to point at the user's source files. + try { + const hydrated = await hydrateRunError( + result.failed.valueBytes, + runId, + undefined // VM bytes are unencrypted + ); + if ( + hydrated && + typeof hydrated === 'object' && + 'stack' in (hydrated as object) && + typeof (hydrated as { stack?: unknown }).stack === 'string' + ) { + const parsedName = parseWorkflowName(workflowName); + const filename = parsedName?.moduleSpecifier || workflowName; + (hydrated as { stack?: string }).stack = remapErrorStack( + (hydrated as { stack: string }).stack, + filename, + workflowCode + ); + } + // Walk the cause chain and remap nested stacks too. + const seen = new WeakSet(); + let node = (hydrated as { cause?: unknown })?.cause; + while (node && typeof node === 'object' && !seen.has(node as object)) { + seen.add(node as object); + const nodeStack = (node as { stack?: unknown }).stack; + if (typeof nodeStack === 'string') { + const parsedName = parseWorkflowName(workflowName); + const filename = parsedName?.moduleSpecifier || workflowName; + (node as { stack?: string }).stack = remapErrorStack( + nodeStack, + filename, + workflowCode + ); + } + node = (node as { cause?: unknown }).cause; + } + dehydratedError = await dehydrateRunError( + hydrated, + runId, + encryptionKey + ); + } catch (rehydrateErr) { + // If hydration / re-dehydration fails for any reason, fall + // back to passing through the original VM bytes (just apply + // encryption if configured). Better to lose source-mapped + // frames than to lose the error entirely. + runtimeLogger.warn( + 'QuickJS runtime: failed to remap workflow error stack, passing VM bytes through', + { + workflowRunId: runId, + message: (rehydrateErr as Error)?.message, + } + ); + dehydratedError = (await maybeEncrypt( + result.failed.valueBytes, + encryptionKey + )) as Uint8Array; + } + } else { + if (errorStack) { + reconstructed.stack = errorStack; + } + try { + dehydratedError = await dehydrateRunError( + reconstructed, + runId, + encryptionKey + ); + } catch (serErr) { + // Fall back to a minimal payload so the run still terminates + // even when the error itself contains unserializable values. + runtimeLogger.warn( + 'QuickJS runtime: failed to dehydrate run error, falling back to bare Error', + { workflowRunId: runId, message: (serErr as Error)?.message } + ); + dehydratedError = await dehydrateRunError( + Object.assign(new Error(result.failed.message), { + name: result.failed.name, + }), + runId, + encryptionKey + ); + } + } + try { + await world.events.create(runId, { + eventType: 'run_failed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + error: dehydratedError, + errorCode, + }, + }); + } catch (err) { + if (EntityConflictError.is(err) || RunExpiredError.is(err)) { + runtimeLogger.warn('Workflow already finished, skipping run_failed', { + workflowRunId: runId, + }); + wfdiag('exit_failed', { result: 'already_finished' }); + return; + } + wfdiag('exit_failed_error', { + errorName: (err as Error)?.name, + errorMessage: (err as Error)?.message, + }); + throw err; + } + wfdiag('exit_failed', { result: 'run_failed_written' }); + } +} diff --git a/packages/core/src/runtime/quickjs-runtime.test.ts b/packages/core/src/runtime/quickjs-runtime.test.ts new file mode 100644 index 0000000000..920d0c61db --- /dev/null +++ b/packages/core/src/runtime/quickjs-runtime.test.ts @@ -0,0 +1,557 @@ +import { describe, expect, it } from 'vitest'; +import { deserialize, serialize } from '../serialization/workflow-vm.js'; +import { runQuickJSWorkflow } from './quickjs-runtime.js'; + +/** Helper to deserialize the format-prefixed result bytes */ +function unwrapResult(result: Uint8Array): unknown { + return deserialize(result); +} + +/** + * A realistic full event log always begins with run_created (carrying the + * serialized workflow arguments). Replay invocations require it — the + * runtime fails loud when other events are present without it. + */ +function runCreatedEvent(run: { runId: string }, args: unknown[] = []) { + return { + eventId: 'evnt_run_created', + runId: run.runId, + eventType: 'run_created' as const, + eventData: { input: serialize(args) }, + // Must not be later than any other event in the log — event + // timestamps drive the VM's monotonic deterministic clock. + createdAt: new Date('2025-01-01T00:00:00Z'), + }; +} + +function makeRun(overrides: Record = {}) { + return { + runId: 'wrun_test123', + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: undefined, + status: 'running' as const, + output: undefined, + error: undefined, + completedAt: undefined, + startedAt: new Date('2025-01-01T00:00:00Z'), + createdAt: new Date('2025-01-01T00:00:00Z'), + updatedAt: new Date('2025-01-01T00:00:00Z'), + specVersion: 2, + ...overrides, + }; +} + +describe('runQuickJSWorkflow', () => { + it('should run a simple workflow with no steps to completion', async () => { + const result = await runQuickJSWorkflow({ + workflowCode: ` + globalThis.__private_workflows = new Map(); + async function hello() { return 42; } + hello.workflowId = "workflow//test//hello"; + globalThis.__private_workflows.set("workflow//test//hello", hello); + `, + workflowId: 'workflow//test//hello', + workflowRun: makeRun(), + events: [], + }); + + expect(result.completed).toBeDefined(); + expect(unwrapResult(result.completed!.result)).toBe(42); + }); + + it('should suspend on first step and return pending operations', async () => { + const result = await runQuickJSWorkflow({ + workflowCode: ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { + var a = await add(10, 7); + return a; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [], + }); + + expect(result.suspended).toBeDefined(); + expect(result.suspended?.pendingOperations).toHaveLength(1); + expect(result.suspended?.pendingOperations[0]).toMatchObject({ + type: 'step', + stepId: 'step//test//add', + }); + expect(result.suspended?.pendingOperations[0].correlationId).toMatch( + /^step_[0-9A-Z]{26}$/ + ); + }); + + it('should complete after step resolves via full event replay', async () => { + const code = ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { return await add(10, 7); } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + const run = makeRun(); + + const r1 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + expect(r1.suspended).toBeDefined(); + const stepCid = r1.suspended!.pendingOperations[0].correlationId; + + // Resumption = fresh VM + FULL event log. The workflow re-executes + // from the top, regenerates the same correlationId (seeded PRNG + + // fixed ULID timestamp), and the recorded step_completed event + // resolves the re-created pending promise. + const r2 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [ + runCreatedEvent(run), + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'step_created', + correlationId: stepCid, + eventData: { stepName: 'step//test//add' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_002', + runId: run.runId, + eventType: 'step_completed', + correlationId: stepCid, + eventData: { result: 17 }, + createdAt: new Date(), + }, + ], + }); + + expect(unwrapResult(r2.completed!.result)).toBe(17); + }); + + it('should handle multi-step workflows across replay invocations', async () => { + const code = ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { + var a = await add(10, 7); + var b = await add(a, 8); + return b; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + const run = makeRun(); + + const r1 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + const step1Cid = r1.suspended?.pendingOperations[0]?.correlationId; + expect(step1Cid).toMatch(/^step_[0-9A-Z]{26}$/); + + const step1Events = [ + runCreatedEvent(run), + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'step_created' as const, + correlationId: step1Cid!, + eventData: { stepName: 'step//test//add' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_002', + runId: run.runId, + eventType: 'step_completed' as const, + correlationId: step1Cid!, + eventData: { result: 17 }, + createdAt: new Date(), + }, + ]; + + const r2 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: step1Events, + }); + // The replayed step 1 is settled (its events exist); only the newly + // reached step 2 is pending. + expect(r2.suspended?.pendingOperations).toHaveLength(1); + const step2Cid = r2.suspended?.pendingOperations[0]?.correlationId; + expect(step2Cid).toMatch(/^step_[0-9A-Z]{26}$/); + expect(step2Cid).not.toBe(step1Cid); + + const r3 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [ + ...step1Events, + { + eventId: 'evnt_003', + runId: run.runId, + eventType: 'step_created' as const, + correlationId: step2Cid!, + eventData: { stepName: 'step//test//add' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_004', + runId: run.runId, + eventType: 'step_completed' as const, + correlationId: step2Cid!, + eventData: { result: 25 }, + createdAt: new Date(), + }, + ], + }); + expect(unwrapResult(r3.completed!.result)).toBe(25); + }); + + it('should handle sleep suspension and wake', async () => { + const code = ` + async function workflow() { + await globalThis[Symbol.for("WORKFLOW_SLEEP")]("5s"); + return "woke up"; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + const run = makeRun(); + + const r1 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + expect(r1.suspended).toBeDefined(); + expect(r1.suspended?.pendingOperations[0]).toMatchObject({ + type: 'wait', + }); + const waitCid = r1.suspended!.pendingOperations[0].correlationId; + expect(waitCid).toMatch(/^wait_[0-9A-Z]{26}$/); + + const r2 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [ + runCreatedEvent(run), + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'wait_created', + correlationId: waitCid, + eventData: { resumeAt: new Date() }, + createdAt: new Date(), + }, + { + eventId: 'evnt_002', + runId: run.runId, + eventType: 'wait_completed', + correlationId: waitCid, + createdAt: new Date(), + }, + ], + }); + expect(unwrapResult(r2.completed!.result)).toBe('woke up'); + }); + + it('should handle step failure with try/catch in workflow', async () => { + const code = ` + var fail = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//fail"); + async function workflow() { + try { await fail(); return "nope"; } + catch (e) { return "caught: " + e.message; } + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + const run = makeRun(); + + const r1 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + expect(r1.suspended).toBeDefined(); + + const failStepCid = r1.suspended!.pendingOperations[0].correlationId; + + const r2 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [ + runCreatedEvent(run), + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'step_failed', + correlationId: failStepCid, + eventData: { error: { message: 'boom' } }, + createdAt: new Date(), + }, + ], + }); + expect(unwrapResult(r2.completed!.result)).toBe('caught: boom'); + }); +}); + +describe('correlationId determinism', () => { + // Full event replay REQUIRES deterministic correlationIds: every + // invocation re-executes the workflow from the top and must regenerate + // the exact same ids so that pending operations re-created by replay + // match the events recorded by earlier invocations. Identical ids + // across CONCURRENT invocations of the same run are also load-bearing — + // both produce the same ids, and the world's per-(runId, correlationId) + // uniqueness turns the duplicate `events.create` into an + // EntityConflictError that the entrypoint swallows. + // + // Mechanism: a deterministic `__ulidTimestamp` (workflowRun.startedAt) + // pins the ULID timestamp portion, and the PRNG is seeded with + // `runId:name:startedAt` so the random portion is identical across + // invocations of the same run. + + const stepWorkflow = ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { return await add(10, 7); } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + + const twoStepWorkflow = ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { + var a = await add(10, 7); + var b = await add(a, 8); + return b; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + + it('produces identical correlationIds for two concurrent first-run invocations', async () => { + const run = makeRun(); + const [r1, r2] = await Promise.all([ + runQuickJSWorkflow({ + workflowCode: stepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }), + runQuickJSWorkflow({ + workflowCode: stepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }), + ]); + + expect(r1.suspended!.pendingOperations[0].correlationId).toBe( + r2.suspended!.pendingOperations[0].correlationId + ); + }); + + it('produces identical correlationIds for two concurrent replay invocations', async () => { + const run = makeRun(); + + // Drive the workflow to its first suspension to learn step 1's id. + const r1 = await runQuickJSWorkflow({ + workflowCode: twoStepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + const step1Cid = r1.suspended!.pendingOperations[0].correlationId; + + const events = [ + runCreatedEvent(run), + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'step_created' as const, + correlationId: step1Cid, + eventData: { stepName: 'step//test//add' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_002', + runId: run.runId, + eventType: 'step_completed' as const, + correlationId: step1Cid, + eventData: { result: 17 }, + createdAt: new Date(), + }, + ]; + + // Two concurrent replays of the same event log must both re-derive + // the same id for the newly reached step 2. + const [ra, rb] = await Promise.all([ + runQuickJSWorkflow({ + workflowCode: twoStepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events, + }), + runQuickJSWorkflow({ + workflowCode: twoStepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events, + }), + ]); + + expect(ra.suspended!.pendingOperations[0].correlationId).toBe( + rb.suspended!.pendingOperations[0].correlationId + ); + expect(ra.suspended!.pendingOperations[0].correlationId).not.toBe(step1Cid); + }); + + it('regenerates the same correlationId for an already-recorded step on replay', async () => { + const run = makeRun(); + + const r1 = await runQuickJSWorkflow({ + workflowCode: stepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + const stepCid = r1.suspended!.pendingOperations[0].correlationId; + + // Replay with only the step_created event (step not yet completed). + // The re-executed workflow must regenerate the SAME id so the + // pending op is recognized as already created (hasCreatedEvent) and + // is not re-dispatched by the entrypoint. + const r2 = await runQuickJSWorkflow({ + workflowCode: stepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [ + runCreatedEvent(run), + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'step_created', + correlationId: stepCid, + eventData: { stepName: 'step//test//add' }, + createdAt: new Date(), + }, + ], + }); + + expect(r2.suspended).toBeDefined(); + const op = r2.suspended!.pendingOperations[0]; + expect(op.correlationId).toBe(stepCid); + expect(op.hasCreatedEvent).toBe(true); + }); + + it('produces different correlationIds for different runs', async () => { + const r1 = await runQuickJSWorkflow({ + workflowCode: stepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: makeRun({ runId: 'wrun_aaa' }), + events: [], + }); + const r2 = await runQuickJSWorkflow({ + workflowCode: stepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: makeRun({ runId: 'wrun_bbb' }), + events: [], + }); + + expect(r1.suspended!.pendingOperations[0].correlationId).not.toBe( + r2.suspended!.pendingOperations[0].correlationId + ); + }); +}); + +describe('deterministic replay clock', () => { + // Date.now() inside the VM is a host-controlled clock that starts at + // the run's creation time and advances to each processed event's + // createdAt — mirroring the node:vm engine. Replay re-executes the + // workflow from the top, so real wall time would make time appear + // frozen across sleeps (start == end) and diverge between invocations. + + const sleepTimingWorkflow = ` + async function workflow() { + var startTime = Date.now(); + await globalThis[Symbol.for("WORKFLOW_SLEEP")]("10s"); + var endTime = Date.now(); + return { startTime: startTime, endTime: endTime }; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + + it('advances Date.now() across a sleep according to event timestamps', async () => { + const run = makeRun(); + + const r1 = await runQuickJSWorkflow({ + workflowCode: sleepTimingWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + const waitCid = r1.suspended!.pendingOperations[0].correlationId; + + const waitCreatedAt = new Date('2025-01-01T00:00:01Z'); + const waitCompletedAt = new Date('2025-01-01T00:00:11Z'); + const events = [ + runCreatedEvent(run), + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'wait_created' as const, + correlationId: waitCid, + eventData: { resumeAt: waitCompletedAt }, + createdAt: waitCreatedAt, + }, + { + eventId: 'evnt_002', + runId: run.runId, + eventType: 'wait_completed' as const, + correlationId: waitCid, + createdAt: waitCompletedAt, + }, + ]; + + const r2 = await runQuickJSWorkflow({ + workflowCode: sleepTimingWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events, + }); + const result = unwrapResult(r2.completed!.result) as { + startTime: number; + endTime: number; + }; + + // startTime is observed before any wait events are processed; endTime + // after wait_completed. The 10s sleep must be visible in the VM clock. + expect(result.endTime - result.startTime).toBeGreaterThanOrEqual(10_000); + expect(result.endTime).toBe(+waitCompletedAt); + + // Replaying the identical log again yields identical timestamps. + const r3 = await runQuickJSWorkflow({ + workflowCode: sleepTimingWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events, + }); + expect(unwrapResult(r3.completed!.result)).toEqual(result); + }); +}); diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts new file mode 100644 index 0000000000..71daee7bb9 --- /dev/null +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -0,0 +1,1255 @@ +/** + * QuickJS WASM workflow VM. + * + * An alternative engine for the event-replay execution model: the workflow + * code runs inside a QuickJS WASM VM (via quickjs-wasi) instead of a + * `node:vm` context. Every invocation creates a fresh VM, re-executes the + * workflow function from the top, and replays the recorded event log to + * resolve awaited primitives — the same replay semantics as the `node:vm` + * engine. + * + * The workflow primitives (useStep, sleep, createHook) are implemented as + * JavaScript code running inside the QuickJS VM. The host communicates with + * the VM by evaluating small JS snippets to read pending operations and + * resolve/reject promises. + * + * The VM bootstrap is deliberately split into two phases: + * 1. Static initialization (`initWorkflowVM`) — run-independent setup: + * VM creation, the serde bundle, and the workflow primitives. + * 2. Per-run initialization (inline in `runQuickJSWorkflow`) — seeded + * PRNG/ULID host functions, workflow bundle evaluation, run metadata, + * workflow input, and start. + * Keeping the phases separate is groundwork for VM-memory snapshotting: + * a follow-up can persist/restore the VM at the phase boundary (e.g. a + * build-time initial snapshot) without restructuring this module. Note + * that bundle evaluation currently sits in the per-run phase so that + * module-scope user code observes the seeded `Math.random`, matching the + * `node:vm` engine's replay determinism. + */ + +import type { Event, RunInput, WorkflowRun } from '@workflow/world'; +import * as nanoid from 'nanoid'; +import { JSException, QuickJS, type WasiOptions } from 'quickjs-wasi'; +import seedrandom from 'seedrandom'; +import type { CryptoKey } from '../encryption.js'; +import { runtimeLogger } from '../logger.js'; +import { decompress } from '../serialization/compression.js'; +import { decrypt } from '../serialization/encryption.js'; +import { quickjsExtensions, quickjsWasm } from './quickjs-assets.generated.js'; +import { runIdCreatedAt } from './run-id-time.js'; +import { VM_SERDE_BUNDLE } from './vm-serde-bundle.generated.js'; + +// ---- Host -> VM payload preparation ---- + +/** + * Prepare persisted payload bytes for consumption inside the VM: decrypt + * (when an encryption key is configured) and decompress (specVersion >= 5 + * payloads may be gzip/zstd-compressed). The VM only understands plain + * format-prefixed 'devl' bytes — it has neither the CryptoKey nor zlib. + * Both stages are format-prefix dispatched, so plaintext/uncompressed + * data passes through unchanged. Mirrors `prepareReplayPayload` in + * serialization.ts (the node:vm engine's equivalent host-side stage). + */ +async function prepareBytesForVM( + data: Uint8Array, + key?: CryptoKey +): Promise { + return (await decompress(await decrypt(data, key))) as Uint8Array; +} + +// ---- Types ---- + +export interface PendingStep { + type: 'step'; + correlationId: string; + stepId: string; + /** Format-prefixed devalue-serialized step input (args + closureVars) */ + input: Uint8Array; + /** Whether a step_created event already exists for this step */ + hasCreatedEvent: boolean; +} + +export interface PendingWait { + type: 'wait'; + correlationId: string; + /** ISO string of when to resume */ + resumeAt: string; + /** Whether a wait_created event already exists for this wait */ + hasCreatedEvent: boolean; +} + +export interface PendingHook { + type: 'hook'; + correlationId: string; + token: string; + isWebhook: boolean; + metadata?: unknown; + hasCreatedEvent: boolean; +} + +export interface PendingAttribute { + type: 'attribute'; + correlationId: string; + /** Normalized attribute changes (plain JSON-able objects) */ + changes: unknown[]; + allowReservedAttributes?: boolean; + /** Whether an attr_set event already exists for this write */ + hasCreatedEvent: boolean; +} + +export interface PendingHookDispose { + type: 'hook_dispose'; + correlationId: string; + hasCreatedEvent: boolean; +} + +export type PendingOperation = + | PendingStep + | PendingWait + | PendingHook + | PendingAttribute + | PendingHookDispose; + +export interface QuickJSRuntimeResult { + /** The workflow completed — result is format-prefixed devalue bytes */ + completed?: { result: Uint8Array }; + /** The workflow suspended with pending operations */ + suspended?: { + pendingOperations: PendingOperation[]; + }; + /** The workflow failed */ + failed?: { + message: string; + stack?: string; + name?: string; + /** + * Format-prefixed devalue bytes of the original thrown value + * (Error subclass with cause chain, plain object, primitive, etc.). + * Set when the VM-side rejection handler successfully serializes + * the thrown value. The host uses these bytes to reconstruct the + * original value through the standard error hydration pipeline, + * preserving type identity (TypeError, FatalError) and non-Error + * throws verbatim. Falls back to the message/stack/name fields + * when this is undefined (e.g. extractError pseudo-failures). + */ + valueBytes?: Uint8Array; + }; +} + +export interface QuickJSRuntimeOptions { + /** The compiled workflow bundle code (workflow mode output from SWC) */ + workflowCode: string; + /** The workflow ID (e.g. "workflow//./workflows/1_simple//simple") */ + workflowId: string; + /** The workflow run entity */ + workflowRun: WorkflowRun; + /** + * The full event log for the run. Every invocation replays the complete + * log from the start (same replay semantics as the `node:vm` engine). + */ + events: Event[]; + /** Encryption key for decrypting event payloads (undefined if unencrypted) */ + encryptionKey?: CryptoKey; + /** + * The local port the workflow server is listening on, used to populate + * `workflowMetadata.url`. Resolved at call time on the host side so the + * VM doesn't have to probe the filesystem. Ignored on Vercel — VERCEL_URL + * takes precedence there. + */ + port?: number; + /** + * Fallback workflow input from the queue message's resilient-start + * payload. Used when the fetched event log lacks a `run_created` event + * (eventually-consistent read after the parent's start() wrote it). + */ + runInput?: RunInput; +} + +// ---- VM Bootstrap Code ---- + +/** + * JavaScript code that runs inside the QuickJS VM to set up the workflow + * primitives. This sets up: + * - globalThis.__private_workflows (Map) - workflow registry + * - globalThis.__resolvers (Object) - pending promise resolve/reject functions + * - globalThis.__pending (Array) - metadata about pending operations + * - globalThis[Symbol.for("WORKFLOW_USE_STEP")] - step proxy factory + * - globalThis[Symbol.for("WORKFLOW_SLEEP")] - sleep function + */ +const VM_BOOTSTRAP = ` +// Symbol.dispose / Symbol.asyncDispose polyfills for QuickJS +if (typeof Symbol.dispose === "undefined") { + Symbol.dispose = Symbol.for("Symbol.dispose"); +} +if (typeof Symbol.asyncDispose === "undefined") { + Symbol.asyncDispose = Symbol.for("Symbol.asyncDispose"); +} + +globalThis.__private_workflows = new Map(); +globalThis.__resolvers = {}; +globalThis.__pending = []; +globalThis.__workflowResult = undefined; +globalThis.__workflowError = undefined; +// Buffer for hook_received payloads that arrive before the hook is awaited. +// Keyed by correlationId → array of payloads (preserves delivery order). +// This mirrors the event-replay runtime's payloadsQueue in hook.ts. +globalThis.__hookPayloadBuffer = {}; + +// Stubs for Web APIs that the workflow bundle may reference but are not +// available in QuickJS. Native C extensions (encoding, headers, url, +// structured-clone) provide the real implementations; these are minimal +// stubs for APIs that don't have native extensions yet. (btoa/atob and +// the Uint8Array base64/hex methods are built into quickjs-wasi >= 3.) + +if (typeof ReadableStream === "undefined") { + // Minimal ReadableStream that stores body data for Response.json()/text() + globalThis.ReadableStream = function() {}; + globalThis.ReadableStream.prototype.__bodyData = null; +} + +if (typeof WritableStream === "undefined") { + globalThis.WritableStream = function() {}; +} + +if (typeof TransformStream === "undefined") { + globalThis.TransformStream = function() {}; +} + +if (typeof console === "undefined") { + globalThis.console = { log: function(){}, error: function(){}, warn: function(){}, info: function(){} }; +} +// Stub exports/module for CJS bundle format +globalThis.exports = {}; +globalThis.module = { exports: globalThis.exports }; +// NOTE: TextEncoder/TextDecoder are provided by the native encoding extension. + +globalThis[Symbol.for("WORKFLOW_USE_STEP")] = function(stepId, closureVarsFn) { + var fn = function() { + var args = Array.prototype.slice.call(arguments); + var correlationId = "step_" + globalThis.__generateUlid(); + // Capture 'this' for method invocations (e.g., MyClass.method()) + var thisVal = (this !== undefined && this !== null && this !== globalThis) ? this : undefined; + // Serialize step input using the host-provided devalue serializer. + // This produces a format-prefixed Uint8Array ("devl" + devalue.stringify). + var input = globalThis[Symbol.for("workflow-serialize")]({ + args: args, + closureVars: closureVarsFn ? closureVarsFn() : undefined, + thisVal: thisVal, + }); + globalThis.__pending.push({ + type: "step", + correlationId: correlationId, + stepId: stepId, + input: input, + hasCreatedEvent: false, + }); + return new Promise(function(resolve, reject) { + globalThis.__resolvers[correlationId] = { resolve: resolve, reject: reject }; + }); + }; + // Set stepId on the proxy so the StepFunction reducer can detect and + // serialize step function references (e.g. when passed as arguments). + fn.stepId = stepId; + if (closureVarsFn) fn.__closureVarsFn = closureVarsFn; + return fn; +}; + +// Parses an "ms" library style duration string into milliseconds. +// Supports the same units as the replay runtime (which uses the "ms" +// package): ms / s / m / h / d / w / y, with verbose aliases +// (seconds, minutes, ...). +globalThis.__parseDurationMs = function(str) { + str = String(str); + if (str.length > 100) return undefined; + var match = str.match( + /^(-?(?:\\d+)?\\.?\\d+) *(milliseconds?|msecs?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d|weeks?|w|years?|yrs?|y)?$/i + ); + if (!match) return undefined; + var n = parseFloat(match[1]); + var type = (match[2] || "ms").toLowerCase(); + var s = 1000, m = 60 * s, h = 60 * m, d = 24 * h, w = 7 * d, y = 365.25 * d; + switch (type) { + case "years": case "year": case "yrs": case "yr": case "y": return n * y; + case "weeks": case "week": case "w": return n * w; + case "days": case "day": case "d": return n * d; + case "hours": case "hour": case "hrs": case "hr": case "h": return n * h; + case "minutes": case "minute": case "mins": case "min": case "m": return n * m; + case "seconds": case "second": case "secs": case "sec": case "s": return n * s; + case "milliseconds": case "millisecond": case "msecs": case "msec": case "ms": return n; + default: return undefined; + } +}; + +globalThis[Symbol.for("WORKFLOW_SLEEP")] = function(param) { + var correlationId = "wait_" + globalThis.__generateUlid(); + var resumeAt; + if (typeof param === "number") { + resumeAt = new Date(Date.now() + param).toISOString(); + } else if (typeof param === "string") { + var ms = globalThis.__parseDurationMs(param); + if (typeof ms === "number" && isFinite(ms)) { + resumeAt = new Date(Date.now() + ms).toISOString(); + } else { + // Not a duration string — try as an absolute date string. + var date = new Date(param); + if (isNaN(date.getTime())) { + throw new Error("Invalid sleep parameter: " + param); + } + resumeAt = date.toISOString(); + } + } else if (param instanceof Date) { + if (isNaN(param.getTime())) { + throw new Error("Invalid sleep parameter: " + param); + } + resumeAt = param.toISOString(); + } else { + throw new Error("Invalid sleep parameter: " + param); + } + globalThis.__pending.push({ + type: "wait", + correlationId: correlationId, + resumeAt: resumeAt, + hasCreatedEvent: false, + }); + return new Promise(function(resolve, reject) { + globalThis.__resolvers[correlationId] = { resolve: resolve, reject: reject }; + }); +}; + +// Response/Request polyfills — .json()/.text()/.arrayBuffer() are useStep +// proxies that execute on the host side. The proxies are assigned directly +// to the prototypes so that 'this' (the Response/Request instance) is +// serialized as thisVal by WORKFLOW_USE_STEP, matching the event-replay +// runtime's approach (commit dcb0761). +if (typeof Response === "undefined") { + var __BODY_INIT = Symbol.for("BODY_INIT"); + + globalThis.Response = function(body, init) { + init = init || {}; + this.status = init.status || 200; + this.statusText = init.statusText || ""; + this.headers = new globalThis.Headers(init.headers || []); + this.type = "default"; + this.url = ""; + this.redirected = false; + if (body !== null && body !== undefined) { + this.body = Object.create(globalThis.ReadableStream.prototype); + this.body[__BODY_INIT] = body; + } else { + this.body = null; + } + }; + Object.defineProperty(globalThis.Response.prototype, "ok", { + get: function() { return this.status >= 200 && this.status < 300; } + }); + Object.defineProperty(globalThis.Response.prototype, "bodyUsed", { + get: function() { return false; } + }); + // Assign useStep proxies directly — 'this' binding provides the + // Response instance, which gets serialized as thisVal by the proxy. + Object.defineProperties(globalThis.Response.prototype, { + arrayBuffer: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_array_buffer"), writable: true, configurable: true }, + json: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_json"), writable: true, configurable: true }, + text: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_text"), writable: true, configurable: true }, + }); + globalThis.Response.prototype.bytes = function() { + return this.arrayBuffer().then(function(buf) { return new Uint8Array(buf); }); + }; + globalThis.Response.prototype.clone = function() { + var r = Object.create(globalThis.Response.prototype); + r.status = this.status; r.statusText = this.statusText; + r.headers = this.headers; r.type = this.type; + r.url = this.url; r.redirected = this.redirected; r.body = this.body; + return r; + }; + globalThis.Response.json = function(data, init) { + var body = JSON.stringify(data); + var headers = new globalThis.Headers(init ? init.headers : []); + if (!headers.has("content-type")) { headers.set("content-type", "application/json"); } + return new globalThis.Response(body, { status: (init && init.status) || 200, statusText: (init && init.statusText) || "", headers: headers }); + }; +} +if (typeof Request === "undefined") { + globalThis.Request = function(input, init) { + init = init || {}; + if (typeof input === "string") { this.url = input; } + else if (input && typeof input === "object") { + this.url = input.url || ""; this.method = input.method; + this.headers = input.headers; this.body = input.body; + } + if (init.method) this.method = init.method.toUpperCase(); + if (!this.method) this.method = "GET"; + if (init.headers) this.headers = new globalThis.Headers(init.headers); + if (!this.headers) this.headers = new globalThis.Headers(); + if (init.body !== undefined) this.body = init.body; + if (!this.body) this.body = null; + this.duplex = init.duplex || "half"; + }; + Object.defineProperty(globalThis.Request.prototype, "bodyUsed", { + get: function() { return false; } + }); + Object.defineProperties(globalThis.Request.prototype, { + arrayBuffer: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_array_buffer"), writable: true, configurable: true }, + json: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_json"), writable: true, configurable: true }, + text: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_text"), writable: true, configurable: true }, + }); +} + +// createHook — returns a Hook object that is both a Thenable and AsyncIterable. +// Each await/yield creates a new promise keyed by the same correlationId. +// The promise is resolved when a hook_received event arrives. +globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")] = function(options) { + options = options || {}; + var token = options.token || globalThis.__generateNanoid(); + var correlationId = "hook_" + globalThis.__generateUlid(); + var isDisposed = false; + var hasCreatedEvent = false; + + // Register in pending operations. + // Serialize metadata inside the VM so Response/Request objects are + // properly handled by the devalue reducers before crossing the boundary. + globalThis.__pending.push({ + type: "hook", + correlationId: correlationId, + token: token, + isWebhook: !!options.isWebhook, + metadata: options.metadata ? globalThis[Symbol.for("workflow-serialize")](options.metadata) : undefined, + hasCreatedEvent: false, + }); + + // Each await creates a new promise for the next payload. + // The correlationId stays the same — the resolver is replaced each time. + function createHookPromise() { + // Check the payload buffer first — if a hook_received event arrived + // before this hook was awaited, the payload was buffered in the VM + // heap. Drain it immediately (matching event-replay payloadsQueue). + var buf = globalThis.__hookPayloadBuffer[correlationId]; + if (buf && buf.length > 0) { + return Promise.resolve(buf.shift()); + } + return new Promise(function(resolve, reject) { + globalThis.__resolvers[correlationId] = { resolve: resolve, reject: reject }; + }); + } + + function disposeHook() { + if (isDisposed) return; + isDisposed = true; + // Signal to the entrypoint to create a hook_disposed event + globalThis.__pending.push({ + type: "hook_dispose", + correlationId: correlationId, + hasCreatedEvent: false, + }); + // If there's a pending resolver, resolve it with undefined to break the iterator + if (globalThis.__resolvers[correlationId]) { + globalThis.__resolvers[correlationId].resolve(undefined); + delete globalThis.__resolvers[correlationId]; + } + } + + var hook = { + token: token, + then: function(onFulfilled, onRejected) { + return createHookPromise().then(onFulfilled, onRejected); + }, + dispose: disposeHook, + }; + + // Symbol.dispose for explicit resource management + hook[Symbol.dispose] = disposeHook; + + // AsyncIterable — yields payloads until disposed + hook[Symbol.asyncIterator] = function() { + return { + next: function() { + if (isDisposed) { + return Promise.resolve({ done: true, value: undefined }); + } + return createHookPromise().then(function(value) { + // If disposed while waiting, signal done + if (isDisposed) return { done: true, value: undefined }; + return { done: false, value: value }; + }); + }, + return: function() { + disposeHook(); + return Promise.resolve({ done: true, value: undefined }); + }, + }; + }; + + return hook; +}; + +// setAttributes — attaches plaintext metadata to the current run. +// Validation happens in library code (normalizeAttributeChanges) before +// this dispatcher is invoked, so "changes" is already normalized. The +// returned promise resolves when the matching attr_set event is +// observed during event processing — mirroring the node:vm engine's +// createSetAttributes (attribute-dispatcher.ts). +globalThis[Symbol.for("WORKFLOW_SET_ATTRIBUTES")] = function(changes, options) { + var correlationId = "attr_" + globalThis.__generateUlid(); + globalThis.__pending.push({ + type: "attribute", + correlationId: correlationId, + changes: changes, + allowReservedAttributes: !!(options && options.allowReservedAttributes), + hasCreatedEvent: false, + }); + return new Promise(function(resolve, reject) { + globalThis.__resolvers[correlationId] = { resolve: resolve, reject: reject }; + }); +}; + +// WORKFLOW_GET_STREAM_ID — generates a stream ID for a workflow run. +// Replicates getWorkflowRunStreamId() from util.ts inside the QuickJS VM. +// Uses the built-in btoa() for base64url encoding. +globalThis[Symbol.for("WORKFLOW_GET_STREAM_ID")] = function(namespace) { + var runId = globalThis[Symbol.for("WORKFLOW_CONTEXT")] + ? globalThis[Symbol.for("WORKFLOW_CONTEXT")].workflowRunId + : ""; + var streamId = runId.replace("wrun_", "strm_") + "_user"; + if (!namespace) return streamId; + // base64url: btoa then replace + with -, / with _, strip = + var b64 = btoa(namespace).replace(/\\+/g, "-").replace(/\\//g, "_").replace(/=+$/, ""); + return streamId + "_" + b64; +}; +`; + +// ---- Runtime ---- + +/** + * Phase 1 — static (run-independent) VM initialization. + * + * Creates a QuickJS VM and loads everything that does not depend on a + * specific workflow run: the serde bundle (devalue-based serialization + * used at the host/VM boundary) and the workflow-primitive bootstrap + * (useStep / sleep / createHook / Response-Request polyfills). + * + * `getNowMs` backs the VM's WASI clock (`Date.now()` / `new Date()` + * inside the VM). The callback itself is static — the per-run state it + * reads lives on the host and is advanced as events are consumed, + * matching the node:vm engine's deterministic replay clock. + * + * This phase is the future boundary for VM-memory snapshotting: a + * build-time snapshot can capture the VM right after this function and + * new runs can restore from it instead of paying VM creation + eval cost + * (`QuickJS.restore` accepts the same wasi override). + */ +async function initWorkflowVM(getNowMs: () => number): Promise { + // Deterministic replay clock: Date.now() / new Date() inside the VM + // read the host-controlled clock instead of wall time. Replay + // re-executes the workflow from the top on every invocation, so the + // clock must be derived from the event log (not real time) for the + // workflow to observe stable timestamps across invocations. + const wasi: WasiOptions = (memory) => ({ + clock_time_get(_clockId: number, _precision: bigint, resultPtr: number) { + const timeNs = BigInt(Math.round(getNowMs())) * 1_000_000n; + new DataView(memory.buffer).setBigUint64(resultPtr, timeNs, true); + return 0; + }, + }); + + const vm = await QuickJS.create({ + wasm: quickjsWasm, + memoryLimit: 256 * 1024 * 1024, + interruptHandler: createInterruptHandler(), + extensions: quickjsExtensions, + wasi, + }); + + // Evaluate the VM serde bundle + vm.evalCode(VM_SERDE_BUNDLE, 'vm-serde.js').dispose(); + + // Bootstrap workflow primitives + vm.evalCode(VM_BOOTSTRAP, 'bootstrap.js').dispose(); + + return vm; +} + +export async function runQuickJSWorkflow( + options: QuickJSRuntimeOptions +): Promise { + const { workflowCode, workflowId, workflowRun, events } = options; + + const startedAt = workflowRun.startedAt ? +workflowRun.startedAt : Date.now(); + + // Deterministic PRNG seed — identical for EVERY invocation of the same + // run. Full event replay requires this: each invocation re-executes the + // workflow from the top and must regenerate the exact same correlationId + // sequence so that pending operations re-created by replay match the + // events recorded by earlier invocations. Sequential operations within + // one execution still get distinct ids because the PRNG advances as the + // workflow draws from it. Identical seeding across CONCURRENT invocations + // of the same run is also load-bearing: both produce the same ids, and + // the world's per-(runId, correlationId) uniqueness turns the duplicate + // `events.create` into an EntityConflictError that the entrypoint + // swallows. + const seed = [ + workflowRun.runId, + workflowRun.workflowName, + String(startedAt), + ].join(':'); + const rng = seedrandom(seed); + + // Seeded nanoid generator — uses the same nanoid package and seeded PRNG + // as the node:vm engine for consistent token generation. + const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => + new Uint8Array(size).map(() => 256 * rng()) + ); + + // Deterministic replay clock, mirroring the node:vm engine (see + // workflow.ts): the initial value is the run's creation time recovered + // from the ULID embedded in `runId` (falling back to `createdAt`), and + // it advances to each processed event's `createdAt` as the event log is + // replayed. Monotonic (Math.max) so the outer processEvents re-scan + // loop can't move the clock backwards mid-execution. + let vmNowMs = + runIdCreatedAt(workflowRun.runId) ?? (+workflowRun.createdAt || startedAt); + const advanceClock = (ms: number) => { + if (Number.isFinite(ms)) vmNowMs = Math.max(vmNowMs, ms); + }; + + // ---- Phase 1: static initialization ---- + const vm = await initWorkflowVM(() => vmNowMs); + + // ---- Phase 2: per-run initialization ---- + + // Seeded Math.random + { + using randomFn = vm.newFunction('random', () => vm.newNumber(rng())); + using math = vm.global.getProp('Math'); + math.setProp('random', randomFn); + } + + // Seeded nanoid generator + { + using nanoidFn = vm.newFunction('__generateNanoid', () => + vm.newString(generateNanoid()) + ); + vm.setProp(vm.global, '__generateNanoid', nanoidFn); + } + + // Inject a deterministic timestamp for the VM's ULID factory. ULIDs + // produced inside the VM use this as their time prefix instead of + // Date.now(), so two concurrent workflow invocations of the same run + // produce IDENTICAL correlationIds (the random portion also matches + // because the PRNG is seeded the same way) and the world's + // EntityConflictError on `events.create` dedups one of each pair. + // Use `startedAt` (constant per-run) so the prefix is stable across + // replay invocations too. + vm.evalCode(`globalThis.__ulidTimestamp = ${startedAt};`).dispose(); + + // Execute the workflow bundle — use the workflowId as the eval filename + // so QuickJS stack traces reference the workflow name, enabling source map + // remapping by remapErrorStack (which matches frames by filename). + // Evaluated in the per-run phase (after Math.random seeding) so that + // module-scope user code draws from the seeded PRNG, matching the + // node:vm engine's replay determinism. + try { + vm.evalCode(workflowCode, workflowId || 'workflow.js').dispose(); + } catch (err) { + return extractError(vm, err, 'Workflow evaluation failed'); + } + + // Extract workflow arguments. Prefer the run_created event; fall back + // to the queue message's runInput if the event log is incomplete + // (eventually-consistent read after start()). Failing to find input + // for a first invocation is fatal — running the workflow function + // with no args would silently turn typed arguments into `undefined` + // and, for recursive workflows, produce exponential fan-out. + const runCreatedEvent = events.find((e) => e.eventType === 'run_created'); + const runCreatedInput = + runCreatedEvent && 'eventData' in runCreatedEvent + ? (runCreatedEvent.eventData as Record)?.input + : undefined; + const runInput: unknown = + runCreatedInput ?? (options.runInput?.input as unknown); + + if (runInput instanceof Uint8Array) { + const decryptedInput = await prepareBytesForVM( + runInput, + options.encryptionKey + ); + runtimeLogger.debug('QuickJS runtime: run input format', { + prefix: new TextDecoder().decode(decryptedInput.subarray(0, 4)), + byteLength: decryptedInput.byteLength, + source: runCreatedInput ? 'run_created' : 'queueMessage.runInput', + }); + const inputHandle = vm.newUint8Array(decryptedInput); + vm.setProp(vm.global, '__wdk_input', inputHandle); + inputHandle.dispose(); + } else if (runInput === undefined && events.length > 0) { + // The event log is non-empty (we got run_started or similar) but + // no run_created event was found and no queue-provided runInput is + // available. This is the race condition observed during the fib + // incident — silently dropping arguments would turn `n` into + // `undefined` and, for recursive workflows, cause exponential + // fan-out. Fail loud so the run goes to `run_failed` and the queue + // can retry. Empty `events` is allowed because tests that bootstrap + // a workflow with no arguments rely on the old permissive behavior. + throw new Error( + `Cannot start workflow run "${workflowRun.runId}": no run_created event found and no runInput in the queue payload, but other events are present (likely a read-after-write race during start()).` + ); + } + + // Set workflow context metadata (for getWorkflowMetadata()). + // Must match the shape that the node:vm engine produces (see + // packages/core/src/workflow.ts: runWorkflow → ctx) so user code + // that compares `getWorkflowMetadata()` values between a step + // (server-side) and the workflow (VM-side) sees identical objects. + { + const metadata = { + workflowName: workflowRun.workflowName, + workflowRunId: workflowRun.runId, + workflowStartedAt: workflowRun.startedAt + ? new Date(+workflowRun.startedAt) + : new Date(), + url: process.env.VERCEL_URL + ? `https://${process.env.VERCEL_URL}` + : `http://localhost:${options.port ?? 3000}`, + features: { encryption: !!options.encryptionKey }, + }; + vm.evalCode( + `globalThis[Symbol.for("WORKFLOW_CONTEXT")] = ${JSON.stringify(metadata)};` + + `globalThis[Symbol.for("WORKFLOW_CONTEXT")].workflowStartedAt = new Date(${JSON.stringify(metadata.workflowStartedAt.toISOString())});` + ).dispose(); + } + + // Start the workflow function. If the workflow isn't registered, + // throw an error tagged with `name = "WorkflowNotRegisteredError"` + // so the host-side entrypoint can reconstruct a real + // WorkflowNotRegisteredError (a WorkflowRuntimeError subclass that + // classifies as RUNTIME_ERROR) rather than a generic user error. + // See quickjs-entrypoint.ts's run_failed branch. + try { + vm.evalCode(` + var __wfn = globalThis.__private_workflows.get(${JSON.stringify(workflowId)}); + if (!__wfn) { + var __wfnErr = new Error("Workflow \\"" + ${JSON.stringify(workflowId)} + "\\" is not registered in the current deployment."); + __wfnErr.name = "WorkflowNotRegisteredError"; + throw __wfnErr; + } + var __args = globalThis.__wdk_input + ? globalThis[Symbol.for("workflow-deserialize")](globalThis.__wdk_input) + : []; + delete globalThis.__wdk_input; + if (!Array.isArray(__args)) __args = [__args]; + __wfn.apply(null, __args).then( + function(result) { globalThis.__workflowResult = globalThis[Symbol.for("workflow-serialize")](result); }, + function(error) { + // Preserve display info on the host-side failed object + // (matches the legacy host-visible shape) AND serialize the + // entire thrown value so the host can dehydrate the original + // type-identity, cause chain, or non-Error throws verbatim + // through the standard error pipeline. + globalThis.__workflowError = { + message: error && error.message != null ? String(error.message) : String(error), + stack: error && error.stack ? error.stack : "", + name: error && error.name ? error.name : (error instanceof Error ? "Error" : typeof error), + valueBytes: globalThis[Symbol.for("workflow-serialize")](error), + }; + } + ); + `).dispose(); + } catch (err) { + return extractError(vm, err, 'Failed to start workflow'); + } + + // Process events and drain jobs in a loop. Events may resolve promises + // that unblock workflow code, which then creates NEW resolvers for + // subsequent events. Re-processing events matches these new resolvers + // against events that were already delivered. + { + let maxIterations = 100; + let madeProgress: boolean; + do { + madeProgress = await processEvents( + vm, + events, + advanceClock, + options.encryptionKey + ); + let batch: number; + do { + batch = vm.executePendingJobs(); + if (batch > 0) madeProgress = true; + } while (batch > 0); + } while (madeProgress && --maxIterations > 0); + } + + // ---- Check result ---- + return checkWorkflowState(vm); +} + +// ---- Event Processing ---- + +async function processEvents( + vm: QuickJS, + events: Event[], + advanceClock: (ms: number) => void, + encryptionKey?: CryptoKey +): Promise { + let resolved = false; + for (const event of events) { + // Advance the VM's deterministic clock to this event's creation time + // BEFORE resolving anything, so workflow code unblocked by this event + // observes Date.now() at (or after — the clock is monotonic) the time + // the event was recorded. Mirrors the node:vm engine's + // `onConsumedEvent → updateTimestamp(+event.createdAt)`. + advanceClock(+event.createdAt); + + const cid = event.correlationId; + if (!cid) continue; + + const escapedCid = cid.replace(/"/g, '\\"'); + const eventData = + 'eventData' in event + ? (event.eventData as Record) + : undefined; + + // Log the event and whether the resolver exists + switch (event.eventType) { + case 'step_completed': { + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + const rawOutput = eventData?.result ?? eventData?.output; + if (hasResolver) { + if (rawOutput instanceof Uint8Array) { + // Decrypt if encrypted — the VM only understands 'devl' format + runtimeLogger.debug('QuickJS runtime: step result raw', { + correlationId: escapedCid, + rawPrefix: new TextDecoder().decode(rawOutput.subarray(0, 4)), + rawByteLength: rawOutput.byteLength, + isBuffer: Buffer.isBuffer(rawOutput), + }); + const decryptedOutput = await prepareBytesForVM( + rawOutput, + encryptionKey + ); + runtimeLogger.debug('QuickJS runtime: step result decrypted', { + correlationId: escapedCid, + prefix: new TextDecoder().decode(decryptedOutput.subarray(0, 4)), + byteLength: decryptedOutput.byteLength, + }); + const bytesHandle = vm.newUint8Array(decryptedOutput); + vm.setProp(vm.global, '__tmp_result', bytesHandle); + bytesHandle.dispose(); + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve(globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result));` + + `delete globalThis.__resolvers["${escapedCid}"];` + + `delete globalThis.__tmp_result;` + ).dispose(); + } else { + runtimeLogger.debug('QuickJS runtime: step result non-binary', { + correlationId: escapedCid, + type: typeof rawOutput, + isNull: rawOutput === null, + isUndefined: rawOutput === undefined, + constructor: rawOutput?.constructor?.name, + }); + const serialized = + rawOutput !== undefined ? JSON.stringify(rawOutput) : 'undefined'; + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve(${serialized});` + + `delete globalThis.__resolvers["${escapedCid}"];` + ).dispose(); + } + // Drain ALL microtasks after resolve + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } + markCreated(vm, escapedCid); + break; + } + case 'step_failed': { + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + if (hasResolver) { + const errorData = eventData?.error; + if (errorData instanceof Uint8Array) { + // Modern path (post-#1851): the step handler dehydrated the + // thrown value through the first-class error pipeline. Decrypt + // (if encrypted) and pass the bytes to the VM-side deserializer + // so the workflow catch sees a properly typed Error subclass + // (TypeError, FatalError with original cause chain, etc.) with + // the original message and stack preserved. + const decrypted = await prepareBytesForVM(errorData, encryptionKey); + const bytesHandle = vm.newUint8Array(decrypted); + vm.setProp(vm.global, '__tmp_error', bytesHandle); + bytesHandle.dispose(); + vm.evalCode( + `(function(){` + + `var e=globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_error);` + + `globalThis.__resolvers["${escapedCid}"].reject(e);` + + `delete globalThis.__resolvers["${escapedCid}"];` + + `delete globalThis.__tmp_error;` + + `})()` + ).dispose(); + } else { + // Legacy path: pre-pipeline events stored error as + // `{ message, stack, code }`. Reconstruct a FatalError so + // workflow catch can detect it via FatalError.is(), matching + // the original V1 step handler behavior. + const isErrorObject = + typeof errorData === 'object' && errorData !== null; + const msg = isErrorObject + ? (((errorData as Record).message as string) ?? + 'Step failed') + : typeof errorData === 'string' + ? errorData + : 'Step failed'; + const errorStack = + (isErrorObject + ? (errorData as Record).stack + : undefined) ?? (eventData?.stack as string | undefined); + const stackAssignment = errorStack + ? `e.stack=${JSON.stringify(errorStack)};` + : ''; + vm.evalCode( + `(function(){var e=new Error(${JSON.stringify(msg)});e.name="FatalError";e.fatal=true;${stackAssignment}` + + `globalThis.__resolvers["${escapedCid}"].reject(e);` + + `delete globalThis.__resolvers["${escapedCid}"];})()` + ).dispose(); + } + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } + markCreated(vm, escapedCid); + break; + } + case 'wait_completed': { + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + if (hasResolver) { + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve();` + + `delete globalThis.__resolvers["${escapedCid}"];` + ).dispose(); + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } + markCreated(vm, escapedCid); + break; + } + case 'attr_set': { + // Only workflow-written attribute events resolve a pending + // setAttributes() promise; step/system writers share no + // correlationIds with VM resolvers, so the guard is defensive. + const writer = (eventData?.writer as { type?: string } | undefined) + ?.type; + if (writer !== 'workflow') break; + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + if (hasResolver) { + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve();` + + `delete globalThis.__resolvers["${escapedCid}"];` + ).dispose(); + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } + markCreated(vm, escapedCid); + break; + } + case 'hook_received': { + // Check if this event was already processed (delivered or + // buffered) within this invocation. Prevents double-delivery when + // the outer loop re-scans events. + const alreadyProcessed = event.eventId + ? vm.dump( + vm.evalCode( + `!!(globalThis.__hookPayloadBuffer.__processedEventIds && globalThis.__hookPayloadBuffer.__processedEventIds[${JSON.stringify(event.eventId)}])` + ) + ) + : false; + if (alreadyProcessed) { + runtimeLogger.debug( + 'QuickJS runtime: hook_received already processed', + { + correlationId: cid, + eventId: event.eventId, + } + ); + markCreated(vm, escapedCid); + break; + } + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + const rawPayload = eventData?.payload ?? eventData?.result; + runtimeLogger.debug('QuickJS runtime: processing hook_received', { + correlationId: cid, + eventId: event.eventId, + hasResolver, + payloadType: typeof rawPayload, + payloadIsUint8Array: rawPayload instanceof Uint8Array, + payloadKeys: + rawPayload && typeof rawPayload === 'object' + ? Object.keys(rawPayload) + : undefined, + }); + if (hasResolver) { + if (rawPayload instanceof Uint8Array) { + // Decrypt if encrypted — the VM only understands 'devl' format + const decryptedPayload = await prepareBytesForVM( + rawPayload, + encryptionKey + ); + const bytesHandle = vm.newUint8Array(decryptedPayload); + vm.setProp(vm.global, '__tmp_result', bytesHandle); + bytesHandle.dispose(); + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve(globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result));` + + `delete globalThis.__resolvers["${escapedCid}"];` + + `delete globalThis.__tmp_result;` + ).dispose(); + } else { + const serialized = + rawPayload !== undefined + ? JSON.stringify(rawPayload) + : 'undefined'; + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve(${serialized});` + + `delete globalThis.__resolvers["${escapedCid}"];` + ).dispose(); + } + // Mark this event as processed in the VM heap to prevent + // double-delivery when the outer loop re-scans events. + if (event.eventId) { + vm.evalCode( + `(globalThis.__hookPayloadBuffer.__processedEventIds = globalThis.__hookPayloadBuffer.__processedEventIds || {})[${JSON.stringify(event.eventId)}] = true;` + ).dispose(); + } + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } else { + // No resolver yet — buffer the payload in the VM heap. When + // createHookPromise() is called later, it will drain this buffer + // first (matching the node:vm engine's payloadsQueue behavior). + const eventIdJs = event.eventId + ? JSON.stringify(event.eventId) + : 'null'; + const bufferAndTrack = + `(globalThis.__hookPayloadBuffer["${escapedCid}"] = globalThis.__hookPayloadBuffer["${escapedCid}"] || [])` + + `.push(%PAYLOAD%);` + + (event.eventId + ? `(globalThis.__hookPayloadBuffer.__processedEventIds = globalThis.__hookPayloadBuffer.__processedEventIds || {})[${eventIdJs}] = true;` + : ''); + if (rawPayload instanceof Uint8Array) { + // Decrypt if encrypted — the VM only understands 'devl' format + const decryptedPayload = await prepareBytesForVM( + rawPayload, + encryptionKey + ); + const bytesHandle = vm.newUint8Array(decryptedPayload); + vm.setProp(vm.global, '__tmp_result', bytesHandle); + bytesHandle.dispose(); + vm.evalCode( + bufferAndTrack.replace( + '%PAYLOAD%', + 'globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result)' + ) + 'delete globalThis.__tmp_result;' + ).dispose(); + } else { + const serialized = + rawPayload !== undefined + ? JSON.stringify(rawPayload) + : 'undefined'; + vm.evalCode( + bufferAndTrack.replace('%PAYLOAD%', serialized) + ).dispose(); + } + } + markCreated(vm, escapedCid); + break; + } + case 'hook_conflict': { + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + if (hasResolver) { + const conflictToken = (eventData?.token as string) ?? 'unknown'; + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].reject(new Error(${JSON.stringify(`Hook token "${conflictToken}" is already in use by another workflow`)}));` + + `delete globalThis.__resolvers["${escapedCid}"];` + ).dispose(); + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } + markCreated(vm, escapedCid); + break; + } + case 'step_created': + case 'step_started': + case 'step_retrying': + case 'wait_created': + case 'hook_created': { + markCreated(vm, escapedCid); + break; + } + case 'hook_disposed': { + // Disambiguate from the `hook` pending op with the same + // correlationId — we want to mark the `hook_dispose` entry. + markCreated(vm, escapedCid, 'hook_dispose'); + break; + } + } + } + return resolved; +} + +function markCreated(vm: QuickJS, escapedCid: string, opType?: string): void { + // `hook` and `hook_dispose` pending ops share the same correlationId, + // so when processing `hook_disposed` events we must disambiguate by + // type — otherwise `.find()` returns the original `hook` op and the + // `hook_dispose` op is never marked, causing the entrypoint to keep + // retrying a hook_disposed for an already-deleted entity. + const predicate = opType + ? `function(p){return p.correlationId==="${escapedCid}"&&p.type==="${opType}";}` + : `function(p){return p.correlationId==="${escapedCid}";}`; + vm.evalCode( + `var __p=globalThis.__pending.find(${predicate});` + + `if(__p)__p.hasCreatedEvent=true;` + ).dispose(); +} + +// ---- State Checking ---- + +function checkWorkflowState(vm: QuickJS): QuickJSRuntimeResult { + // Check completed — __workflowResult is a format-prefixed Uint8Array + { + using h = vm.evalCode('globalThis.__workflowResult'); + if (!h.isUndefined) { + const resultBytes = h.toUint8Array(); + vm.dispose(); + return { completed: { result: resultBytes } }; + } + } + + // Check failed + { + using h = vm.evalCode('globalThis.__workflowError'); + if (!h.isUndefined) { + const errorObj = vm.dump(h) as + | { + message: string; + stack?: string; + name?: string; + valueBytes?: Uint8Array; + } + | string; + const failed = + typeof errorObj === 'string' + ? { message: errorObj } + : { + message: errorObj.message, + stack: errorObj.stack || undefined, + name: errorObj.name || undefined, + valueBytes: errorObj.valueBytes, + }; + runtimeLogger.error('QuickJS runtime: workflow failed in VM', { + errorMessage: failed.message, + errorName: failed.name, + errorStack: failed.stack, + }); + vm.dispose(); + return { failed }; + } + } + + // Check suspended — the workflow is suspended if there are active resolvers + // OR pending operations that haven't been created yet (e.g. hooks created + // upfront but not yet awaited) + { + using h = vm.evalCode( + 'Object.keys(globalThis.__resolvers).length > 0 || globalThis.__pending.some(function(p){return!p.hasCreatedEvent;})' + ); + if (vm.dump(h)) { + using pendingH = vm.evalCode( + `globalThis.__pending.filter(function(p){return!!globalThis.__resolvers[p.correlationId] || !p.hasCreatedEvent;})` + ); + const pendingOps = vm.dump(pendingH) as PendingOperation[]; + vm.dispose(); + + return { + suspended: { + pendingOperations: pendingOps, + }, + }; + } + } + + vm.dispose(); + return { failed: { message: 'Workflow ended in unknown state' } }; +} + +// ---- Helpers ---- + +function extractError( + vm: QuickJS, + err: unknown, + fallbackMessage: string +): QuickJSRuntimeResult { + let message = fallbackMessage; + let stack: string | undefined; + let name: string | undefined; + + if (err instanceof JSException) { + const error = vm.dump(err.handle) as Record | null; + err.handle.dispose(); + message = (error?.message as string) ?? err.message ?? fallbackMessage; + stack = (error?.stack as string) ?? err.stack; + name = (error?.name as string) ?? err.name; + } else if (err instanceof Error) { + message = err.message ?? fallbackMessage; + stack = err.stack; + name = err.name; + } + + vm.dispose(); + return { + failed: { message, stack, name }, + }; +} + +function createInterruptHandler(): () => boolean { + const start = Date.now(); + const timeout = 30_000; + return () => Date.now() - start > timeout; +} diff --git a/packages/core/src/runtime/start.ts b/packages/core/src/runtime/start.ts index 5cee64f49c..e5b43f35a2 100644 --- a/packages/core/src/runtime/start.ts +++ b/packages/core/src/runtime/start.ts @@ -35,6 +35,7 @@ import { version as workflowCoreVersion } from '../version.js'; import { getWorldLazy } from './get-world-lazy.js'; import { getWorkflowQueueName, healthCheck } from './helpers.js'; import { Run } from './run.js'; +import { getWorkflowVmFromEnv } from './vm-mode.js'; import { safeWaitUntil, waitedUntil } from './wait-until.js'; import { assertWorldSupportsRuntimeProtocol } from './world-compatibility.js'; @@ -525,10 +526,18 @@ export async function start( // is simply absent. const creatorEnvironment = world.getEnvironment?.(); + // If WORKFLOW_VM is set on the client starting the run, stamp the + // engine choice into the run's executionContext so the run keeps + // executing on the engine it started on (the same deployment can + // serve both VM engines). Unknown values throw — see + // getWorkflowVmFromEnv(). + const workflowVm = getWorkflowVmFromEnv(); + const executionContext = { traceCarrier, workflowCoreVersion, features: { encryption: !!encryptionKey }, + ...(workflowVm ? { workflowVm } : {}), ...(opts.replayedFromRunId ? { replayedFromRunId: opts.replayedFromRunId } : {}), diff --git a/packages/core/src/runtime/vm-mode.test.ts b/packages/core/src/runtime/vm-mode.test.ts new file mode 100644 index 0000000000..1f451e016b --- /dev/null +++ b/packages/core/src/runtime/vm-mode.test.ts @@ -0,0 +1,109 @@ +import { WorkflowRuntimeError } from '@workflow/errors'; +import type { WorkflowRun } from '@workflow/world'; +import { afterEach, describe, expect, it } from 'vitest'; +import { getWorkflowVmFromEnv, useQuickJSVm, WORKFLOW_VMS } from './vm-mode.js'; + +describe('getWorkflowVmFromEnv', () => { + it('returns undefined when WORKFLOW_VM is not set', () => { + expect(getWorkflowVmFromEnv({})).toBeUndefined(); + }); + + it('returns undefined when WORKFLOW_VM is empty', () => { + expect(getWorkflowVmFromEnv({ WORKFLOW_VM: '' })).toBeUndefined(); + }); + + it('returns "node" when WORKFLOW_VM=node', () => { + expect(getWorkflowVmFromEnv({ WORKFLOW_VM: 'node' })).toBe('node'); + }); + + it('returns "quickjs" when WORKFLOW_VM=quickjs', () => { + expect(getWorkflowVmFromEnv({ WORKFLOW_VM: 'quickjs' })).toBe('quickjs'); + }); + + it('throws WorkflowRuntimeError on unknown values', () => { + expect(() => getWorkflowVmFromEnv({ WORKFLOW_VM: 'bogus' })).toThrow( + WorkflowRuntimeError + ); + expect(() => getWorkflowVmFromEnv({ WORKFLOW_VM: 'bogus' })).toThrow( + /Invalid WORKFLOW_VM value: "bogus"/ + ); + }); + + it('is case-sensitive: uppercase values are rejected', () => { + expect(() => getWorkflowVmFromEnv({ WORKFLOW_VM: 'QUICKJS' })).toThrow( + WorkflowRuntimeError + ); + expect(() => getWorkflowVmFromEnv({ WORKFLOW_VM: 'Node' })).toThrow( + WorkflowRuntimeError + ); + }); + + it('rejects leading/trailing whitespace', () => { + expect(() => getWorkflowVmFromEnv({ WORKFLOW_VM: ' quickjs' })).toThrow( + WorkflowRuntimeError + ); + expect(() => getWorkflowVmFromEnv({ WORKFLOW_VM: 'node ' })).toThrow( + WorkflowRuntimeError + ); + }); + + it('error message lists valid options', () => { + try { + getWorkflowVmFromEnv({ WORKFLOW_VM: 'bogus' }); + expect.fail('expected to throw'); + } catch (err) { + expect(err).toBeInstanceOf(WorkflowRuntimeError); + for (const mode of WORKFLOW_VMS) { + expect((err as Error).message).toContain(mode); + } + } + }); +}); + +describe('useQuickJSVm', () => { + const makeRun = (executionContext?: Record) => + ({ + runId: 'wrun_test', + workflowName: 'test', + executionContext, + }) as unknown as WorkflowRun; + + afterEach(() => { + delete process.env.WORKFLOW_VM; + }); + + it('defaults to node:vm (false) when nothing is configured', () => { + expect(useQuickJSVm(makeRun())).toBe(false); + }); + + it('returns true when WORKFLOW_VM=quickjs is set in the environment', () => { + process.env.WORKFLOW_VM = 'quickjs'; + expect(useQuickJSVm(makeRun())).toBe(true); + }); + + it('returns false when WORKFLOW_VM=node is set in the environment', () => { + process.env.WORKFLOW_VM = 'node'; + expect(useQuickJSVm(makeRun())).toBe(false); + }); + + it('executionContext.workflowVm=quickjs wins over env node', () => { + process.env.WORKFLOW_VM = 'node'; + expect(useQuickJSVm(makeRun({ workflowVm: 'quickjs' }))).toBe(true); + }); + + it('executionContext.workflowVm=node wins over env quickjs (run affinity)', () => { + process.env.WORKFLOW_VM = 'quickjs'; + expect(useQuickJSVm(makeRun({ workflowVm: 'node' }))).toBe(false); + }); + + it('throws on unknown executionContext.workflowVm values', () => { + expect(() => useQuickJSVm(makeRun({ workflowVm: 'bogus' }))).toThrow( + WorkflowRuntimeError + ); + }); + + it('throws on unknown WORKFLOW_VM env values', () => { + process.env.WORKFLOW_VM = 'bogus'; + expect(() => useQuickJSVm(makeRun())).toThrow(WorkflowRuntimeError); + }); +}); diff --git a/packages/core/src/runtime/vm-mode.ts b/packages/core/src/runtime/vm-mode.ts new file mode 100644 index 0000000000..e8a7864003 --- /dev/null +++ b/packages/core/src/runtime/vm-mode.ts @@ -0,0 +1,75 @@ +/** + * VM engine selection for workflow execution. + * + * The Node.js `node:vm` engine is the default. The QuickJS WASM engine is + * opt-in via the `WORKFLOW_VM` env var or `executionContext.workflowVm`. + * + * Both engines implement the same event-replay execution model: on every + * workflow handler invocation the workflow function is re-executed from the + * top and the recorded event log resolves awaited primitives. The QuickJS + * engine runs the workflow code in a QuickJS WASM VM (via quickjs-wasi) + * instead of a `node:vm` context, which makes it usable on platforms that + * do not implement `node:vm` (e.g. Cloudflare Workers) and is the + * foundation for VM-memory snapshotting. + */ + +import { WorkflowRuntimeError } from '@workflow/errors'; +import type { WorkflowRun } from '@workflow/world'; + +/** + * Known workflow VM engines. Any other `WORKFLOW_VM` value is treated as + * a misconfiguration and rejected at startup. + */ +export const WORKFLOW_VMS = ['node', 'quickjs'] as const; + +export type WorkflowVmMode = (typeof WORKFLOW_VMS)[number]; + +/** + * Read and validate the `WORKFLOW_VM` env var. + * + * Returns the configured engine, or `undefined` if unset/empty. + * Throws {@link WorkflowRuntimeError} if the value is set but not one of + * the known engines — catching misconfiguration early is better than + * silently falling back to the default. + */ +export function getWorkflowVmFromEnv( + env: NodeJS.ProcessEnv = process.env +): WorkflowVmMode | undefined { + const raw = env.WORKFLOW_VM; + if (raw === undefined || raw === '') return undefined; + if ((WORKFLOW_VMS as readonly string[]).includes(raw)) { + return raw as WorkflowVmMode; + } + throw new WorkflowRuntimeError( + `Invalid WORKFLOW_VM value: "${raw}". ` + + `Expected one of: ${WORKFLOW_VMS.join(', ')}.` + ); +} + +/** + * Whether to use the QuickJS WASM VM for a given run. + * + * The run's `executionContext.workflowVm` (stamped by the SDK at `start()` + * when `WORKFLOW_VM` is set on the client) takes precedence so a run keeps + * executing on the engine it started on. When the run doesn't specify an + * engine, the `WORKFLOW_VM` env var on the workflow handler decides. + * The default is the `node:vm` engine. + * + * Throws if `WORKFLOW_VM` or `executionContext.workflowVm` is set to an + * unknown value. + */ +export function useQuickJSVm(workflowRun: WorkflowRun): boolean { + const vmFromRun = ( + workflowRun.executionContext as { workflowVm?: string } | undefined + )?.workflowVm; + if (vmFromRun !== undefined) { + if (!(WORKFLOW_VMS as readonly string[]).includes(vmFromRun)) { + throw new WorkflowRuntimeError( + `Invalid executionContext.workflowVm value: "${vmFromRun}". ` + + `Expected one of: ${WORKFLOW_VMS.join(', ')}.` + ); + } + return vmFromRun === 'quickjs'; + } + return getWorkflowVmFromEnv() === 'quickjs'; +} diff --git a/packages/core/src/serialization/codec-devalue-vm.ts b/packages/core/src/serialization/codec-devalue-vm.ts new file mode 100644 index 0000000000..3713132058 --- /dev/null +++ b/packages/core/src/serialization/codec-devalue-vm.ts @@ -0,0 +1,93 @@ +/** + * VM-compatible devalue codec. + * + * Same as codec-devalue.ts but uses VM-compatible reducers/revivers + * (no Node.js Buffer, no node:util). Safe to bundle into the QuickJS VM. + */ + +import { parse, stringify, unflatten } from 'devalue'; +import { SerializationFormat, type Reducers, type Revivers } from './types.js'; +import type { Codec, SerializationMode } from './codec.js'; +import { getClassReducers, getClassRevivers } from './reducers/class.js'; +import { getCommonReducers, getCommonRevivers } from './reducers/common-vm.js'; +import { + getStepFunctionReducer, + getStepFunctionReviver, +} from './reducers/step-function.js'; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +function getReducersForMode(mode: SerializationMode): Partial { + switch (mode) { + case 'workflow': + return { + ...getClassReducers(), + ...getStepFunctionReducer(), + ...getCommonReducers(), + }; + case 'step': + return { + ...getClassReducers(), + ...getCommonReducers(), + }; + case 'client': + return { + ...getClassReducers(), + ...getCommonReducers(), + }; + } +} + +function getReviversForMode(mode: SerializationMode): Partial { + switch (mode) { + case 'workflow': + return { + ...getClassRevivers(), + ...getStepFunctionReviver(), + ...getCommonRevivers(), + }; + case 'step': + return { + ...getClassRevivers(), + ...getCommonRevivers(), + }; + case 'client': + return { + ...getClassRevivers(), + ...getCommonRevivers(), + StepFunction: () => { + throw new Error( + 'Step functions cannot be deserialized in client context.' + ); + }, + }; + } +} + +export const devalueVmCodec: Codec = { + formatPrefix: SerializationFormat.DEVALUE_V1, + + serialize(value: unknown, mode: SerializationMode): Uint8Array { + const reducers = getReducersForMode(mode); + const str = stringify( + value, + reducers as Record any> + ); + return encoder.encode(str); + }, + + deserialize(data: Uint8Array, mode: SerializationMode): unknown { + const revivers = getReviversForMode(mode); + const str = decoder.decode(data); + return parse(str, revivers as Record any>); + }, + + deserializeLegacy(data: unknown, mode: SerializationMode): unknown { + const revivers = getReviversForMode(mode); + return unflatten( + data as any[], + revivers as Record any> + ); + }, +}; diff --git a/packages/core/src/serialization/compat.test.ts b/packages/core/src/serialization/compat.test.ts new file mode 100644 index 0000000000..b8e2580d1c --- /dev/null +++ b/packages/core/src/serialization/compat.test.ts @@ -0,0 +1,185 @@ +/** + * Compatibility tests: verify that data serialized by the new modules + * can be deserialized by the old serialization.ts functions, and vice versa. + * + * This ensures the new modules are safe to use alongside the old code + * during the migration period. + */ + +import { describe, it, expect } from 'vitest'; +import * as workflow from './workflow.js'; +import * as step from './step.js'; +import * as client from './client.js'; +import { + dehydrateWorkflowArguments, + hydrateWorkflowArguments, + dehydrateWorkflowReturnValue, + hydrateWorkflowReturnValue, + dehydrateStepArguments, + hydrateStepArguments, + dehydrateStepReturnValue, + hydrateStepReturnValue, +} from '../serialization.js'; +import { importKey } from '../encryption.js'; + +const testData = { + primitives: [42, 'hello', true, null], + date: new Date('2025-06-15T12:00:00Z'), + error: Object.assign(new Error('test'), { name: 'TypeError' }), + map: new Map([ + ['a', 1], + ['b', 2], + ]), + set: new Set([1, 2, 3]), + bigint: 9007199254740993n, + uint8: new Uint8Array([1, 2, 3]), + url: new URL('https://example.com'), + regexp: /foo.*bar/gi, + nested: { + items: [1, 'two', new Date('2025-01-01')], + inner: { x: 42 }, + }, +}; + +describe('new workflow.serialize → old hydrateStepReturnValue', () => { + it('should round-trip primitives', async () => { + for (const val of testData.primitives) { + const serialized = workflow.serialize(val); + const hydrated = await hydrateStepReturnValue( + serialized, + 'run-123', + undefined + ); + expect(hydrated).toEqual(val); + } + }); + + it('should round-trip Date', async () => { + const serialized = workflow.serialize(testData.date); + const hydrated = (await hydrateStepReturnValue( + serialized, + 'run-123', + undefined + )) as Date; + expect(hydrated).toBeInstanceOf(Date); + expect(hydrated.toISOString()).toBe('2025-06-15T12:00:00.000Z'); + }); + + it('should round-trip Map', async () => { + const serialized = workflow.serialize(testData.map); + const hydrated = (await hydrateStepReturnValue( + serialized, + 'run-123', + undefined + )) as Map; + expect(hydrated).toBeInstanceOf(Map); + expect(hydrated.get('a')).toBe(1); + }); + + it('should round-trip nested objects', async () => { + const serialized = workflow.serialize(testData.nested); + const hydrated = (await hydrateStepReturnValue( + serialized, + 'run-123', + undefined + )) as any; + expect(hydrated.items[0]).toBe(1); + expect(hydrated.items[2]).toBeInstanceOf(Date); + expect(hydrated.inner.x).toBe(42); + }); +}); + +describe('old dehydrateStepReturnValue → new workflow.deserialize', () => { + it('should round-trip primitives', async () => { + for (const val of testData.primitives) { + const dehydrated = await dehydrateStepReturnValue( + val, + 'run-123', + undefined, + [] + ); + const deserialized = workflow.deserialize(dehydrated); + expect(deserialized).toEqual(val); + } + }); + + it('should round-trip Date', async () => { + const dehydrated = await dehydrateStepReturnValue( + testData.date, + 'run-123', + undefined, + [] + ); + const deserialized = workflow.deserialize(dehydrated) as Date; + expect(deserialized).toBeInstanceOf(Date); + expect(deserialized.toISOString()).toBe('2025-06-15T12:00:00.000Z'); + }); + + it('should round-trip nested objects', async () => { + const dehydrated = await dehydrateStepReturnValue( + testData.nested, + 'run-123', + undefined, + [] + ); + const deserialized = workflow.deserialize(dehydrated) as any; + expect(deserialized.items[0]).toBe(1); + expect(deserialized.items[2]).toBeInstanceOf(Date); + }); +}); + +describe('old dehydrateWorkflowArguments → new workflow.deserialize', () => { + it('should round-trip when unencrypted', async () => { + const dehydrated = await dehydrateWorkflowArguments( + [42, 'hello'], + 'run-123', + undefined + ); + const deserialized = workflow.deserialize(dehydrated); + expect(deserialized).toEqual([42, 'hello']); + }); +}); + +describe('new client.serialize → old hydrateWorkflowArguments', () => { + it('should round-trip when unencrypted', async () => { + const serialized = await client.serialize([42, 'hello']); + const hydrated = await hydrateWorkflowArguments( + serialized, + 'run-123', + undefined + ); + expect(hydrated).toEqual([42, 'hello']); + }); +}); + +describe('encryption compat: new step.serialize → old hydrateStepArguments', () => { + it('should round-trip with encryption', async () => { + const rawKey = new Uint8Array(32); + rawKey.fill(0x42); + const key = await importKey(rawKey); + + const value = { x: 42, date: new Date('2025-01-01') }; + const serialized = await step.serialize(value, key); + const hydrated = (await hydrateStepArguments( + serialized, + 'run-123', + key + )) as any; + expect(hydrated.x).toBe(42); + expect(hydrated.date).toBeInstanceOf(Date); + }); +}); + +describe('encryption compat: old dehydrateStepArguments → new step.deserialize', () => { + it('should round-trip with encryption', async () => { + const rawKey = new Uint8Array(32); + rawKey.fill(0x42); + const key = await importKey(rawKey); + + const value = { x: 42, date: new Date('2025-01-01') }; + const dehydrated = await dehydrateStepArguments(value, 'run-123', key); + const deserialized = (await step.deserialize(dehydrated, key)) as any; + expect(deserialized.x).toBe(42); + expect(deserialized.date).toBeInstanceOf(Date); + }); +}); diff --git a/packages/core/src/serialization/reducers/common-vm.ts b/packages/core/src/serialization/reducers/common-vm.ts new file mode 100644 index 0000000000..da54470800 --- /dev/null +++ b/packages/core/src/serialization/reducers/common-vm.ts @@ -0,0 +1,542 @@ +/** + * VM-compatible common reducers and revivers. + * + * Identical to common.ts but without Node.js dependencies: + * - Uses native `btoa` / `atob` (provided by quickjs-wasi's base64 + * extension, see `quickjs-assets.generated.ts`) instead of Buffer + * or pure-JS base64. + * - Uses `instanceof Error` instead of `types.isNativeError()`. + * + * This module is safe to bundle into the QuickJS WASM VM. + */ + +import type { Reducers, Revivers, SerializableSpecial } from '../types.js'; + +// ---- Base64 helpers (native btoa/atob from the quickjs-wasi base64 extension) ---- + +function arrayBufferToBase64( + value: ArrayBufferLike, + offset: number, + length: number +): string { + if (length === 0) return '.'; + // btoa requires a binary string. Build it from the byte view. + const uint8 = new Uint8Array(value, offset, length); + let binary = ''; + for (let i = 0; i < uint8.length; i++) { + binary += String.fromCharCode(uint8[i]!); + } + return btoa(binary); +} + +function viewToBase64(value: ArrayBufferView): string { + return arrayBufferToBase64(value.buffer, value.byteOffset, value.byteLength); +} + +function reviveArrayBuffer(value: string): ArrayBuffer { + if (value === '.') return new ArrayBuffer(0); + const binary = atob(value); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes.buffer as ArrayBuffer; +} + +// ---- Error subclass helper ---- + +// Creates a reducer for a built-in Error subclass whose serialized shape +// is exactly { message, stack, cause? }. Matches by `value.name` +// (instance property) for cross-realm + bundler-output robustness — see +// the host-side common.ts for full rationale. +function makeNamedErrorSubclassReducer(subclassName: string) { + return ( + value: unknown + ): { message: string; stack?: string; cause?: unknown } | false => { + if (!(value instanceof Error)) return false; + if (value.name !== subclassName) return false; + const reduced: { message: string; stack?: string; cause?: unknown } = { + message: value.message, + stack: value.stack, + }; + if ('cause' in value) reduced.cause = (value as { cause: unknown }).cause; + return reduced; + }; +} + +// Creates a reviver for a built-in Error subclass. Looks up the +// constructor on globalThis so the resulting object passes +// `instanceof TypeError` etc. in the consuming realm. Falls back to +// a base Error with the right `.name` if the constructor is not +// available (defensive — built-ins always exist). +function makeNamedErrorSubclassReviver(subclassName: string) { + return (value: { message: string; stack?: string; cause?: unknown }) => { + const Cls = (globalThis as any)[subclassName]; + let error: Error; + if (typeof Cls === 'function') { + error = new Cls(value.message); + } else { + error = new Error(value.message); + error.name = subclassName; + } + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = (value as any).cause; + return error; + }; +} + +// ---- Reducers ---- + +export function getCommonReducers(): Partial { + return { + ArrayBuffer: (value) => + value instanceof ArrayBuffer && + arrayBufferToBase64(value, 0, value.byteLength), + BigInt: (value) => typeof value === 'bigint' && value.toString(), + BigInt64Array: (value) => + value instanceof BigInt64Array && viewToBase64(value), + BigUint64Array: (value) => + value instanceof BigUint64Array && viewToBase64(value), + Date: (value) => { + if (!(value instanceof Date)) return false; + const valid = !Number.isNaN(value.getDate()); + return valid ? value.toISOString() : '.'; + }, + // DOMException is checked before Error so that DOMException-specific + // shape (name, message, stack, cause) survives the round-trip. + DOMException: (value) => { + if (!(value instanceof DOMException)) return false; + const reduced: SerializableSpecial['DOMException'] = { + message: value.message, + name: value.name, + stack: value.stack, + }; + if ('cause' in value) reduced.cause = (value as any).cause; + return reduced; + }, + // First-class Error subclass reducers. Order matters: each subclass + // reducer is checked before the generic `Error` catch-all so that + // e.g. a TypeError instance routes through the TypeError reducer + // instead of the base Error reducer. Matching is by `value.name` + // (the instance property) for cross-realm + bundler robustness; + // see common.ts for full rationale. + AggregateError: (value) => { + if (!(value instanceof Error) || value.name !== 'AggregateError') + return false; + const reduced: SerializableSpecial['AggregateError'] = { + message: value.message, + stack: value.stack, + errors: (value as AggregateError).errors, + }; + if ('cause' in value) reduced.cause = (value as any).cause; + return reduced; + }, + EvalError: makeNamedErrorSubclassReducer('EvalError'), + FatalError: makeNamedErrorSubclassReducer('FatalError'), + // HookConflictError carries an extra token (+ optional + // conflictingRunId); mirror the host-side common.ts reducer. + HookConflictError: (value) => { + if (!(value instanceof Error) || value.name !== 'HookConflictError') + return false; + const reduced: SerializableSpecial['HookConflictError'] = { + message: value.message, + stack: value.stack, + token: (value as any).token, + }; + if ((value as any).conflictingRunId !== undefined) { + reduced.conflictingRunId = (value as any).conflictingRunId; + } + if ('cause' in value) reduced.cause = (value as any).cause; + return reduced; + }, + RangeError: makeNamedErrorSubclassReducer('RangeError'), + ReferenceError: makeNamedErrorSubclassReducer('ReferenceError'), + // RetryableError carries an extra retryAfter; serialize as numeric + // epoch timestamp for cross-realm safety (see host-side common.ts). + RetryableError: (value) => { + if (!(value instanceof Error) || value.name !== 'RetryableError') + return false; + const retryAfterRaw = (value as any).retryAfter; + let retryAfter: number; + if ( + retryAfterRaw && + typeof retryAfterRaw === 'object' && + typeof (retryAfterRaw as { getTime?: unknown }).getTime === 'function' + ) { + const t = (retryAfterRaw as Date).getTime(); + retryAfter = Number.isNaN(t) ? Date.now() + 1000 : t; + } else if ( + typeof retryAfterRaw === 'string' || + typeof retryAfterRaw === 'number' + ) { + const t = new Date(retryAfterRaw).getTime(); + retryAfter = Number.isNaN(t) ? Date.now() + 1000 : t; + } else { + retryAfter = Date.now() + 1000; + } + const reduced: SerializableSpecial['RetryableError'] = { + message: value.message, + stack: value.stack, + retryAfter, + }; + if ('cause' in value) reduced.cause = (value as any).cause; + return reduced; + }, + // RuntimeDecryptionError carries an extra `context` object (operation, + // byteLength, formatPrefix) that the generic Error reducer would drop. + RuntimeDecryptionError: (value) => { + if (!(value instanceof Error) || value.name !== 'RuntimeDecryptionError') + return false; + const reduced: SerializableSpecial['RuntimeDecryptionError'] = { + message: value.message, + stack: value.stack, + }; + const context = (value as any).context; + if (context !== undefined) { + reduced.context = context; + } + if ('cause' in value) reduced.cause = (value as any).cause; + return reduced; + }, + SyntaxError: makeNamedErrorSubclassReducer('SyntaxError'), + TypeError: makeNamedErrorSubclassReducer('TypeError'), + URIError: makeNamedErrorSubclassReducer('URIError'), + // Base Error reducer — catch-all. Matched LAST after subclass-specific + // reducers above. Preserves `name` so user Error subclasses without + // dedicated reducers retain their identity through the round-trip. + Error: (value) => { + // In the VM, use instanceof Error (no node:util available) + if (!(value instanceof Error)) return false; + const reduced: SerializableSpecial['Error'] = { + name: value.name, + message: value.message, + stack: value.stack, + }; + if ('cause' in value) reduced.cause = (value as any).cause; + return reduced; + }, + Float32Array: (value) => + value instanceof Float32Array && viewToBase64(value), + Float64Array: (value) => + value instanceof Float64Array && viewToBase64(value), + Int8Array: (value) => value instanceof Int8Array && viewToBase64(value), + Int16Array: (value) => value instanceof Int16Array && viewToBase64(value), + Int32Array: (value) => value instanceof Int32Array && viewToBase64(value), + Map: (value) => value instanceof Map && Array.from(value), + RegExp: (value) => + value instanceof RegExp && { + source: value.source, + flags: value.flags, + }, + // Request/Response/Headers — serialize using the polyfill constructors + Headers: (value) => { + const H = (globalThis as any).Headers; + if (!H || !(value instanceof H)) return false; + return Array.from(value as Iterable<[string, string]>); + }, + Request: (value) => { + const R = (globalThis as any).Request; + if (!R) return false; + // Use instanceof OR check for the Request-specific .json method + // (duck-typing on method/url alone would match plain objects and + // cause infinite recursion since the reducer output also has those) + if (!(value instanceof R) && typeof value?.json !== 'function') + return false; + if (typeof value?.method !== 'string') return false; + const data: any = { + method: value.method, + url: value.url, + headers: value.headers, + body: value.body, + duplex: value.duplex, + }; + // Include the webhook response writable stream if present + const responseWritable = value[Symbol.for('WEBHOOK_RESPONSE_WRITABLE')]; + if (responseWritable) { + data.responseWritable = responseWritable; + } + return data; + }, + Response: (value) => { + const R = (globalThis as any).Response; + if (!R) return false; + // Use instanceof OR check for Response-specific .clone method + if (!(value instanceof R) && typeof value?.clone !== 'function') + return false; + if (typeof value?.status !== 'number') return false; + return { + type: value.type, + url: value.url, + status: value.status, + statusText: value.statusText, + headers: value.headers, + body: value.body, + redirected: value.redirected, + }; + }, + ReadableStream: ((value: any) => { + if (value == null) return false; + const RS = (globalThis as any).ReadableStream; + if ( + !RS || + !(value instanceof RS || Object.getPrototypeOf(value) === RS.prototype) + ) + return false; + const bodyInit = value[Symbol.for('BODY_INIT')]; + if (bodyInit !== undefined) { + return { bodyInit }; + } + // Preserve stream name if present (opaque pointer for passing to steps) + const name = value[Symbol.for('WORKFLOW_STREAM_NAME')]; + if (name) { + const s: any = { name }; + const type = value[Symbol.for('WORKFLOW_STREAM_TYPE')]; + if (type) s.type = type; + return s; + } + return { name: '__empty' }; + }) as any, + WritableStream: ((value: any) => { + if (value == null) return false; + const WS = (globalThis as any).WritableStream; + if ( + !WS || + !(value instanceof WS || Object.getPrototypeOf(value) === WS.prototype) + ) + return false; + const name = value[Symbol.for('WORKFLOW_STREAM_NAME')]; + return { name: name || '__empty' }; + }) as any, + Set: (value) => value instanceof Set && Array.from(value), + URL: (value) => value instanceof URL && value.href, + WorkflowFunction: (value) => { + // Only match function references with a workflowId property (set by + // the SWC compiler on workflow functions). Plain { workflowId } objects + // are NOT matched — this prevents infinite recursion since the reduced + // form { workflowId } is a plain object, not a function. + if (typeof value !== 'function') return false; + const workflowId = (value as any).workflowId; + if (typeof workflowId !== 'string') return false; + return { workflowId }; + }, + URLSearchParams: (value) => { + if (!(value instanceof URLSearchParams)) return false; + return value.size === 0 ? '.' : String(value); + }, + Uint8Array: (value) => value instanceof Uint8Array && viewToBase64(value), + Uint8ClampedArray: (value) => + value instanceof Uint8ClampedArray && viewToBase64(value), + Uint16Array: (value) => value instanceof Uint16Array && viewToBase64(value), + Uint32Array: (value) => value instanceof Uint32Array && viewToBase64(value), + }; +} + +// ---- Revivers ---- + +export function getCommonRevivers(): Partial { + return { + ArrayBuffer: (value: string) => reviveArrayBuffer(value), + BigInt: (value: string) => BigInt(value), + BigInt64Array: (value: string) => + new BigInt64Array(reviveArrayBuffer(value)), + BigUint64Array: (value: string) => + new BigUint64Array(reviveArrayBuffer(value)), + Date: (value) => new Date(value), + DOMException: (value) => { + const error = new DOMException(value.message, value.name); + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = value.cause; + return error; + }, + AggregateError: (value) => { + const error = new AggregateError(value.errors ?? [], value.message); + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = value.cause; + return error; + }, + EvalError: makeNamedErrorSubclassReviver('EvalError'), + FatalError: (value) => { + // Prefer the host-registered FatalError class (registered via + // Symbol.for keys by @workflow/errors so `instanceof FatalError` + // works across realms). Fall back to a synthesized Error with + // the right .name when no registration is present. + // FatalError's constructor takes only `message`, so cause is + // attached as a property after construction (matching the host + // reviver in common.ts). + const Cls = (globalThis as any)[ + Symbol.for('@workflow/errors//FatalError') + ]; + let error: Error; + if (typeof Cls === 'function') { + error = new Cls(value.message); + } else { + error = new Error(value.message); + error.name = 'FatalError'; + } + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = (value as any).cause; + return error; + }, + HookConflictError: (value) => { + // Prefer the registered HookConflictError class (see the FatalError + // reviver above). Its constructor takes (token, conflictingRunId). + const Cls = (globalThis as any)[ + Symbol.for('@workflow/errors//HookConflictError') + ]; + let error: Error; + if (typeof Cls === 'function') { + error = new Cls(value.token, value.conflictingRunId); + } else { + error = new Error(value.message); + error.name = 'HookConflictError'; + (error as any).token = value.token; + if (value.conflictingRunId !== undefined) { + (error as any).conflictingRunId = value.conflictingRunId; + } + } + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = (value as any).cause; + return error; + }, + RangeError: makeNamedErrorSubclassReviver('RangeError'), + ReferenceError: makeNamedErrorSubclassReviver('ReferenceError'), + RetryableError: (value) => { + // RetryableError's constructor accepts (message, { retryAfter }). + // Cause is attached after construction (the constructor does not + // forward it). retryAfter is stored as a Date in the VM realm. + const Cls = (globalThis as any)[ + Symbol.for('@workflow/errors//RetryableError') + ]; + const retryAfter = new Date(value.retryAfter); + let error: Error; + if (typeof Cls === 'function') { + error = new Cls(value.message, { retryAfter }); + } else { + error = new Error(value.message); + error.name = 'RetryableError'; + (error as any).retryAfter = retryAfter; + } + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = (value as any).cause; + return error; + }, + RuntimeDecryptionError: (value) => { + // Prefer the registered RuntimeDecryptionError class (see the + // FatalError reviver above). Its constructor accepts + // (message, { cause, context }). + const Cls = (globalThis as any)[ + Symbol.for('@workflow/errors//RuntimeDecryptionError') + ]; + let error: Error; + if (typeof Cls === 'function') { + const opts: { cause?: unknown; context?: unknown } = {}; + if ('cause' in value) opts.cause = (value as any).cause; + if (value.context !== undefined) opts.context = value.context; + error = new Cls(value.message, opts); + } else { + error = new Error(value.message); + error.name = 'RuntimeDecryptionError'; + if (value.context !== undefined) { + (error as any).context = value.context; + } + if ('cause' in value) (error as any).cause = (value as any).cause; + } + if (value.stack !== undefined) error.stack = value.stack; + return error; + }, + SyntaxError: makeNamedErrorSubclassReviver('SyntaxError'), + TypeError: makeNamedErrorSubclassReviver('TypeError'), + URIError: makeNamedErrorSubclassReviver('URIError'), + Error: (value) => { + const error = new Error(value.message); + error.name = value.name; + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = (value as any).cause; + return error; + }, + Float32Array: (value: string) => new Float32Array(reviveArrayBuffer(value)), + Float64Array: (value: string) => new Float64Array(reviveArrayBuffer(value)), + Int8Array: (value: string) => new Int8Array(reviveArrayBuffer(value)), + Int16Array: (value: string) => new Int16Array(reviveArrayBuffer(value)), + Int32Array: (value: string) => new Int32Array(reviveArrayBuffer(value)), + Map: (value) => new Map(value), + RegExp: (value) => new RegExp(value.source, value.flags), + Set: (value) => new Set(value), + URL: (value) => new URL(value), + WorkflowFunction: (value) => + Object.assign( + () => { + throw new Error( + 'Workflow functions cannot be called directly. Use start() to invoke them.' + ); + }, + { workflowId: value.workflowId } + ), + URLSearchParams: (value) => new URLSearchParams(value === '.' ? '' : value), + Uint8Array: (value: string) => new Uint8Array(reviveArrayBuffer(value)), + Uint8ClampedArray: (value: string) => + new Uint8ClampedArray(reviveArrayBuffer(value)), + Uint16Array: (value: string) => new Uint16Array(reviveArrayBuffer(value)), + Uint32Array: (value: string) => new Uint32Array(reviveArrayBuffer(value)), + // Web API types — revived as plain objects in the VM since the real + // constructors (Headers, Request, Response) are not available in QuickJS. + // The workflow code can access the properties but not call Web API methods. + Headers: (value) => { + return new (globalThis as any).Headers(value); + }, + Request: (value: any) => { + const Req = (globalThis as any).Request; + if (Req) { + value.json = Req.prototype.json; + value.text = Req.prototype.text; + value.arrayBuffer = Req.prototype.arrayBuffer; + } + // Carry over the webhook response writable stream to the symbol property + if (value.responseWritable) { + value[Symbol.for('WEBHOOK_RESPONSE_WRITABLE')] = value.responseWritable; + } + return value; + }, + Response: (value: any) => { + // Don't use Object.setPrototypeOf — devalue continues to set properties + // on the object after the reviver runs, and getter-only properties + // (like 'ok') on the prototype would cause "no setter" errors. + // Instead, copy methods directly onto the object. + const Resp = (globalThis as any).Response; + if (Resp) { + value.json = Resp.prototype.json; + value.text = Resp.prototype.text; + value.arrayBuffer = Resp.prototype.arrayBuffer; + if (Resp.prototype.bytes) value.bytes = Resp.prototype.bytes; + if (Resp.prototype.clone) value.clone = Resp.prototype.clone; + } + value._body = value.body; + value.ok = value.status >= 200 && value.status < 300; + value.bodyUsed = false; + return value; + }, + ReadableStream: (value) => { + const RS = (globalThis as any).ReadableStream; + const stream = Object.create(RS ? RS.prototype : {}); + if (value && 'bodyInit' in value) { + // Body from Response/Request constructor — store the raw data + stream[Symbol.for('BODY_INIT')] = value.bodyInit; + } else if (value && 'name' in value) { + // Named stream reference — preserve the name/type for re-serialization. + // Streams are opaque pointers in the VM — they can be passed to steps + // but not consumed directly. + stream[Symbol.for('WORKFLOW_STREAM_NAME')] = value.name; + if (value.type) stream[Symbol.for('WORKFLOW_STREAM_TYPE')] = value.type; + } + return stream; + }, + WritableStream: (value) => { + const WS = (globalThis as any).WritableStream; + const stream = Object.create(WS ? WS.prototype : {}); + if (value && 'name' in value) { + stream[Symbol.for('WORKFLOW_STREAM_NAME')] = value.name; + } + return stream; + }, + }; +} diff --git a/packages/core/src/serialization/vm-bundle-entry.ts b/packages/core/src/serialization/vm-bundle-entry.ts new file mode 100644 index 0000000000..081a66f40f --- /dev/null +++ b/packages/core/src/serialization/vm-bundle-entry.ts @@ -0,0 +1,62 @@ +/** + * Entry point for the VM serialization bundle. + * + * This file is bundled by esbuild into a self-contained IIFE that + * sets up serialize/deserialize on globalThis. The bundled output + * is evaluated inside the QuickJS VM during bootstrap. + * + * TextEncoder, TextDecoder, and Headers are provided by native C + * extensions in quickjs-wasi, so no polyfills are needed. + */ + +import { monotonicFactory } from 'ulid'; +import { deserialize, serialize } from './workflow-vm.js'; + +// Install on global scope under the public well-known symbols. The +// snapshot runtime's bootstrap (and the various inline-evaluated JS +// strings in `snapshot-runtime.ts`) reach the same functions via +// `globalThis[Symbol.for('workflow-serialize')]` etc. +(globalThis as any)[Symbol.for('workflow-serialize')] = serialize; +(globalThis as any)[Symbol.for('workflow-deserialize')] = deserialize; + +// ULID generator for correlationIds — uses the same monotonicFactory +// as the node:vm engine. Both inputs MUST be set by the host before the +// first ULID is drawn, otherwise the seeded-ULID determinism guarantee +// is silently broken: +// +// * `Math.random` must be replaced with the host's seeded PRNG via +// `vm.newFunction('random', …)` (see `quickjs-runtime.ts`, the +// `Seeded Math.random` block). Two workflow invocations of the same +// run MUST observe an identical random sequence so their +// correlationIds collide and the world's EntityConflictError dedup +// applies. We pass it explicitly to `monotonicFactory` because +// ULID's auto-detect (`detectPRNG`) only knows about +// `crypto.getRandomValues` / `crypto.randomBytes`, neither of which +// exist in QuickJS. The PRNG is deliberately LATE-BOUND (the arrow +// reads `Math.random` at draw time, not at bundle-eval time) so +// that this bundle can be evaluated during static VM initialization +// — before the per-run seeded PRNG is installed — without capturing +// the unseeded built-in. This is also what allows a future VM +// snapshot taken after bundle eval to have its PRNG swapped +// post-restore. +// * `globalThis.__ulidTimestamp` must be a number (typically +// `workflowRun.startedAt`). It's used in place of `Date.now()` so +// the time portion of the ULID is also stable across concurrent +// invocations of the same run. +// +// The timestamp prerequisite is validated below — fail loudly rather +// than fall back to `Date.now()`, which would re-introduce +// non-determinism that replay relies on us NOT having. +const ulid = monotonicFactory(() => Math.random()); +(globalThis as any).__generateUlid = () => { + const t = (globalThis as any).__ulidTimestamp; + if (typeof t !== 'number') { + throw new Error( + '__generateUlid: globalThis.__ulidTimestamp must be a number set by ' + + 'the host before the serde bundle is evaluated. Without it, ULIDs ' + + 'would fall back to Date.now() and concurrent workflow invocations ' + + 'of the same resumption would produce divergent correlationIds.' + ); + } + return ulid(t); +}; diff --git a/packages/core/src/serialization/workflow-vm.test.ts b/packages/core/src/serialization/workflow-vm.test.ts new file mode 100644 index 0000000000..37faba45bd --- /dev/null +++ b/packages/core/src/serialization/workflow-vm.test.ts @@ -0,0 +1,186 @@ +/** + * Tests for the VM-compatible workflow serializer. + * + * Verifies that: + * 1. The VM serializer produces the same wire format as the Node.js serializer. + * 2. Data serialized by the VM can be deserialized by Node.js and vice versa. + */ + +import { describe, expect, it } from 'vitest'; +import { peekFormatPrefix } from './format.js'; +import { + deserialize as nodeDeserialize, + serialize as nodeSerialize, +} from './workflow.js'; +import { + deserialize as vmDeserialize, + serialize as vmSerialize, +} from './workflow-vm.js'; + +describe('VM workflow serializer', () => { + it('should produce format-prefixed output', () => { + const serialized = vmSerialize(42); + expect(serialized).toBeInstanceOf(Uint8Array); + expect(peekFormatPrefix(serialized)).toBe('devl'); + }); + + it('should round-trip primitives', () => { + for (const val of [42, 'hello', true, null, undefined]) { + expect(vmDeserialize(vmSerialize(val))).toEqual(val); + } + }); + + it('should round-trip Date', () => { + const date = new Date('2025-01-01T00:00:00Z'); + const result = vmDeserialize(vmSerialize(date)) as Date; + expect(result).toBeInstanceOf(Date); + expect(result.toISOString()).toBe('2025-01-01T00:00:00.000Z'); + }); + + it('should round-trip Map', () => { + const map = new Map([ + ['a', 1], + ['b', 2], + ]); + const result = vmDeserialize(vmSerialize(map)) as Map; + expect(result).toBeInstanceOf(Map); + expect(result.get('a')).toBe(1); + }); + + it('should round-trip Uint8Array', () => { + const u8 = new Uint8Array([1, 2, 3, 4, 5]); + const result = vmDeserialize(vmSerialize(u8)) as Uint8Array; + expect(result).toBeInstanceOf(Uint8Array); + expect(Array.from(result)).toEqual([1, 2, 3, 4, 5]); + }); + + it('should round-trip nested objects', () => { + const val = { a: 1, b: [2, new Date('2025-01-01')], c: { d: 'e' } }; + const result = vmDeserialize(vmSerialize(val)) as any; + expect(result.a).toBe(1); + expect(result.b[0]).toBe(2); + expect(result.b[1]).toBeInstanceOf(Date); + expect(result.c.d).toBe('e'); + }); + + it('should round-trip WorkflowFunction reference', () => { + // Simulate an SWC-compiled workflow function: a function with a + // `workflowId` property that the runtime treats as an opaque handle. + const fn = Object.assign(() => {}, { + workflowId: 'workflow//./src/foo//myWorkflow', + }); + const revived = vmDeserialize(vmSerialize(fn)) as any; + expect(typeof revived).toBe('function'); + expect(revived.workflowId).toBe('workflow//./src/foo//myWorkflow'); + // Calling the revived stub throws — workflow functions must be invoked + // via start(), not directly. + expect(() => revived()).toThrow(/Use start\(\)/); + }); + + it('should round-trip DOMException', () => { + const ex = new DOMException('boom', 'AbortError'); + const revived = vmDeserialize(vmSerialize(ex)) as Error; + // The revived value is a DOMException (or Error fallback with the same + // name) — either way it should preserve name/message and be instanceof Error. + expect(revived).toBeInstanceOf(Error); + expect(revived.name).toBe('AbortError'); + expect(revived.message).toBe('boom'); + }); +}); + +describe('VM ↔ Node.js cross-compatibility', () => { + it('VM serialize → Node.js deserialize', () => { + const values = [ + 42, + 'hello', + new Date('2025-06-15'), + new Map([['x', 1]]), + new Set([1, 2, 3]), + new Uint8Array([10, 20, 30]), + { nested: { arr: [1, 2, 3] } }, + ]; + for (const val of values) { + const vmBytes = vmSerialize(val); + const nodeResult = nodeDeserialize(vmBytes); + const vmResult = vmDeserialize(vmBytes); + // Both should produce equivalent values + expect(JSON.stringify(nodeResult)).toBe(JSON.stringify(vmResult)); + } + }); + + it('Node.js serialize → VM deserialize', () => { + const values = [ + 42, + 'hello', + new Date('2025-06-15'), + new Map([['x', 1]]), + new Set([1, 2, 3]), + new Uint8Array([10, 20, 30]), + { nested: { arr: [1, 2, 3] } }, + ]; + for (const val of values) { + const nodeBytes = nodeSerialize(val); + const vmResult = vmDeserialize(nodeBytes); + const nodeResult = nodeDeserialize(nodeBytes); + expect(JSON.stringify(vmResult)).toBe(JSON.stringify(nodeResult)); + } + }); + + it('step args format: VM serialize → Node.js hydrateStepArguments', async () => { + // This is the critical path: VM serializes step args, step handler deserializes + const { hydrateStepArguments } = await import('../serialization.js'); + + const stepInput = { args: [10, 7], closureVars: { x: 42 } }; + const vmBytes = vmSerialize(stepInput); + + const hydrated = (await hydrateStepArguments( + vmBytes, + 'run-123', + undefined + )) as any; + expect(hydrated.args).toEqual([10, 7]); + expect(hydrated.closureVars).toEqual({ x: 42 }); + }); + + it('Node.js serialize TypeError → VM deserialize keeps subclass identity + cause', () => { + const cause = new TypeError('underlying'); + const wrapped = new Error('outer'); + (wrapped as any).cause = cause; + const nodeBytes = nodeSerialize(wrapped); + const result = vmDeserialize(nodeBytes) as Error; + expect(result).toBeInstanceOf(Error); + expect(result.message).toBe('outer'); + expect((result as any).cause).toBeInstanceOf(TypeError); + expect(((result as any).cause as Error).message).toBe('underlying'); + }); + + it('Node.js serialize built-in subclasses → VM deserialize preserves type identity', () => { + const cases: Array<[Error, new (...args: any[]) => Error]> = [ + [new TypeError('t'), TypeError], + [new RangeError('r'), RangeError], + [new SyntaxError('s'), SyntaxError], + [new ReferenceError('rf'), ReferenceError], + ]; + for (const [err, ctor] of cases) { + const result = vmDeserialize(nodeSerialize(err)) as Error; + expect(result).toBeInstanceOf(ctor); + expect(result.message).toBe(err.message); + } + }); + + it('step result format: Node.js dehydrateStepReturnValue → VM deserialize', async () => { + // This is the other critical path: step handler serializes result, VM deserializes + const { dehydrateStepReturnValue } = await import('../serialization.js'); + + const result = { sum: 17, computed: true }; + const nodeBytes = await dehydrateStepReturnValue( + result, + 'run-123', + undefined, + [] + ); + const vmResult = vmDeserialize(nodeBytes) as any; + expect(vmResult.sum).toBe(17); + expect(vmResult.computed).toBe(true); + }); +}); diff --git a/packages/core/src/serialization/workflow-vm.ts b/packages/core/src/serialization/workflow-vm.ts new file mode 100644 index 0000000000..61fc9d14a4 --- /dev/null +++ b/packages/core/src/serialization/workflow-vm.ts @@ -0,0 +1,73 @@ +/** + * VM-compatible workflow mode serialization. + * + * This module is designed to be bundled into the QuickJS WASM VM. + * It has NO Node.js dependencies (no Buffer, no node:util). + * + * Produces and consumes the same wire format as the Node.js workflow.ts — + * format-prefixed devalue data ("devl" + devalue.stringify output). + */ + +import { devalueVmCodec } from './codec-devalue-vm.js'; +import { SerializationFormat, isFormatPrefix } from './types.js'; + +const FORMAT_PREFIX_LENGTH = 4; +let _encoder: { encode(s: string): Uint8Array }; +let _decoder: { decode(d: Uint8Array): string }; +function getEncoder() { + if (!_encoder) _encoder = new (globalThis as any).TextEncoder(); + return _encoder; +} +function getDecoder() { + if (!_decoder) _decoder = new (globalThis as any).TextDecoder(); + return _decoder; +} + +/** + * Serialize a value to format-prefixed bytes. + * + * @param value - The value to serialize + * @returns Uint8Array with "devl" prefix + devalue payload + */ +export function serialize(value: unknown): Uint8Array { + const payload = devalueVmCodec.serialize(value, 'workflow'); + const prefix = getEncoder().encode(SerializationFormat.DEVALUE_V1); + const result = new Uint8Array(prefix.length + payload.length); + result.set(prefix, 0); + result.set(payload, prefix.length); + return result; +} + +/** + * Deserialize format-prefixed bytes back to a value. + * + * @param data - Uint8Array with format prefix, or legacy non-binary data + * @returns The deserialized value + */ +export function deserialize(data: Uint8Array | unknown): unknown { + // Legacy: non-binary data + if (!(data instanceof Uint8Array)) { + if (devalueVmCodec.deserializeLegacy) { + return devalueVmCodec.deserializeLegacy(data, 'workflow'); + } + throw new Error( + 'Cannot deserialize non-binary data without legacy support' + ); + } + + if (data.length < FORMAT_PREFIX_LENGTH) { + throw new Error('Data too short to contain format prefix'); + } + + const prefixStr = getDecoder().decode(data.subarray(0, FORMAT_PREFIX_LENGTH)); + if (!isFormatPrefix(prefixStr)) { + throw new Error(`Invalid format prefix: "${prefixStr}"`); + } + + if (prefixStr === SerializationFormat.DEVALUE_V1) { + const payload = data.subarray(FORMAT_PREFIX_LENGTH); + return devalueVmCodec.deserialize(payload, 'workflow'); + } + + throw new Error(`Unsupported serialization format: ${prefixStr}`); +} diff --git a/packages/core/src/source-map.test.ts b/packages/core/src/source-map.test.ts index f178760abe..5c7e12361a 100644 --- a/packages/core/src/source-map.test.ts +++ b/packages/core/src/source-map.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { remapErrorStack } from './source-map.js'; +import { remapErrorStack, stripInlineSourceMap } from './source-map.js'; describe('remapErrorStack', () => { afterEach(() => { @@ -99,3 +99,48 @@ describe('remapErrorStack', () => { ).toBe(false); }); }); + +describe('stripInlineSourceMap', () => { + it('returns the input unchanged when there is no inline map', () => { + const code = 'const x = 1;\nconsole.log(x);\n'; + expect(stripInlineSourceMap(code)).toBe(code); + }); + + it('strips a trailing inline source map comment', () => { + const code = + 'var workflow = { name: "test" };\nconst result = workflow.name;\n' + + '//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozfQ==\n'; + const stripped = stripInlineSourceMap(code); + expect(stripped).not.toMatch(/sourceMappingURL/); + expect(stripped).toContain('var workflow'); + expect(stripped).toContain('workflow.name'); + }); + + it('strips a long source map comment without trailing newline', () => { + // Many bundlers emit the comment as the very last line with no + // trailing newline. The regex must match end-of-input too. + const longBase64 = 'A'.repeat(4 * 1024 * 1024); // 4 MB of payload + const code = `globalThis.x = 1;\n//# sourceMappingURL=data:application/json;base64,${longBase64}`; + const stripped = stripInlineSourceMap(code); + expect(stripped).not.toMatch(/sourceMappingURL/); + expect(stripped.length).toBeLessThan(code.length); + // The bundle proper is preserved — only the trailing comment is gone. + expect(stripped).toContain('globalThis.x = 1;'); + }); + + it('only strips the trailing inline map (not embedded substrings)', () => { + // A workflow could legitimately contain the literal string + // "sourceMappingURL" inside JS code (e.g. inside a string literal + // for an unrelated reason). The regex anchors to end-of-line/end + // and only matches the comment form, so non-comment occurrences + // are preserved. + const code = ` +const literal = "sourceMappingURL=foo"; +console.log(literal); +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbXSwibWFwcGluZ3MiOiIifQ== +`; + const stripped = stripInlineSourceMap(code); + expect(stripped).toContain(`"sourceMappingURL=foo"`); + expect(stripped).not.toMatch(/\/\/# sourceMappingURL/); + }); +}); diff --git a/packages/core/src/source-map.ts b/packages/core/src/source-map.ts index 14425825b6..5773b421fc 100644 --- a/packages/core/src/source-map.ts +++ b/packages/core/src/source-map.ts @@ -1,5 +1,31 @@ import { originalPositionFor, TraceMap } from '@jridgewell/trace-mapping'; +/** + * Pattern matching the trailing inline source map comment that bundlers + * (esbuild, etc.) emit. The comment is purely host-side metadata for + * `remapErrorStack` — the VM never needs it. Stripping it before + * passing the bundle to `vm.evalCode` materially reduces the QuickJS + * heap, because QuickJS retains source text for stack-trace line + * lookups. + */ +const INLINE_SOURCE_MAP_COMMENT_RE = + /\/\/# sourceMappingURL=data:application\/json;base64,[A-Za-z0-9+/=]+\s*$/m; + +/** + * Strip the trailing `//# sourceMappingURL=data:…` comment from a JS + * bundle. Returns the input unchanged if no inline map is present. + * + * Use this on the host side before evaluating workflow bundles inside + * the QuickJS VM — the inline map can account for several MB of bundle + * text (measured ~30%+ of VM heap bytes on the example workbench's + * bundle), and the VM never needs it; only host-side `remapErrorStack` + * reads the map (and it can do so against the original, unstripped + * string). + */ +export function stripInlineSourceMap(workflowCode: string): string { + return workflowCode.replace(INLINE_SOURCE_MAP_COMMENT_RE, ''); +} + function isBase64Char(code: number): boolean { return ( (code >= 0x41 && code <= 0x5a) || diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index 8ab58fe88b..f0e0c532e7 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -92,6 +92,36 @@ export const WorkflowTracePropagated = SemanticConvention( 'workflow.trace.propagated' ); +// QuickJS VM engine attributes + +/** The VM engine executing the workflow function for this invocation */ +export const WorkflowVm = SemanticConvention<'node' | 'quickjs'>('workflow.vm'); + +/** Outcome of a QuickJS VM workflow invocation */ +export const QuickJSOutcome = SemanticConvention< + 'completed' | 'suspended' | 'failed' +>('quickjs.outcome'); + +/** Whether preloaded events from `events.create('run_started')` were used */ +export const QuickJSEventsPreloaded = SemanticConvention( + 'quickjs.events.preloaded' +); + +/** Total number of events fetched from the world for this invocation */ +export const QuickJSEventsFetchedCount = SemanticConvention( + 'quickjs.events.fetched_count' +); + +/** Number of pages required to fetch all events */ +export const QuickJSEventsFetchedPages = SemanticConvention( + 'quickjs.events.fetched_pages' +); + +/** Number of pending VM operations captured at suspension */ +export const QuickJSPendingOpsCount = SemanticConvention( + 'quickjs.pending_ops_count' +); + /** Active trace-correlation mode for this invocation (linked or continuous) */ export const WorkflowTraceMode = SemanticConvention<'linked' | 'continuous'>( 'workflow.trace.mode' diff --git a/packages/core/turbo.json b/packages/core/turbo.json index e503fb6757..aa04cd0e81 100644 --- a/packages/core/turbo.json +++ b/packages/core/turbo.json @@ -3,7 +3,12 @@ "tasks": { "build": { "dependsOn": ["^build"], - "outputs": ["dist", "src/version.ts"] + "outputs": [ + "dist", + "src/version.ts", + "src/runtime/vm-serde-bundle.generated.ts", + "src/runtime/quickjs-assets.generated.ts" + ] } } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index da1422c2b2..49c576847c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -555,6 +555,9 @@ importers: nanoid: specifier: 5.1.6 version: 5.1.6 + quickjs-wasi: + specifier: 3.1.0 + version: 3.1.0 seedrandom: specifier: 3.0.5 version: 3.0.5 @@ -15051,6 +15054,9 @@ packages: resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} engines: {node: '>=10'} + quickjs-wasi@3.1.0: + resolution: {integrity: sha512-Vw2g4GhAh/QVgPIoDRgpPMBv9Z+E1LjUGgwrLewjjvTqONtty0GukgE+2IoZU1Z4anNF3uZIWA5EtBa3m0QiWQ==} + radix-ui@1.4.3: resolution: {integrity: sha512-aWizCQiyeAenIdUbqEpXgRA1ya65P13NKn/W8rWkcN0OPkRDxdBVLWnIEDsS2RpwCK2nobI7oMUSmexzTDyAmA==} peerDependencies: @@ -33220,6 +33226,8 @@ snapshots: quick-lru@5.1.1: {} + quickjs-wasi@3.1.0: {} + radix-ui@1.4.3(@types/react-dom@19.1.9(@types/react@19.1.13))(@types/react@19.1.13)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: '@radix-ui/primitive': 1.1.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 52da4f2ff1..35ce8560cd 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -55,3 +55,4 @@ minimumReleaseAgeExclude: - '@workflow/*' - 'esbuild' - '@esbuild/*' + - 'quickjs-wasi' diff --git a/scripts/create-test-matrix.mjs b/scripts/create-test-matrix.mjs index 821197b6cf..284a27113e 100644 --- a/scripts/create-test-matrix.mjs +++ b/scripts/create-test-matrix.mjs @@ -164,4 +164,21 @@ matrix.app.push({ ...DEV_TEST_CONFIGS['tanstack-start'], }); +// QuickJS WASM VM engine leg (opt-in via WORKFLOW_VM=quickjs). One app is +// enough while the engine is experimental — nextjs-turbopack is the most +// feature-complete workbench. The `vm` field is surfaced to the workflow +// dev server via the WORKFLOW_VM env var in tests.yml. +matrix.app.push( + createMatrixEntry( + 'nextjs-turbopack', + 'example-nextjs-workflow-turbopack', + DEV_TEST_CONFIGS['nextjs-turbopack'], + { + vm: 'quickjs', + runLabel: 'quickjs', + artifactSuffix: 'quickjs', + } + ) +); + console.log(JSON.stringify(matrix)); From adb3d0e2b1ee3e2529ed064c11eafecb6499cc85 Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 22 Jul 2026 01:00:26 -0700 Subject: [PATCH 02/18] QuickJS engine: AbortController, setAttributes, terminal drain, turbo-safe requeue, stable PRNG seed --- packages/core/src/runtime.ts | 16 +- .../core/src/runtime/quickjs-entrypoint.ts | 531 +++++++++++------- .../core/src/runtime/quickjs-runtime.test.ts | 166 ++++++ packages/core/src/runtime/quickjs-runtime.ts | 310 +++++++++- .../src/serialization/codec-devalue-vm.ts | 78 +++ 5 files changed, 902 insertions(+), 199 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index dd06ca975a..98193167e6 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1622,6 +1622,12 @@ export function workflowEntrypoint( workflowRunId: runId, loopIteration, }); + // Under turbo, run_started is backgrounded. The QuickJS + // entrypoint fetches the event log and writes events + // directly, so wait for the run to be durably started + // first — it does not thread the turbo runReadyBarrier + // the way handleSuspension does. + await awaitRunReady(); const quickjsResult = await runWorkflowWithQuickJS({ workflowCode, workflowName, @@ -1631,7 +1637,15 @@ export function workflowEntrypoint( parentSpan: span, }); if (quickjsResult?.timeoutSeconds !== undefined) { - return { timeoutSeconds: quickjsResult.timeoutSeconds }; + // Use `reinvoke` rather than returning + // `{ timeoutSeconds }` directly: under turbo the + // current message carries `runInput` and a reschedule + // would re-engage turbo on redelivery (replaying + // against a stale preloaded log and wedging the run — + // see the reinvoke() docs above). reinvoke enqueues an + // explicit continuation without `runInput` in that + // case. + return await reinvoke(quickjsResult.timeoutSeconds); } return; } diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index ea538a13b2..5ad1ae125b 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -43,6 +43,7 @@ import { getWorkflowQueueName, queueMessage } from './helpers.js'; import { type PendingAttribute, type PendingHook, + type PendingOperation, type PendingStep, type PendingWait, runQuickJSWorkflow, @@ -82,6 +83,292 @@ export function isFirstInvocation( ); } +/** + * Dispatch durable side effects for a set of pending VM operations: + * step_created (+ optional queueing), hook_created / hook_received (aborts), + * attr_set, hook_disposed, and wait_created events. + * + * Used in two modes: + * - suspension (queueSteps: true): normal suspension processing; new steps + * are queued for execution. + * - terminal drain (queueSteps: false): flush leftover side effects when + * the workflow completed or failed — mirrors the node:vm engine's + * drainPendingQueueItems. Steps are created but NOT queued, and the run + * is never requeued. + */ +async function dispatchPendingOps(params: { + world: Awaited>; + runId: string; + workflowRun: WorkflowRun; + encryptionKey: Awaited> | undefined; + pendingOperations: PendingOperation[]; + queueSteps: boolean; + wfdiag: (checkpoint: string, fields: Record) => void; +}): Promise<{ createdAttributeEvent: boolean }> { + const { world, runId, workflowRun, encryptionKey, pendingOperations } = + params; + const wfdiag = params.wfdiag; + // Set when a new attr_set event is written this invocation. The + // workflow must be re-invoked to consume it (resolving the pending + // setAttributes() promise), so the entrypoint requeues immediately — + // same pattern as an elapsed wait. + let createdAttributeEvent = false; + const opsPromises: Promise[] = []; + + for (const op of pendingOperations) { + if (op.type === 'step' && !op.hasCreatedEvent) { + const step = op as PendingStep; + opsPromises.push( + (async () => { + // Create step_created event. `step.input` is the + // format-prefixed devalue bytes ("devl" + devalue) produced + // by `globalThis[Symbol.for('workflow-serialize')]({args, + // closureVars, thisVal})` inside the VM. The VM has no + // access to the CryptoKey, so encryption is applied here + // on the host side — matching what + // `dehydrateStepArguments` does in the node:vm engine. + try { + await world.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: step.correlationId, + eventData: { + stepName: step.stepId, + input: await encryptSerializedData(step.input, encryptionKey), + }, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + + // Queue the step execution via the unified workflow queue + // (V2 architecture). The combined handler in runtime.ts + // dispatches messages with `stepId` to executeStep, which + // works for both VM engines — so the QuickJS engine reuses + // the same step execution path as the node:vm engine + // instead of needing a separate step route. Skipped in + // terminal-drain mode (the workflow already finished; the + // event is the durable record, matching the node:vm drain). + if (params.queueSteps) { + const traceCarrier = await serializeTraceCarrier(); + await queueMessage( + world, + getWorkflowQueueName(workflowRun.workflowName), + { + runId, + stepId: step.correlationId, + stepName: step.stepId, + traceCarrier, + requestedAt: new Date(), + }, + { + idempotencyKey: step.correlationId, + } + ); + wfdiag('step_queued', { + stepId: step.stepId, + correlationId: step.correlationId, + }); + } + })() + ); + } else if ( + op.type === 'hook' && + (!op.hasCreatedEvent || (op as PendingHook).abortRequested) + ) { + const hook = op as PendingHook; + runtimeLogger.debug('QuickJS runtime: processing hook op', { + workflowRunId: runId, + correlationId: hook.correlationId, + token: hook.token, + tokenType: typeof hook.token, + isWebhook: hook.isWebhook, + isSystem: hook.isSystem, + hasCreatedEvent: hook.hasCreatedEvent, + abortRequested: hook.abortRequested, + }); + + opsPromises.push( + (async () => { + if (!hook.hasCreatedEvent) { + // `hook.metadata` is the format-prefixed devalue bytes + // produced by `globalThis[Symbol.for('workflow-serialize')] + // (options.metadata)` inside the VM. Encrypt on the host + // side before writing — matches the node:vm engine's + // `dehydrateStepArguments` flow. + // + // No pre-check via hooks.list: with deterministic correlationIds + // (same VM seed across replays) and per-(runId, correlationId) + // uniqueness in worlds, the storage layer rejects duplicates as + // EntityConflictError, which we swallow below. This drops one + // network round-trip per pending hook. + try { + const encryptedMetadata = + typeof hook.metadata === 'undefined' + ? undefined + : await encryptSerializedData(hook.metadata, encryptionKey); + const result = await world.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: hook.correlationId, + eventData: { + token: hook.token, + metadata: encryptedMetadata, + // Always include isWebhook explicitly. Worlds default it to + // `true` when absent, which would break the public webhook + // endpoint's 404 guard for hooks created via createHook(). + isWebhook: hook.isWebhook, + // System hooks (AbortController) are exempt from user + // token namespace conflict checks. + ...(hook.isSystem ? { isSystem: true } : {}), + } as any, + }); + + // If storage detected a real token conflict with another + // workflow's hook, re-queue so the workflow handler can + // process the conflict event and fail gracefully. + if (result.event?.eventType === 'hook_conflict') { + await queueMessage( + world, + `__wkf_workflow_${workflowRun.workflowName}`, + { + runId, + }, + { idempotencyKey: `hook_conflict_${hook.correlationId}` } + ); + } + } catch (err) { + // Already created by a concurrent invocation — fall through + // to abort processing below (if any) instead of bailing. + if (!EntityConflictError.is(err)) throw err; + } + } + + if (hook.abortRequested) { + // Record the abort durably: a hook_received event carrying + // the VM-serialized `{ aborted: true, reason }` payload, + // plus a best-effort stream packet for real-time step + // propagation. Mirrors the node:vm engine's suspension + // handler (hooksNeedingAbort). + const abortPayload = + hook.abortPayload instanceof Uint8Array + ? ((await encryptSerializedData( + hook.abortPayload, + encryptionKey + )) as Uint8Array) + : undefined; + try { + await world.events.create(runId, { + eventType: 'hook_received', + specVersion: SPEC_VERSION_CURRENT, + correlationId: hook.correlationId, + eventData: { + token: hook.token, + payload: abortPayload, + } as any, + }); + } catch (err) { + if (!EntityConflictError.is(err)) throw err; + } + // streamName is derived from the abort hook token + // (`abrt_{id}` → `strm_{id}_system_abort`). + if (hook.token.startsWith('abrt_') && abortPayload) { + const streamName = `strm_${hook.token.slice('abrt_'.length)}_system_abort`; + try { + await world.streams.write(runId, streamName, abortPayload); + await world.streams.close(runId, streamName); + } catch { + // Best-effort — the hook event provides the durable + // fallback. + runtimeLogger.debug( + 'QuickJS runtime: failed to write abort stream packet', + { + workflowRunId: runId, + correlationId: hook.correlationId, + } + ); + } + } + wfdiag('abort_recorded', { + correlationId: hook.correlationId, + token: hook.token, + }); + } + })() + ); + } else if (op.type === 'attribute' && !op.hasCreatedEvent) { + const attr = op as PendingAttribute; + opsPromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'attr_set', + specVersion: SPEC_VERSION_CURRENT, + correlationId: attr.correlationId, + eventData: { + changes: attr.changes, + writer: { type: 'workflow' }, + ...(attr.allowReservedAttributes + ? { allowReservedAttributes: true } + : {}), + } as any, + }); + createdAttributeEvent = true; + } catch (err) { + if (EntityConflictError.is(err)) { + // Event already exists (concurrent invocation) — the + // replay still needs to consume it, so requeue. + createdAttributeEvent = true; + return; + } + throw err; + } + })() + ); + } else if (op.type === 'hook_dispose' && !op.hasCreatedEvent) { + opsPromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'hook_disposed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: op.correlationId, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } else if (op.type === 'wait' && !op.hasCreatedEvent) { + const wait = op as PendingWait; + opsPromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'wait_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: wait.correlationId, + eventData: { + resumeAt: new Date(wait.resumeAt), + }, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } + } + + // Per-op dispatch runs in parallel. + await Promise.all(opsPromises); + + return { createdAttributeEvent }; +} + /** * Run a workflow using the QuickJS WASM VM engine. * @@ -318,6 +605,30 @@ export async function runWorkflowWithQuickJS(params: { ...Attribute.QuickJSOutcome('completed'), }); + // Flush leftover pending side effects (abort recordings, system-hook + // disposals, fire-and-forget attribute/hook events) BEFORE writing + // run_completed — mirrors the node:vm engine's drainPendingQueueItems. + // Drain failures are swallowed: the workflow's own outcome is the + // source of truth. + if (result.completed.drainOperations?.length) { + try { + await dispatchPendingOps({ + world, + runId, + workflowRun, + encryptionKey, + pendingOperations: result.completed.drainOperations, + queueSteps: false, + wfdiag, + }); + } catch (err) { + runtimeLogger.warn('QuickJS runtime: terminal drain failed', { + workflowRunId: runId, + message: (err as Error)?.message, + }); + } + } + // Create run_completed event. // The VM serializes the workflow result as format-prefixed devalue bytes // ("devl" + devalue) with no encryption (the VM has no access to the @@ -385,196 +696,15 @@ export async function runWorkflowWithQuickJS(params: { // on cloud worlds (e.g. Vercel) where each storage call is a // network round-trip. let minTimeoutSeconds: number | undefined; - // Set when a new attr_set event is written this invocation. The - // workflow must be re-invoked to consume it (resolving the pending - // setAttributes() promise), so the entrypoint requeues immediately — - // same pattern as an elapsed wait. - let createdAttributeEvent = false; - const opsPromises: Promise[] = []; - - for (const op of pendingOperations) { - if (op.type === 'step' && !op.hasCreatedEvent) { - const step = op as PendingStep; - opsPromises.push( - (async () => { - // Create step_created event. `step.input` is the - // format-prefixed devalue bytes ("devl" + devalue) produced - // by `globalThis[Symbol.for('workflow-serialize')]({args, - // closureVars, thisVal})` inside the VM. The VM has no - // access to the CryptoKey, so encryption is applied here - // on the host side — matching what - // `dehydrateStepArguments` does in the node:vm engine. - try { - await world.events.create(runId, { - eventType: 'step_created', - specVersion: SPEC_VERSION_CURRENT, - correlationId: step.correlationId, - eventData: { - stepName: step.stepId, - input: await encryptSerializedData(step.input, encryptionKey), - }, - }); - } catch (err) { - if (EntityConflictError.is(err)) return; - throw err; - } - - // Queue the step execution via the unified workflow queue - // (V2 architecture). The combined handler in runtime.ts - // dispatches messages with `stepId` to executeStep, which - // works for both VM engines — so the QuickJS engine reuses - // the same step execution path as the node:vm engine - // instead of needing a separate step route. - const traceCarrier = await serializeTraceCarrier(); - await queueMessage( - world, - getWorkflowQueueName(workflowRun.workflowName), - { - runId, - stepId: step.correlationId, - stepName: step.stepId, - traceCarrier, - requestedAt: new Date(), - }, - { - idempotencyKey: step.correlationId, - } - ); - wfdiag('step_queued', { - stepId: step.stepId, - correlationId: step.correlationId, - }); - })() - ); - } else if (op.type === 'hook' && !op.hasCreatedEvent) { - const hook = op as PendingHook; - runtimeLogger.debug('QuickJS runtime: creating hook_created event', { - workflowRunId: runId, - correlationId: hook.correlationId, - token: hook.token, - tokenType: typeof hook.token, - isWebhook: hook.isWebhook, - }); - - opsPromises.push( - (async () => { - // `hook.metadata` is the format-prefixed devalue bytes - // produced by `globalThis[Symbol.for('workflow-serialize')] - // (options.metadata)` inside the VM. Encrypt on the host - // side before writing — matches the node:vm engine's - // `dehydrateStepArguments` flow. - // - // No pre-check via hooks.list: with deterministic correlationIds - // (same VM seed across replays) and per-(runId, correlationId) - // uniqueness in worlds, the storage layer rejects duplicates as - // EntityConflictError, which we swallow below. This drops one - // network round-trip per pending hook. - try { - const encryptedMetadata = - typeof hook.metadata === 'undefined' - ? undefined - : await encryptSerializedData(hook.metadata, encryptionKey); - const result = await world.events.create(runId, { - eventType: 'hook_created', - specVersion: SPEC_VERSION_CURRENT, - correlationId: hook.correlationId, - eventData: { - token: hook.token, - metadata: encryptedMetadata, - // Always include isWebhook explicitly. Worlds default it to - // `true` when absent, which would break the public webhook - // endpoint's 404 guard for hooks created via createHook(). - isWebhook: hook.isWebhook, - } as any, - }); - - // If storage detected a real token conflict with another - // workflow's hook, re-queue so the workflow handler can - // process the conflict event and fail gracefully. - if (result.event?.eventType === 'hook_conflict') { - await queueMessage( - world, - `__wkf_workflow_${workflowRun.workflowName}`, - { - runId, - }, - { idempotencyKey: `hook_conflict_${hook.correlationId}` } - ); - } - } catch (err) { - if (EntityConflictError.is(err)) return; - throw err; - } - })() - ); - } else if (op.type === 'attribute' && !op.hasCreatedEvent) { - const attr = op as PendingAttribute; - opsPromises.push( - (async () => { - try { - await world.events.create(runId, { - eventType: 'attr_set', - specVersion: SPEC_VERSION_CURRENT, - correlationId: attr.correlationId, - eventData: { - changes: attr.changes, - writer: { type: 'workflow' }, - ...(attr.allowReservedAttributes - ? { allowReservedAttributes: true } - : {}), - } as any, - }); - createdAttributeEvent = true; - } catch (err) { - if (EntityConflictError.is(err)) { - // Event already exists (concurrent invocation) — the - // replay still needs to consume it, so requeue. - createdAttributeEvent = true; - return; - } - throw err; - } - })() - ); - } else if (op.type === 'hook_dispose' && !op.hasCreatedEvent) { - opsPromises.push( - (async () => { - try { - await world.events.create(runId, { - eventType: 'hook_disposed', - specVersion: SPEC_VERSION_CURRENT, - correlationId: op.correlationId, - }); - } catch (err) { - if (EntityConflictError.is(err)) return; - throw err; - } - })() - ); - } else if (op.type === 'wait' && !op.hasCreatedEvent) { - const wait = op as PendingWait; - opsPromises.push( - (async () => { - try { - await world.events.create(runId, { - eventType: 'wait_created', - specVersion: SPEC_VERSION_CURRENT, - correlationId: wait.correlationId, - eventData: { - resumeAt: new Date(wait.resumeAt), - }, - }); - } catch (err) { - if (EntityConflictError.is(err)) return; - throw err; - } - })() - ); - } - } - - // Per-op dispatch runs in parallel. - await Promise.all(opsPromises); + const { createdAttributeEvent } = await dispatchPendingOps({ + world, + runId, + workflowRun, + encryptionKey, + pendingOperations, + queueSteps: true, + wfdiag, + }); // Handle pending waits — both newly created and still-pending from // earlier invocations. For each wait, either create a wait_completed @@ -679,6 +809,27 @@ export async function runWorkflowWithQuickJS(params: { ...Attribute.QuickJSOutcome('failed'), }); + // Flush leftover pending side effects before writing run_failed — + // same drain semantics as the completed branch. + if (result.failed.drainOperations?.length) { + try { + await dispatchPendingOps({ + world, + runId, + workflowRun, + encryptionKey, + pendingOperations: result.failed.drainOperations, + queueSteps: false, + wfdiag, + }); + } catch (err) { + runtimeLogger.warn('QuickJS runtime: terminal drain failed', { + workflowRunId: runId, + message: (err as Error)?.message, + }); + } + } + // Create run_failed event. Serialize the error through the // first-class dehydration pipeline so consumers (CLI, observability, // run.returnValue) get the same hydrated value shape as the node:vm diff --git a/packages/core/src/runtime/quickjs-runtime.test.ts b/packages/core/src/runtime/quickjs-runtime.test.ts index 920d0c61db..3e5751a051 100644 --- a/packages/core/src/runtime/quickjs-runtime.test.ts +++ b/packages/core/src/runtime/quickjs-runtime.test.ts @@ -555,3 +555,169 @@ describe('deterministic replay clock', () => { expect(unwrapResult(r3.completed!.result)).toEqual(result); }); }); + +describe('AbortController (hook-backed)', () => { + it('registers a system hook and surfaces abort requests at suspension', async () => { + const run = makeRun(); + const result = await runQuickJSWorkflow({ + workflowCode: ` + var slowStep = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//slow"); + async function workflow() { + var controller = new AbortController(); + var p = slowStep(controller.signal); + controller.abort(new Error("stop it")); + return await p; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + + expect(result.suspended).toBeDefined(); + const ops = result.suspended!.pendingOperations; + const hookOp = ops.find((o) => o.type === 'hook') as any; + expect(hookOp).toBeDefined(); + expect(hookOp.isSystem).toBe(true); + expect(hookOp.token).toMatch(/^abrt_/); + expect(hookOp.abortRequested).toBe(true); + expect(hookOp.abortPayload).toBeInstanceOf(Uint8Array); + // The aborted signal was serialized into the step input by symbol. + const stepOp = ops.find((o) => o.type === 'step'); + expect(stepOp).toBeDefined(); + }); + + it('delivers a recorded abort to the signal on replay', async () => { + const code = ` + var checkStep = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//check"); + async function workflow() { + var controller = new AbortController(); + var observed = []; + controller.signal.addEventListener("abort", function() { + observed.push("listener:" + (controller.signal.reason && controller.signal.reason.message)); + }); + await checkStep(1); + // On replay, the recorded hook_received flips the signal during + // event processing, so this abort() is a no-op. + controller.abort(new Error("stop it")); + return { + aborted: controller.signal.aborted, + reason: controller.signal.reason && controller.signal.reason.message, + observed: observed, + }; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + const run = makeRun(); + + const r1 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + const ops1 = r1.suspended!.pendingOperations; + const hookOp = ops1.find((o) => o.type === 'hook') as any; + const stepOp = ops1.find((o) => o.type === 'step') as any; + + // Simulate the entrypoint having recorded step completion, the hook + // creation, and the abort (hook_received with serialized payload from + // a prior invocation's abortPayload). + const { serialize } = await import('../serialization/workflow-vm.js'); + const abortPayload = serialize({ + aborted: true, + reason: new Error('stop it'), + }); + + const r2 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [ + runCreatedEvent(run), + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'hook_created', + correlationId: hookOp.correlationId, + eventData: { token: hookOp.token, isWebhook: false, isSystem: true }, + createdAt: new Date('2025-01-01T00:00:01Z'), + }, + { + eventId: 'evnt_002', + runId: run.runId, + eventType: 'step_created', + correlationId: stepOp.correlationId, + eventData: { stepName: 'step//test//check' }, + createdAt: new Date('2025-01-01T00:00:02Z'), + }, + { + eventId: 'evnt_003', + runId: run.runId, + eventType: 'step_completed', + correlationId: stepOp.correlationId, + eventData: { result: 1 }, + createdAt: new Date('2025-01-01T00:00:03Z'), + }, + { + eventId: 'evnt_004', + runId: run.runId, + eventType: 'hook_received', + correlationId: hookOp.correlationId, + eventData: { token: hookOp.token, payload: abortPayload }, + createdAt: new Date('2025-01-01T00:00:04Z'), + }, + ], + }); + + const value = unwrapResult(r2.completed!.result) as { + aborted: boolean; + reason?: string; + observed: string[]; + }; + expect(value.aborted).toBe(true); + expect(value.reason).toBe('stop it'); + expect(value.observed).toEqual(['listener:stop it']); + }); + + it('AbortSignal statics work in the VM', async () => { + const result = await runQuickJSWorkflow({ + workflowCode: ` + async function workflow() { + var pre = AbortSignal.abort(new Error("pre")); + var composite = AbortSignal.any([pre]); + var live = new AbortController(); + var mixed = AbortSignal.any([live.signal]); + live.abort(new Error("live")); + var timeoutThrew = false; + try { AbortSignal.timeout(1000); } catch (e) { timeoutThrew = true; } + return { + pre: pre.aborted && pre.reason.message, + composite: composite.aborted && composite.reason.message, + mixed: mixed.aborted && mixed.reason.message, + timeoutThrew: timeoutThrew, + }; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [], + }); + + // The workflow aborts a live controller, so it suspends with the abort + // request pending... unless it completes first — the return happens + // synchronously after abort(), so the workflow completes and the + // abort request is moot. Either outcome must expose the values. + expect(result.completed).toBeDefined(); + const value = unwrapResult(result.completed!.result) as any; + expect(value.pre).toBe('pre'); + expect(value.composite).toBe('pre'); + expect(value.mixed).toBe('live'); + expect(value.timeoutThrew).toBe(true); + }); +}); diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index 71daee7bb9..3a29958de1 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -85,6 +85,21 @@ export interface PendingHook { isWebhook: boolean; metadata?: unknown; hasCreatedEvent: boolean; + /** + * True for internal system hooks (e.g. AbortController's hook), which + * are exempt from user-hook token namespace conflict checks. + */ + isSystem?: boolean; + /** + * Set when the workflow called AbortController.abort() during this + * invocation. The host must record the abort: create a hook_received + * event carrying `abortPayload` and write/close the abort stream. + */ + abortRequested?: boolean; + /** VM-serialized `{ aborted: true, reason }` payload for the abort. */ + abortPayload?: Uint8Array; + /** Set by the completion drain when a system hook is implicitly disposed. */ + disposed?: boolean; } export interface PendingAttribute { @@ -112,7 +127,17 @@ export type PendingOperation = export interface QuickJSRuntimeResult { /** The workflow completed — result is format-prefixed devalue bytes */ - completed?: { result: Uint8Array }; + completed?: { + result: Uint8Array; + /** + * Leftover pending operations that still need durable side effects at + * completion: abort recordings, system-hook disposals, fire-and-forget + * attribute/hook/step events. Mirrors the node:vm engine's + * drainPendingQueueItems. The entrypoint dispatches these WITHOUT + * queueing steps or requeuing the run. + */ + drainOperations?: PendingOperation[]; + }; /** The workflow suspended with pending operations */ suspended?: { pendingOperations: PendingOperation[]; @@ -122,6 +147,8 @@ export interface QuickJSRuntimeResult { message: string; stack?: string; name?: string; + /** See completed.drainOperations — same semantics on failure. */ + drainOperations?: PendingOperation[]; /** * Format-prefixed devalue bytes of the original thrown value * (Error subclass with cause chain, plain object, primitive, etc.). @@ -502,6 +529,164 @@ globalThis[Symbol.for("WORKFLOW_SET_ATTRIBUTES")] = function(changes, options) { }); }; +// ---- AbortController / AbortSignal (hook-backed) ---- +// Port of workflow/abort-controller.ts to the VM pending-op model: +// the controller registers a system hook; abort() flips the signal +// synchronously and marks the pending op so the host records the abort +// (hook_received event + stream packet). On replay, the recorded +// hook_received event calls _setAborted during event processing and the +// workflow's own abort() call becomes a no-op. +var __ABORT_STREAM_NAME = Symbol.for("WORKFLOW_ABORT_STREAM_NAME"); +var __ABORT_HOOK_TOKEN = Symbol.for("WORKFLOW_ABORT_HOOK_TOKEN"); + +function __makeAbortError() { + if (typeof DOMException !== "undefined") { + return new DOMException("The operation was aborted.", "AbortError"); + } + var e = new Error("The operation was aborted."); + e.name = "AbortError"; + return e; +} + +function WorkflowAbortSignal(streamName, hookToken) { + this.aborted = false; + this.reason = undefined; + this[__ABORT_STREAM_NAME] = streamName; + this[__ABORT_HOOK_TOKEN] = hookToken; + this.__listeners = []; + this.__onabort = null; +} +Object.defineProperty(WorkflowAbortSignal.prototype, "onabort", { + get: function() { return this.__onabort; }, + set: function(handler) { + this.__onabort = handler; + if (handler && this.aborted) handler.call(this); + }, +}); +WorkflowAbortSignal.prototype._setAborted = function(reason) { + if (this.aborted) return; + this.aborted = true; + this.reason = reason; + if (this.__onabort) this.__onabort.call(this); + var listeners = this.__listeners; + this.__listeners = []; + for (var i = 0; i < listeners.length; i++) listeners[i](); +}; +WorkflowAbortSignal.prototype.addEventListener = function(type, listener) { + if (type !== "abort") return; + if (this.aborted) { + // Fire synchronously (not on a microtask) for deterministic replay — + // matches the node:vm engine's WorkflowAbortSignal. + listener(); + return; + } + this.__listeners.push(listener); +}; +WorkflowAbortSignal.prototype.removeEventListener = function(type, listener) { + if (type !== "abort") return; + this.__listeners = this.__listeners.filter(function(l) { return l !== listener; }); +}; +WorkflowAbortSignal.prototype.throwIfAborted = function() { + if (this.aborted) { + throw this.reason !== undefined && this.reason !== null + ? this.reason + : __makeAbortError(); + } +}; +// Expose for the serde bundle's revivers (evaluated before this bootstrap; +// they look the class up lazily at revive time). +globalThis.__WorkflowAbortSignal = WorkflowAbortSignal; + +// Registry of live abort signals keyed by their hook correlationId. The +// host delivers hook_received events for these ids as _setAborted calls. +globalThis.__abortSignals = {}; + +globalThis.AbortController = function WorkflowAbortController() { + var id = globalThis.__generateUlid(); + var streamName = "strm_" + id + "_system_abort"; + var hookToken = "abrt_" + id; + this[__ABORT_STREAM_NAME] = streamName; + this[__ABORT_HOOK_TOKEN] = hookToken; + this.signal = new WorkflowAbortSignal(streamName, hookToken); + var correlationId = "hook_" + globalThis.__generateUlid(); + // Register an internal system hook. isSystem prevents token namespace + // conflicts with user hooks. + globalThis.__pending.push({ + type: "hook", + correlationId: correlationId, + token: hookToken, + isWebhook: false, + isSystem: true, + hasCreatedEvent: false, + }); + globalThis.__abortSignals[correlationId] = this.signal; +}; +globalThis.AbortController.prototype.abort = function(reason) { + if (this.signal.aborted) return; // already aborted (e.g. from replay) + this.signal._setAborted(reason); + // Mark the pending hook op so the host records the abort. The payload + // is serialized in the VM so the reason crosses the boundary with + // type fidelity (Errors, DOMException, custom values). + var token = this[__ABORT_HOOK_TOKEN]; + for (var i = 0; i < globalThis.__pending.length; i++) { + var item = globalThis.__pending[i]; + if (item.type === "hook" && item.token === token) { + item.abortRequested = true; + item.abortPayload = globalThis[Symbol.for("workflow-serialize")]({ + aborted: true, + reason: reason, + }); + break; + } + } +}; + +globalThis.AbortSignal = { + abort: function(reason) { + var s = new WorkflowAbortSignal("", ""); + s._setAborted(reason !== undefined ? reason : __makeAbortError()); + return s; + }, + any: function(signals) { + var composite = new WorkflowAbortSignal("", ""); + var arr = Array.from(signals); + for (var i = 0; i < arr.length; i++) { + if (arr[i].aborted) { + composite._setAborted(arr[i].reason); + return composite; + } + } + var listeners = []; + var cleanup = function() { + for (var j = 0; j < listeners.length; j++) { + if (listeners[j].signal.removeEventListener) { + listeners[j].signal.removeEventListener("abort", listeners[j].listener); + } + } + listeners.length = 0; + }; + arr.forEach(function(signal) { + if (!signal.addEventListener) return; + var listener = function() { + if (!composite.aborted) { + composite._setAborted(signal.reason); + cleanup(); + } + }; + listeners.push({ signal: signal, listener: listener }); + signal.addEventListener("abort", listener); + }); + return composite; + }, + timeout: function() { + throw new Error( + "AbortSignal.timeout() is not supported in workflow functions. " + + "Use sleep() with an AbortController instead. " + + "See: /docs/errors/abort-signal-timeout-in-workflow" + ); + }, +}; + // WORKFLOW_GET_STREAM_ID — generates a stream ID for a workflow run. // Replicates getWorkflowRunStreamId() from util.ts inside the QuickJS VM. // Uses the built-in btoa() for base64url encoding. @@ -586,10 +771,16 @@ export async function runQuickJSWorkflow( // the world's per-(runId, correlationId) uniqueness turns the duplicate // `events.create` into an EntityConflictError that the entrypoint // swallows. + // + // The seed inputs MUST be stable across invocations. Notably + // `startedAt` is NOT: under turbo the first invocation runs against a + // synthesized run object whose timestamps differ from the durably + // stored ones that later invocations load. Matches the node:vm + // engine's seed (workflow.ts). const seed = [ workflowRun.runId, workflowRun.workflowName, - String(startedAt), + workflowRun.deploymentId, ].join(':'); const rng = seedrandom(seed); @@ -637,9 +828,12 @@ export async function runQuickJSWorkflow( // produce IDENTICAL correlationIds (the random portion also matches // because the PRNG is seeded the same way) and the world's // EntityConflictError on `events.create` dedups one of each pair. - // Use `startedAt` (constant per-run) so the prefix is stable across - // replay invocations too. - vm.evalCode(`globalThis.__ulidTimestamp = ${startedAt};`).dispose(); + // Derived from the runId's embedded ULID (stable across invocations by + // construction — unlike `startedAt`, which differs between turbo's + // synthesized run object and the durably stored run). + vm.evalCode( + `globalThis.__ulidTimestamp = ${runIdCreatedAt(workflowRun.runId) ?? (+workflowRun.createdAt || startedAt)};` + ).dispose(); // Execute the workflow bundle — use the workflowId as the eval filename // so QuickJS stack traces reference the workflow name, enabling source map @@ -999,6 +1193,53 @@ async function processEvents( markCreated(vm, escapedCid); break; } + + // Abort delivery: hook_received for an AbortController's system + // hook flips the registered signal instead of resolving a promise. + // The payload is the dehydrated `{ aborted: true, reason }` object. + const isAbortHook = vm.dump( + vm.evalCode( + `!!(globalThis.__abortSignals && globalThis.__abortSignals["${escapedCid}"])` + ) + ); + if (isAbortHook) { + const rawAbortPayload = eventData?.payload; + if (rawAbortPayload instanceof Uint8Array) { + const decrypted = await prepareBytesForVM( + rawAbortPayload, + encryptionKey + ); + const bytesHandle = vm.newUint8Array(decrypted); + vm.setProp(vm.global, '__tmp_abort', bytesHandle); + bytesHandle.dispose(); + vm.evalCode( + `(function(){` + + `var p=globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_abort);` + + `delete globalThis.__tmp_abort;` + + `globalThis.__abortSignals["${escapedCid}"]._setAborted(p&&typeof p==="object"?p.reason:undefined);` + + `})()` + ).dispose(); + } else { + vm.evalCode( + `globalThis.__abortSignals["${escapedCid}"]._setAborted(undefined);` + ).dispose(); + } + if (event.eventId) { + vm.evalCode( + `(globalThis.__hookPayloadBuffer.__processedEventIds = globalThis.__hookPayloadBuffer.__processedEventIds || {})[${JSON.stringify(event.eventId)}] = true;` + ).dispose(); + } + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + markCreated(vm, escapedCid); + break; + } + const hasResolver = vm.dump( vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) ); @@ -1151,14 +1392,58 @@ function markCreated(vm: QuickJS, escapedCid: string, opType?: string): void { // ---- State Checking ---- +/** + * Collect leftover pending operations that need durable side effects when + * the workflow reaches a terminal state. Mirrors the node:vm engine's + * drainPendingQueueItems (workflow.ts): still-alive system hooks + * (AbortController) without an abort in flight are implicitly disposed so + * they don't leak hook rows; ops without created events (fire-and-forget + * attributes/hooks/steps/waits) and pending abort recordings are surfaced + * for the entrypoint to flush. + */ +function collectDrainOperations(vm: QuickJS): PendingOperation[] { + using h = vm.evalCode(`(function(){ + var toDispose = []; + globalThis.__pending.forEach(function(p){ + if (p.type === "hook" && p.isSystem && !p.abortRequested && !p.disposed) { + p.disposed = true; + // Only dispose hooks that were durably created; a hook that never + // reached storage has nothing to clean up. + if (p.hasCreatedEvent) { + toDispose.push({ + type: "hook_dispose", + correlationId: p.correlationId, + hasCreatedEvent: false, + }); + } + } + }); + toDispose.forEach(function(d){ globalThis.__pending.push(d); }); + return globalThis.__pending.filter(function(p){ + if (p.abortRequested) return true; + if (p.hasCreatedEvent) return false; + // Skip system hooks that were disposed before ever being created. + if (p.type === "hook" && p.disposed) return false; + return true; + }); + })()`); + return vm.dump(h) as PendingOperation[]; +} + function checkWorkflowState(vm: QuickJS): QuickJSRuntimeResult { // Check completed — __workflowResult is a format-prefixed Uint8Array { using h = vm.evalCode('globalThis.__workflowResult'); if (!h.isUndefined) { const resultBytes = h.toUint8Array(); + const drainOperations = collectDrainOperations(vm); vm.dispose(); - return { completed: { result: resultBytes } }; + return { + completed: { + result: resultBytes, + ...(drainOperations.length > 0 ? { drainOperations } : {}), + }, + }; } } @@ -1188,8 +1473,14 @@ function checkWorkflowState(vm: QuickJS): QuickJSRuntimeResult { errorName: failed.name, errorStack: failed.stack, }); + const drainOperations = collectDrainOperations(vm); vm.dispose(); - return { failed }; + return { + failed: { + ...failed, + ...(drainOperations.length > 0 ? { drainOperations } : {}), + }, + }; } } @@ -1202,7 +1493,10 @@ function checkWorkflowState(vm: QuickJS): QuickJSRuntimeResult { ); if (vm.dump(h)) { using pendingH = vm.evalCode( - `globalThis.__pending.filter(function(p){return!!globalThis.__resolvers[p.correlationId] || !p.hasCreatedEvent;})` + // Ops with an active resolver or without a created event are + // pending; abort-requested hooks are also surfaced (even when + // already created and unawaited) so the host records the abort. + `globalThis.__pending.filter(function(p){return!!globalThis.__resolvers[p.correlationId] || !p.hasCreatedEvent || p.abortRequested;})` ); const pendingOps = vm.dump(pendingH) as PendingOperation[]; vm.dispose(); diff --git a/packages/core/src/serialization/codec-devalue-vm.ts b/packages/core/src/serialization/codec-devalue-vm.ts index 3713132058..fafabb10c8 100644 --- a/packages/core/src/serialization/codec-devalue-vm.ts +++ b/packages/core/src/serialization/codec-devalue-vm.ts @@ -18,10 +18,87 @@ import { const encoder = new TextEncoder(); const decoder = new TextDecoder(); +// ---- AbortController / AbortSignal (workflow VM context) ---- +// Mirrors the node:vm engine's workflow-context abort reducers/revivers in +// serialization.ts: reduce by reading the stream/hook symbols stamped at +// controller construction; revive to the bootstrap's WorkflowAbortSignal +// class (looked up lazily on globalThis — the serde bundle is evaluated +// before the bootstrap defines it). +const ABORT_STREAM_NAME = Symbol.for('WORKFLOW_ABORT_STREAM_NAME'); +const ABORT_HOOK_TOKEN = Symbol.for('WORKFLOW_ABORT_HOOK_TOKEN'); + +type AbortSerialized = { + streamName: string; + hookToken: string; + aborted: boolean; + reason?: unknown; +}; + +function reduceAbortBySymbol( + signal: { aborted: boolean; reason?: unknown }, + holder: any +): AbortSerialized { + const streamName = + holder[ABORT_STREAM_NAME] ?? holder.signal?.[ABORT_STREAM_NAME]; + const hookToken = + holder[ABORT_HOOK_TOKEN] ?? holder.signal?.[ABORT_HOOK_TOKEN]; + if (!streamName) { + throw new Error('AbortController/AbortSignal stream name is not set'); + } + return { + streamName, + hookToken, + aborted: signal.aborted, + reason: signal.aborted ? signal.reason : undefined, + }; +} + +function reviveAbortSignalVM(value: AbortSerialized) { + const Cls = (globalThis as any).__WorkflowAbortSignal; + if (typeof Cls !== 'function') { + throw new Error( + 'WorkflowAbortSignal is not registered in the VM (bootstrap not evaluated)' + ); + } + const signal = new Cls(value.streamName, value.hookToken); + if (value.aborted) signal._setAborted(value.reason); + return signal; +} + +function getAbortReducersVM(): Partial { + return { + AbortController: (value) => { + if (!value || typeof value !== 'object' || !value.signal) return false; + const hasAbortSymbol = + value[ABORT_STREAM_NAME] ?? value.signal?.[ABORT_STREAM_NAME]; + if (hasAbortSymbol === undefined) return false; + return reduceAbortBySymbol(value.signal, value); + }, + AbortSignal: (value) => { + if (!value || typeof value !== 'object') return false; + if ((value as any)[ABORT_STREAM_NAME] === undefined) return false; + return reduceAbortBySymbol(value as any, value); + }, + }; +} + +function getAbortReviversVM(): Partial { + return { + AbortController: (value: AbortSerialized) => ({ + [ABORT_STREAM_NAME]: value.streamName, + [ABORT_HOOK_TOKEN]: value.hookToken, + signal: reviveAbortSignalVM(value), + abort: () => {}, + }), + AbortSignal: (value: AbortSerialized) => reviveAbortSignalVM(value), + }; +} + function getReducersForMode(mode: SerializationMode): Partial { switch (mode) { case 'workflow': return { + ...getAbortReducersVM(), ...getClassReducers(), ...getStepFunctionReducer(), ...getCommonReducers(), @@ -43,6 +120,7 @@ function getReviversForMode(mode: SerializationMode): Partial { switch (mode) { case 'workflow': return { + ...getAbortReviversVM(), ...getClassRevivers(), ...getStepFunctionReviver(), ...getCommonRevivers(), From 6be76e2ba621af1cf08d4857ebad58e60471e779 Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 22 Jul 2026 01:12:35 -0700 Subject: [PATCH 03/18] QuickJS engine: hook.getConflict support, cross-run writable forwarding symbols --- .../core/src/runtime/quickjs-entrypoint.ts | 47 ++++-- packages/core/src/runtime/quickjs-runtime.ts | 147 +++++++++++++++--- .../src/serialization/reducers/common-vm.ts | 30 +++- 3 files changed, 189 insertions(+), 35 deletions(-) diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index 5ad1ae125b..a7827718a0 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -104,10 +104,17 @@ async function dispatchPendingOps(params: { pendingOperations: PendingOperation[]; queueSteps: boolean; wfdiag: (checkpoint: string, fields: Record) => void; -}): Promise<{ createdAttributeEvent: boolean }> { +}): Promise<{ + createdAttributeEvent: boolean; + createdGetConflictHook: boolean; +}> { const { world, runId, workflowRun, encryptionKey, pendingOperations } = params; const wfdiag = params.wfdiag; + // Set when a hook with a parked getConflict() awaiter had its + // hook_created written this invocation. The workflow must be re-invoked + // so replay can confirm creation and resolve the awaiter. + let createdGetConflictHook = false; // Set when a new attr_set event is written this invocation. The // workflow must be re-invoked to consume it (resolving the pending // setAttributes() promise), so the entrypoint requeues immediately — @@ -243,6 +250,9 @@ async function dispatchPendingOps(params: { // to abort processing below (if any) instead of bailing. if (!EntityConflictError.is(err)) throw err; } + if (hook.hasGetConflictAwaiter) { + createdGetConflictHook = true; + } } if (hook.abortRequested) { @@ -366,7 +376,7 @@ async function dispatchPendingOps(params: { // Per-op dispatch runs in parallel. await Promise.all(opsPromises); - return { createdAttributeEvent }; + return { createdAttributeEvent, createdGetConflictHook }; } /** @@ -696,15 +706,16 @@ export async function runWorkflowWithQuickJS(params: { // on cloud worlds (e.g. Vercel) where each storage call is a // network round-trip. let minTimeoutSeconds: number | undefined; - const { createdAttributeEvent } = await dispatchPendingOps({ - world, - runId, - workflowRun, - encryptionKey, - pendingOperations, - queueSteps: true, - wfdiag, - }); + const { createdAttributeEvent, createdGetConflictHook } = + await dispatchPendingOps({ + world, + runId, + workflowRun, + encryptionKey, + pendingOperations, + queueSteps: true, + wfdiag, + }); // Handle pending waits — both newly created and still-pending from // earlier invocations. For each wait, either create a wait_completed @@ -748,12 +759,16 @@ export async function runWorkflowWithQuickJS(params: { await Promise.all(waitCompletePromises); } - if (needsRequeue || createdAttributeEvent) { - // An elapsed wait was completed or a new attr_set event was - // written — re-queue immediately so the next invocation can - // process the new event. + if (needsRequeue || createdAttributeEvent || createdGetConflictHook) { + // An elapsed wait was completed, a new attr_set event was written, + // or a getConflict()-awaited hook was created — re-queue immediately + // so the next invocation can process the new event. wfdiag('exit_suspended', { - action: needsRequeue ? 'wait_elapsed_requeue' : 'attr_set_requeue', + action: needsRequeue + ? 'wait_elapsed_requeue' + : createdAttributeEvent + ? 'attr_set_requeue' + : 'get_conflict_requeue', timeoutSeconds: 0, }); return { timeoutSeconds: 0 }; diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index 3a29958de1..e9ef30ce79 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -100,6 +100,12 @@ export interface PendingHook { abortPayload?: Uint8Array; /** Set by the completion drain when a system hook is implicitly disposed. */ disposed?: boolean; + /** + * True when the workflow is awaiting hook.getConflict() for this hook. + * The entrypoint re-invokes the workflow right after writing + * hook_created so replay can confirm creation and resolve the awaiter. + */ + hasGetConflictAwaiter?: boolean; } export interface PendingAttribute { @@ -435,14 +441,28 @@ globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")] = function(options) { // Register in pending operations. // Serialize metadata inside the VM so Response/Request objects are // properly handled by the devalue reducers before crossing the boundary. - globalThis.__pending.push({ + var pendingOp = { type: "hook", correlationId: correlationId, token: token, isWebhook: !!options.isWebhook, metadata: options.metadata ? globalThis[Symbol.for("workflow-serialize")](options.metadata) : undefined, hasCreatedEvent: false, - }); + }; + globalThis.__pending.push(pendingOp); + + // Per-hook lifecycle state backing hook.getConflict(): resolves null + // once creation is confirmed (hook_created), or resolves with the + // conflicting Run handle / rejects with HookConflictError on + // hook_conflict. State transitions are driven by the host during event + // processing (see processEvents). + globalThis.__hooks = globalThis.__hooks || {}; + globalThis.__hooks[correlationId] = { + token: token, + created: false, + conflict: null, + getConflictResolvers: [], + }; // Each await creates a new promise for the next payload. // The correlationId stays the same — the resolver is replaced each time. @@ -475,11 +495,32 @@ globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")] = function(options) { } } + function getConflict() { + var state = globalThis.__hooks[correlationId]; + if (state.conflict) { + return state.conflict.run + ? Promise.resolve(state.conflict.run) + : Promise.reject(state.conflict.error); + } + if (state.created) { + return Promise.resolve(null); + } + // Creation not yet confirmed by the event log — park the awaiter and + // flag the pending op so the entrypoint re-invokes the workflow right + // after writing hook_created (nothing external resumes a getConflict + // awaiter; confirmation only comes from replaying the new event). + pendingOp.hasGetConflictAwaiter = true; + return new Promise(function(resolve, reject) { + state.getConflictResolvers.push({ resolve: resolve, reject: reject }); + }); + } + var hook = { token: token, then: function(onFulfilled, onRejected) { return createHookPromise().then(onFulfilled, onRejected); }, + getConflict: getConflict, dispose: disposeHook, }; @@ -1336,22 +1377,67 @@ async function processEvents( break; } case 'hook_conflict': { - const hasResolver = vm.dump( - vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) - ); - if (hasResolver) { - const conflictToken = (eventData?.token as string) ?? 'unknown'; + // Another workflow owns this hook token. Payload awaiters reject + // with HookConflictError; getConflict() awaiters resolve with a + // Run handle for the conflicting run (revived through the VM's + // class registry so its methods are durable step proxies) or + // reject with the error when no handle can be constructed — + // mirroring the node:vm engine's hook.ts hook_conflict handling. + const conflictToken = (eventData?.token as string) ?? 'unknown'; + const conflictingRunId = eventData?.conflictingRunId as + | string + | undefined; + const didSettle = vm.dump( vm.evalCode( - `globalThis.__resolvers["${escapedCid}"].reject(new Error(${JSON.stringify(`Hook token "${conflictToken}" is already in use by another workflow`)}));` + - `delete globalThis.__resolvers["${escapedCid}"];` - ).dispose(); - { - resolved = true; - let b: number; - do { - b = vm.executePendingJobs(); - } while (b > 0); - } + `(function(){ + var cid = ${JSON.stringify(cid)}; + var token = ${JSON.stringify(conflictToken)}; + var conflictingRunId = ${JSON.stringify(conflictingRunId ?? null)}; + var ErrCls = globalThis[Symbol.for('@workflow/errors//HookConflictError')]; + var err; + if (typeof ErrCls === 'function') { + err = new ErrCls(token, conflictingRunId || undefined); + } else { + err = new Error('Hook token "' + token + '" is already in use by another workflow'); + err.name = 'HookConflictError'; + err.token = token; + if (conflictingRunId) err.conflictingRunId = conflictingRunId; + } + var run = null; + if (conflictingRunId) { + var reg = globalThis[Symbol.for('workflow-class-registry')]; + var RunCls = reg && reg.get('class//workflow//Run'); + var des = RunCls && RunCls[Symbol.for('workflow-deserialize')]; + if (typeof des === 'function') { + run = des.call(RunCls, { runId: conflictingRunId }); + } + } + var settled = false; + var state = globalThis.__hooks && globalThis.__hooks[cid]; + if (state && !state.conflict) { + state.conflict = { error: err, run: run }; + var gc = state.getConflictResolvers; + state.getConflictResolvers = []; + for (var i = 0; i < gc.length; i++) { + if (run) { gc[i].resolve(run); } else { gc[i].reject(err); } + settled = true; + } + } + if (globalThis.__resolvers[cid]) { + globalThis.__resolvers[cid].reject(err); + delete globalThis.__resolvers[cid]; + settled = true; + } + return settled; + })()` + ) + ); + if (didSettle) { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); } markCreated(vm, escapedCid); break; @@ -1359,8 +1445,33 @@ async function processEvents( case 'step_created': case 'step_started': case 'step_retrying': - case 'wait_created': + case 'wait_created': { + markCreated(vm, escapedCid); + break; + } case 'hook_created': { + // Confirm creation for getConflict() awaiters: resolve them with + // null (no conflict) once the event log proves the hook exists. + const settledGetConflict = vm.dump( + vm.evalCode( + `(function(){ + var state = globalThis.__hooks && globalThis.__hooks[${JSON.stringify(cid)}]; + if (!state) return false; + state.created = true; + var gc = state.getConflictResolvers; + state.getConflictResolvers = []; + for (var i = 0; i < gc.length; i++) gc[i].resolve(null); + return gc.length > 0; + })()` + ) + ); + if (settledGetConflict) { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } markCreated(vm, escapedCid); break; } diff --git a/packages/core/src/serialization/reducers/common-vm.ts b/packages/core/src/serialization/reducers/common-vm.ts index da54470800..4af8bf4a3c 100644 --- a/packages/core/src/serialization/reducers/common-vm.ts +++ b/packages/core/src/serialization/reducers/common-vm.ts @@ -305,7 +305,21 @@ export function getCommonReducers(): Partial { ) return false; const name = value[Symbol.for('WORKFLOW_STREAM_NAME')]; - return { name: name || '__empty' }; + const s: { name: string; runId?: string; deploymentId?: string } = { + name: name || '__empty', + }; + // When the handle was forwarded from another run (parent -> child + // via start()), preserve the foreign runId/deploymentId so the + // step-side reviver opens the writable against the original stream. + // Mirrors the node:vm workflow reducer in serialization.ts. + const foreignRunId = value[Symbol.for('WORKFLOW_STREAM_SERVER_RUN_ID')]; + if (typeof foreignRunId === 'string') s.runId = foreignRunId; + const foreignDeploymentId = + value[Symbol.for('WORKFLOW_STREAM_SERVER_DEPLOYMENT_ID')]; + if (typeof foreignDeploymentId === 'string') { + s.deploymentId = foreignDeploymentId; + } + return s; }) as any, Set: (value) => value instanceof Set && Array.from(value), URL: (value) => value instanceof URL && value.href, @@ -536,6 +550,20 @@ export function getCommonRevivers(): Partial { if (value && 'name' in value) { stream[Symbol.for('WORKFLOW_STREAM_NAME')] = value.name; } + // Preserve the foreign runId/deploymentId, if present, so that when + // the handle is later passed to a step the workflow reducer can + // forward it through to the step reviver (cross-run writable + // forwarding via start()). + if (value && typeof (value as any).runId === 'string') { + stream[Symbol.for('WORKFLOW_STREAM_SERVER_RUN_ID')] = ( + value as any + ).runId; + } + if (value && typeof (value as any).deploymentId === 'string') { + stream[Symbol.for('WORKFLOW_STREAM_SERVER_DEPLOYMENT_ID')] = ( + value as any + ).deploymentId; + } return stream; }, }; From 9d2530e749a8a924be4b62ce5425ae4417a28305 Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 22 Jul 2026 01:21:04 -0700 Subject: [PATCH 04/18] QuickJS engine: stream framing round-trip, bound step proxies, webhook fidelity --- packages/core/src/runtime/quickjs-runtime.ts | 15 +++++++++++++++ .../core/src/serialization/reducers/common-vm.ts | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index e9ef30ce79..6300e48af8 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -284,6 +284,21 @@ globalThis[Symbol.for("WORKFLOW_USE_STEP")] = function(stepId, closureVarsFn) { // serialize step function references (e.g. when passed as arguments). fn.stepId = stepId; if (closureVarsFn) fn.__closureVarsFn = closureVarsFn; + // Override .bind so a bound step proxy (e.g. the SWC plugin's + // useStep(...).bind(this) for lexical-this arrow steps) keeps its + // stepId and records the bound receiver / prefilled args — the native + // bind drops own properties, which would make the StepFunction + // reducer fail to recognize the proxy when it crosses a serialization + // boundary. Mirrors the node:vm engine's override in step.ts. + fn.bind = function(thisArg) { + var partialArgs = Array.prototype.slice.call(arguments, 1); + var bound = Function.prototype.bind.apply(this, [thisArg].concat(partialArgs)); + bound.stepId = stepId; + if (closureVarsFn) bound.__closureVarsFn = closureVarsFn; + bound.__boundThis = thisArg; + if (partialArgs.length > 0) bound.__boundArgs = partialArgs; + return bound; + }; return fn; }; diff --git a/packages/core/src/serialization/reducers/common-vm.ts b/packages/core/src/serialization/reducers/common-vm.ts index 4af8bf4a3c..66cb458e56 100644 --- a/packages/core/src/serialization/reducers/common-vm.ts +++ b/packages/core/src/serialization/reducers/common-vm.ts @@ -292,6 +292,11 @@ export function getCommonReducers(): Partial { const s: any = { name }; const type = value[Symbol.for('WORKFLOW_STREAM_TYPE')]; if (type) s.type = type; + // Preserve wire framing so the step-side reviver can unframe + // byte streams (framed-v1) — dropping it turns a framed webhook + // body into raw length-prefixed bytes for the consumer. + const framing = value[Symbol.for('WORKFLOW_STREAM_FRAMING')]; + if (framing) s.framing = framing; return s; } return { name: '__empty' }; @@ -541,6 +546,9 @@ export function getCommonRevivers(): Partial { // but not consumed directly. stream[Symbol.for('WORKFLOW_STREAM_NAME')] = value.name; if (value.type) stream[Symbol.for('WORKFLOW_STREAM_TYPE')] = value.type; + if (value.framing) { + stream[Symbol.for('WORKFLOW_STREAM_FRAMING')] = value.framing; + } } return stream; }, From c57127d885ff630964cde871b3a522283c48414d Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 22 Jul 2026 01:41:11 -0700 Subject: [PATCH 05/18] Apply biome fixes to QuickJS engine files --- packages/core/src/runtime/quickjs-entrypoint.ts | 2 +- packages/core/src/serialization/codec-devalue-vm.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index a7827718a0..ea220fc03b 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -29,12 +29,12 @@ import { import { classifyRunError } from '../classify-error.js'; import { importKey } from '../encryption.js'; import { runtimeLogger } from '../logger.js'; +import { encrypt as encryptSerializedData } from '../serialization/encryption.js'; import { dehydrateRunError, hydrateRunError, maybeEncrypt, } from '../serialization.js'; -import { encrypt as encryptSerializedData } from '../serialization/encryption.js'; import { remapErrorStack, stripInlineSourceMap } from '../source-map.js'; import * as Attribute from '../telemetry/semantic-conventions.js'; import { serializeTraceCarrier } from '../telemetry.js'; diff --git a/packages/core/src/serialization/codec-devalue-vm.ts b/packages/core/src/serialization/codec-devalue-vm.ts index fafabb10c8..fd74954434 100644 --- a/packages/core/src/serialization/codec-devalue-vm.ts +++ b/packages/core/src/serialization/codec-devalue-vm.ts @@ -6,7 +6,6 @@ */ import { parse, stringify, unflatten } from 'devalue'; -import { SerializationFormat, type Reducers, type Revivers } from './types.js'; import type { Codec, SerializationMode } from './codec.js'; import { getClassReducers, getClassRevivers } from './reducers/class.js'; import { getCommonReducers, getCommonRevivers } from './reducers/common-vm.js'; @@ -14,6 +13,7 @@ import { getStepFunctionReducer, getStepFunctionReviver, } from './reducers/step-function.js'; +import { type Reducers, type Revivers, SerializationFormat } from './types.js'; const encoder = new TextEncoder(); const decoder = new TextDecoder(); From ff400af275eab33685847e967bbdd05f9539491c Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 22 Jul 2026 16:16:13 -0700 Subject: [PATCH 06/18] Address review feedback: anchor source-map strip to end-of-input, use getWorkflowQueueName for conflict requeue, Buffer-free asset decoding, function replacers for payload injection, maxEventsLimit guard --- packages/core/scripts/build-quickjs-assets.js | 20 ++++- .../core/scripts/build-vm-serde-bundle.js | 6 +- packages/core/src/runtime.ts | 3 +- .../core/src/runtime/quickjs-entrypoint.ts | 18 ++++- .../core/src/runtime/quickjs-runtime.test.ts | 80 +++++++++++++++++++ packages/core/src/runtime/quickjs-runtime.ts | 11 ++- packages/core/src/source-map.ts | 2 +- 7 files changed, 132 insertions(+), 8 deletions(-) diff --git a/packages/core/scripts/build-quickjs-assets.js b/packages/core/scripts/build-quickjs-assets.js index 807589fbb6..546e9158a4 100644 --- a/packages/core/scripts/build-quickjs-assets.js +++ b/packages/core/scripts/build-quickjs-assets.js @@ -43,6 +43,24 @@ let output = `/** */ import type { ExtensionDescriptor } from 'quickjs-wasi'; +/** + * Decode base64 without a hard dependency on Node's Buffer, so this + * module also loads on WASM-only platforms (e.g. Cloudflare Workers) + * where only atob() is available. Node's Buffer path is preferred when + * present — it is significantly faster for multi-hundred-KB payloads. + */ +function decodeBase64(b64: string): Uint8Array { + if (typeof Buffer !== 'undefined') { + return Buffer.from(b64, 'base64'); + } + const binary = atob(b64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +} + `; let totalSize = 0; @@ -51,7 +69,7 @@ for (const [name, filePath] of Object.entries(files)) { const buf = readFileSync(filePath); const b64 = buf.toString('base64'); totalSize += buf.length; - output += `const ${name} = Buffer.from('${b64}', 'base64');\n\n`; + output += `const ${name} = decodeBase64('${b64}');\n\n`; } output += `export { quickjsWasm };\n\n`; diff --git a/packages/core/scripts/build-vm-serde-bundle.js b/packages/core/scripts/build-vm-serde-bundle.js index f738853b9a..2574b542fb 100644 --- a/packages/core/scripts/build-vm-serde-bundle.js +++ b/packages/core/scripts/build-vm-serde-bundle.js @@ -51,8 +51,10 @@ writeFileSync( * Do not edit manually. * * This is the VM serialization bundle — a self-contained IIFE that sets up - * serialize/deserialize + TextEncoder/TextDecoder polyfills inside the - * QuickJS WASM VM. It includes devalue and all workflow-mode reducers. + * the serialize/deserialize functions inside the QuickJS WASM VM. It + * includes devalue and all workflow-mode reducers/revivers. (TextEncoder, + * TextDecoder, and Headers are provided by quickjs-wasi's native C + * extensions — no JS polyfills are bundled.) * * Size: ${(bundleCode.length / 1024).toFixed(1)} KB minified */ diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 98193167e6..355ae48be4 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -81,7 +81,6 @@ import { DEFAULT_STEP_MAX_RETRIES, executeStep, } from './runtime/step-executor.js'; -import { useQuickJSVm } from './runtime/vm-mode.js'; import { computeStepLatencyTracking } from './runtime/step-latency.js'; import { backstopIdempotencyKey, @@ -91,6 +90,7 @@ import { } from './runtime/step-ownership.js'; import { runStepSingleFlight } from './runtime/step-single-flight.js'; import { handleSuspension } from './runtime/suspension-handler.js'; +import { useQuickJSVm } from './runtime/vm-mode.js'; import { getWaitContinuationDispatch } from './runtime/wait-continuation.js'; import { getWorld, @@ -1635,6 +1635,7 @@ export function workflowEntrypoint( preloadedEvents, runInput, parentSpan: span, + maxEventsLimit, }); if (quickjsResult?.timeoutSeconds !== undefined) { // Use `reinvoke` rather than returning diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index ea220fc03b..c95a73e037 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -238,7 +238,7 @@ async function dispatchPendingOps(params: { if (result.event?.eventType === 'hook_conflict') { await queueMessage( world, - `__wkf_workflow_${workflowRun.workflowName}`, + getWorkflowQueueName(workflowRun.workflowName), { runId, }, @@ -408,6 +408,14 @@ export async function runWorkflowWithQuickJS(params: { * to it for end-to-end visibility. */ parentSpan?: Span; + /** + * Server-supplied per-run event ceiling from the run_started response + * (undefined ⇒ no enforcement). Mirrors the node:vm engine's guard: + * a runaway run is failed once its log reaches the ceiling. The throw + * propagates to the queue handler's catch, which records run_failed + * with MAX_EVENTS_EXCEEDED. + */ + maxEventsLimit?: number; }): Promise<{ timeoutSeconds?: number } | void> { const { workflowCode, @@ -416,6 +424,7 @@ export async function runWorkflowWithQuickJS(params: { preloadedEvents, runInput, parentSpan, + maxEventsLimit, } = params; const world = await getWorld(); const runId = workflowRun.runId; @@ -502,6 +511,13 @@ export async function runWorkflowWithQuickJS(params: { events = allEvents; } + // Event-limit guard: fail a runaway run once its log reaches the + // server-supplied ceiling — same enforcement point as the node:vm + // engine's replay loop. + if (maxEventsLimit !== undefined && events.length >= maxEventsLimit) { + throw new MaxEventsExceededError(events.length, maxEventsLimit); + } + parentSpan?.setAttributes({ ...Attribute.QuickJSEventsPreloaded(usePreloaded), ...Attribute.QuickJSEventsFetchedCount(events.length), diff --git a/packages/core/src/runtime/quickjs-runtime.test.ts b/packages/core/src/runtime/quickjs-runtime.test.ts index 3e5751a051..2f0ddc5696 100644 --- a/packages/core/src/runtime/quickjs-runtime.test.ts +++ b/packages/core/src/runtime/quickjs-runtime.test.ts @@ -721,3 +721,83 @@ describe('AbortController (hook-backed)', () => { expect(value.timeoutThrew).toBe(true); }); }); + +describe('hook payload buffering', () => { + it('buffers payloads containing String.replace special patterns verbatim', async () => { + // Regression: the buffered-payload path injects the JSON-serialized + // payload via String.replace('%PAYLOAD%', ...). With a string + // replacement, `$&`/`$'`/"$\`" sequences in the payload would be + // expanded as replacement patterns, corrupting the injected code. + const code = ` + var prime = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//prime"); + async function workflow() { + var hook = globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]({ token: "tok" }); + // Await a step first so the hook payload arrives with no resolver + // registered and takes the buffered path. + await prime(1); + var payload = await hook; + return payload; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + const run = makeRun(); + + const r1 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + const ops = r1.suspended!.pendingOperations; + const stepCid = ops.find((o) => o.type === 'step')!.correlationId; + const hookCid = ops.find((o) => o.type === 'hook')!.correlationId; + + const trickyPayload = { msg: "$& $' $` $1 $$", nested: { v: '$&' } }; + const r2 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [ + runCreatedEvent(run), + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'hook_created', + correlationId: hookCid, + eventData: { token: 'tok', isWebhook: false }, + createdAt: new Date('2025-01-01T00:00:01Z'), + }, + { + eventId: 'evnt_002', + runId: run.runId, + eventType: 'step_created', + correlationId: stepCid, + eventData: { stepName: 'step//test//prime' }, + createdAt: new Date('2025-01-01T00:00:02Z'), + }, + // The hook payload lands BEFORE the step completes, so no + // resolver exists yet and the payload is buffered in the VM heap. + { + eventId: 'evnt_003', + runId: run.runId, + eventType: 'hook_received', + correlationId: hookCid, + eventData: { payload: trickyPayload }, + createdAt: new Date('2025-01-01T00:00:03Z'), + }, + { + eventId: 'evnt_004', + runId: run.runId, + eventType: 'step_completed', + correlationId: stepCid, + eventData: { result: 1 }, + createdAt: new Date('2025-01-01T00:00:04Z'), + }, + ], + }); + + expect(r2.completed).toBeDefined(); + expect(unwrapResult(r2.completed!.result)).toEqual(trickyPayload); + }); +}); diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index 6300e48af8..40bd57ff97 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -1372,10 +1372,14 @@ async function processEvents( const bytesHandle = vm.newUint8Array(decryptedPayload); vm.setProp(vm.global, '__tmp_result', bytesHandle); bytesHandle.dispose(); + // NOTE: replacement is a function so `$`-sequences in the + // substituted JS never get interpreted as String.replace + // special replacement patterns. vm.evalCode( bufferAndTrack.replace( '%PAYLOAD%', - 'globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result)' + () => + 'globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result)' ) + 'delete globalThis.__tmp_result;' ).dispose(); } else { @@ -1383,8 +1387,11 @@ async function processEvents( rawPayload !== undefined ? JSON.stringify(rawPayload) : 'undefined'; + // Function replacement: a JSON-serialized payload can contain + // `$&`, `$'`, `$\``, ... which String.replace would otherwise + // expand, silently corrupting the injected code. vm.evalCode( - bufferAndTrack.replace('%PAYLOAD%', serialized) + bufferAndTrack.replace('%PAYLOAD%', () => serialized) ).dispose(); } } diff --git a/packages/core/src/source-map.ts b/packages/core/src/source-map.ts index 5773b421fc..aa8548b98c 100644 --- a/packages/core/src/source-map.ts +++ b/packages/core/src/source-map.ts @@ -9,7 +9,7 @@ import { originalPositionFor, TraceMap } from '@jridgewell/trace-mapping'; * lookups. */ const INLINE_SOURCE_MAP_COMMENT_RE = - /\/\/# sourceMappingURL=data:application\/json;base64,[A-Za-z0-9+/=]+\s*$/m; + /\/\/# sourceMappingURL=data:application\/json;base64,[A-Za-z0-9+/=]+\s*$/; /** * Strip the trailing `//# sourceMappingURL=data:…` comment from a JS From 5d4c5785f560162f6d94074b961df81fc797fd3f Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 22 Jul 2026 16:25:53 -0700 Subject: [PATCH 07/18] CI: include generated QuickJS source assets in shared e2e build artifacts --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 070f1e8abb..bde5730946 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -310,6 +310,8 @@ jobs: packages/*/dist packages/*/.well-known packages/*/src/version.ts + packages/core/src/runtime/vm-serde-bundle.generated.ts + packages/core/src/runtime/quickjs-assets.generated.ts packages/swc-plugin-workflow/swc_plugin_workflow.wasm packages/swc-plugin-workflow/build-hash.json include-hidden-files: true From 24017ccec7f317fc45b3d4dde3c77c6ee6ce02a5 Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 22 Jul 2026 17:19:42 -0700 Subject: [PATCH 08/18] Fix same-token hook ordering and conflicted-hook disposal in the QuickJS engine --- .../core/src/runtime/quickjs-entrypoint.ts | 333 ++++++++++-------- packages/core/src/runtime/quickjs-runtime.ts | 30 +- 2 files changed, 215 insertions(+), 148 deletions(-) diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index c95a73e037..900c883e0a 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -16,6 +16,8 @@ import type { Span } from '@opentelemetry/api'; import { EntityConflictError, + HookNotFoundError, + MaxEventsExceededError, RunExpiredError, WorkflowNotRegisteredError, } from '@workflow/errors'; @@ -43,6 +45,7 @@ import { getWorkflowQueueName, queueMessage } from './helpers.js'; import { type PendingAttribute, type PendingHook, + type PendingHookDispose, type PendingOperation, type PendingStep, type PendingWait, @@ -122,6 +125,194 @@ async function dispatchPendingOps(params: { let createdAttributeEvent = false; const opsPromises: Promise[] = []; + const processHookOp = async (hook: PendingHook): Promise => { + runtimeLogger.debug('QuickJS runtime: processing hook op', { + workflowRunId: runId, + correlationId: hook.correlationId, + token: hook.token, + tokenType: typeof hook.token, + isWebhook: hook.isWebhook, + isSystem: hook.isSystem, + hasCreatedEvent: hook.hasCreatedEvent, + abortRequested: hook.abortRequested, + }); + + if (!hook.hasCreatedEvent) { + // `hook.metadata` is the format-prefixed devalue bytes + // produced by `globalThis[Symbol.for('workflow-serialize')] + // (options.metadata)` inside the VM. Encrypt on the host + // side before writing — matches the node:vm engine's + // `dehydrateStepArguments` flow. + // + // No pre-check via hooks.list: with deterministic correlationIds + // (same VM seed across replays) and per-(runId, correlationId) + // uniqueness in worlds, the storage layer rejects duplicates as + // EntityConflictError, which we swallow below. This drops one + // network round-trip per pending hook. + try { + const encryptedMetadata = + typeof hook.metadata === 'undefined' + ? undefined + : await encryptSerializedData(hook.metadata, encryptionKey); + const result = await world.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: hook.correlationId, + eventData: { + token: hook.token, + metadata: encryptedMetadata, + // Always include isWebhook explicitly. Worlds default it to + // `true` when absent, which would break the public webhook + // endpoint's 404 guard for hooks created via createHook(). + isWebhook: hook.isWebhook, + // System hooks (AbortController) are exempt from user + // token namespace conflict checks. + ...(hook.isSystem ? { isSystem: true } : {}), + } as any, + }); + + // If storage detected a real token conflict with another + // workflow's hook, re-queue so the workflow handler can + // process the conflict event and fail gracefully. + if (result.event?.eventType === 'hook_conflict') { + await queueMessage( + world, + getWorkflowQueueName(workflowRun.workflowName), + { + runId, + }, + { idempotencyKey: `hook_conflict_${hook.correlationId}` } + ); + } + } catch (err) { + // Already created by a concurrent invocation — fall through + // to abort processing below (if any) instead of bailing. + if (!EntityConflictError.is(err)) throw err; + } + if (hook.hasGetConflictAwaiter) { + createdGetConflictHook = true; + } + } + + if (hook.abortRequested) { + // Record the abort durably: a hook_received event carrying + // the VM-serialized `{ aborted: true, reason }` payload, + // plus a best-effort stream packet for real-time step + // propagation. Mirrors the node:vm engine's suspension + // handler (hooksNeedingAbort). + const abortPayload = + hook.abortPayload instanceof Uint8Array + ? ((await encryptSerializedData( + hook.abortPayload, + encryptionKey + )) as Uint8Array) + : undefined; + try { + await world.events.create(runId, { + eventType: 'hook_received', + specVersion: SPEC_VERSION_CURRENT, + correlationId: hook.correlationId, + eventData: { + token: hook.token, + payload: abortPayload, + } as any, + }); + } catch (err) { + if (!EntityConflictError.is(err)) throw err; + } + // streamName is derived from the abort hook token + // (`abrt_{id}` → `strm_{id}_system_abort`). + if (hook.token.startsWith('abrt_') && abortPayload) { + const streamName = `strm_${hook.token.slice('abrt_'.length)}_system_abort`; + try { + await world.streams.write(runId, streamName, abortPayload); + await world.streams.close(runId, streamName); + } catch { + // Best-effort — the hook event provides the durable + // fallback. + runtimeLogger.debug( + 'QuickJS runtime: failed to write abort stream packet', + { + workflowRunId: runId, + correlationId: hook.correlationId, + } + ); + } + } + wfdiag('abort_recorded', { + correlationId: hook.correlationId, + token: hook.token, + }); + } + }; + + const processHookDisposeOp = async ( + op: PendingHookDispose + ): Promise => { + try { + await world.events.create(runId, { + eventType: 'hook_disposed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: op.correlationId, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + // Disposing a hook whose entity no longer (or never) exists is an + // idempotent no-op: the entity may have been torn down by a + // concurrent run cancellation, or the hook may have lost its + // token claim to a conflict. There is nothing left to release. + if (HookNotFoundError.is(err)) return; + throw err; + } + }; + + // Hook operations are grouped by token and processed SEQUENTIALLY in + // code order within each group, mirroring the node:vm suspension + // handler (hookItemsByToken): a dispose() of an earlier hook must + // release the token before a later same-token hook's creation is + // validated by the world — parallel dispatch would otherwise record a + // spurious hook_conflict against the run's own disposed hook (e.g. a + // dispose→recreate loop reusing one token). Different tokens have no + // claim interaction, so token groups run in parallel with each other + // and with the non-hook ops below. + const hookOpsByToken = new Map< + string, + (PendingHook | PendingHookDispose)[] + >(); + for (const op of pendingOperations) { + let key: string | undefined; + if ( + op.type === 'hook' && + (!op.hasCreatedEvent || (op as PendingHook).abortRequested) + ) { + key = (op as PendingHook).token; + } else if (op.type === 'hook_dispose' && !op.hasCreatedEvent) { + // Per-op fallback group when the token is unknown — no ordering + // guarantees, matching the previous parallel behavior. + key = (op as PendingHookDispose).token ?? `__cid:${op.correlationId}`; + } + if (key === undefined) continue; + const group = hookOpsByToken.get(key); + if (group) { + group.push(op as PendingHook | PendingHookDispose); + } else { + hookOpsByToken.set(key, [op as PendingHook | PendingHookDispose]); + } + } + for (const group of hookOpsByToken.values()) { + opsPromises.push( + (async () => { + for (const op of group) { + if (op.type === 'hook') { + await processHookOp(op); + } else { + await processHookDisposeOp(op); + } + } + })() + ); + } + for (const op of pendingOperations) { if (op.type === 'step' && !op.hasCreatedEvent) { const step = op as PendingStep; @@ -180,133 +371,6 @@ async function dispatchPendingOps(params: { } })() ); - } else if ( - op.type === 'hook' && - (!op.hasCreatedEvent || (op as PendingHook).abortRequested) - ) { - const hook = op as PendingHook; - runtimeLogger.debug('QuickJS runtime: processing hook op', { - workflowRunId: runId, - correlationId: hook.correlationId, - token: hook.token, - tokenType: typeof hook.token, - isWebhook: hook.isWebhook, - isSystem: hook.isSystem, - hasCreatedEvent: hook.hasCreatedEvent, - abortRequested: hook.abortRequested, - }); - - opsPromises.push( - (async () => { - if (!hook.hasCreatedEvent) { - // `hook.metadata` is the format-prefixed devalue bytes - // produced by `globalThis[Symbol.for('workflow-serialize')] - // (options.metadata)` inside the VM. Encrypt on the host - // side before writing — matches the node:vm engine's - // `dehydrateStepArguments` flow. - // - // No pre-check via hooks.list: with deterministic correlationIds - // (same VM seed across replays) and per-(runId, correlationId) - // uniqueness in worlds, the storage layer rejects duplicates as - // EntityConflictError, which we swallow below. This drops one - // network round-trip per pending hook. - try { - const encryptedMetadata = - typeof hook.metadata === 'undefined' - ? undefined - : await encryptSerializedData(hook.metadata, encryptionKey); - const result = await world.events.create(runId, { - eventType: 'hook_created', - specVersion: SPEC_VERSION_CURRENT, - correlationId: hook.correlationId, - eventData: { - token: hook.token, - metadata: encryptedMetadata, - // Always include isWebhook explicitly. Worlds default it to - // `true` when absent, which would break the public webhook - // endpoint's 404 guard for hooks created via createHook(). - isWebhook: hook.isWebhook, - // System hooks (AbortController) are exempt from user - // token namespace conflict checks. - ...(hook.isSystem ? { isSystem: true } : {}), - } as any, - }); - - // If storage detected a real token conflict with another - // workflow's hook, re-queue so the workflow handler can - // process the conflict event and fail gracefully. - if (result.event?.eventType === 'hook_conflict') { - await queueMessage( - world, - getWorkflowQueueName(workflowRun.workflowName), - { - runId, - }, - { idempotencyKey: `hook_conflict_${hook.correlationId}` } - ); - } - } catch (err) { - // Already created by a concurrent invocation — fall through - // to abort processing below (if any) instead of bailing. - if (!EntityConflictError.is(err)) throw err; - } - if (hook.hasGetConflictAwaiter) { - createdGetConflictHook = true; - } - } - - if (hook.abortRequested) { - // Record the abort durably: a hook_received event carrying - // the VM-serialized `{ aborted: true, reason }` payload, - // plus a best-effort stream packet for real-time step - // propagation. Mirrors the node:vm engine's suspension - // handler (hooksNeedingAbort). - const abortPayload = - hook.abortPayload instanceof Uint8Array - ? ((await encryptSerializedData( - hook.abortPayload, - encryptionKey - )) as Uint8Array) - : undefined; - try { - await world.events.create(runId, { - eventType: 'hook_received', - specVersion: SPEC_VERSION_CURRENT, - correlationId: hook.correlationId, - eventData: { - token: hook.token, - payload: abortPayload, - } as any, - }); - } catch (err) { - if (!EntityConflictError.is(err)) throw err; - } - // streamName is derived from the abort hook token - // (`abrt_{id}` → `strm_{id}_system_abort`). - if (hook.token.startsWith('abrt_') && abortPayload) { - const streamName = `strm_${hook.token.slice('abrt_'.length)}_system_abort`; - try { - await world.streams.write(runId, streamName, abortPayload); - await world.streams.close(runId, streamName); - } catch { - // Best-effort — the hook event provides the durable - // fallback. - runtimeLogger.debug( - 'QuickJS runtime: failed to write abort stream packet', - { - workflowRunId: runId, - correlationId: hook.correlationId, - } - ); - } - } - wfdiag('abort_recorded', { - correlationId: hook.correlationId, - token: hook.token, - }); - } - })() - ); } else if (op.type === 'attribute' && !op.hasCreatedEvent) { const attr = op as PendingAttribute; opsPromises.push( @@ -336,21 +400,6 @@ async function dispatchPendingOps(params: { } })() ); - } else if (op.type === 'hook_dispose' && !op.hasCreatedEvent) { - opsPromises.push( - (async () => { - try { - await world.events.create(runId, { - eventType: 'hook_disposed', - specVersion: SPEC_VERSION_CURRENT, - correlationId: op.correlationId, - }); - } catch (err) { - if (EntityConflictError.is(err)) return; - throw err; - } - })() - ); } else if (op.type === 'wait' && !op.hasCreatedEvent) { const wait = op as PendingWait; opsPromises.push( diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index 40bd57ff97..edbabb76d3 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -121,6 +121,11 @@ export interface PendingAttribute { export interface PendingHookDispose { type: 'hook_dispose'; correlationId: string; + /** + * Token of the hook being disposed. Used by the entrypoint to order + * same-token hook operations sequentially in code order. + */ + token?: string; hasCreatedEvent: boolean; } @@ -497,12 +502,25 @@ globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")] = function(options) { function disposeHook() { if (isDisposed) return; isDisposed = true; - // Signal to the entrypoint to create a hook_disposed event - globalThis.__pending.push({ - type: "hook_dispose", - correlationId: correlationId, - hasCreatedEvent: false, - }); + // A conflicted hook was never created (the world rejected its claim + // — the token belongs to another run), so there is no entity to + // dispose. Mirrors the node:vm engine, where hook_conflict removes + // the invocation-queue item before dispose can mark it. Emitting a + // hook_disposed here would be rejected by the world's + // hook-existence validation. + var state = globalThis.__hooks[correlationId]; + if (!state || !state.conflict) { + // Signal to the entrypoint to create a hook_disposed event. The + // token is carried so the entrypoint can order same-token hook + // operations sequentially (a dispose must release the token before + // a later same-token hook's creation is validated). + globalThis.__pending.push({ + type: "hook_dispose", + correlationId: correlationId, + token: token, + hasCreatedEvent: false, + }); + } // If there's a pending resolver, resolve it with undefined to break the iterator if (globalThis.__resolvers[correlationId]) { globalThis.__resolvers[correlationId].resolve(undefined); From ad80ba87cb3f6976b3fed8da5acc25fbdc363790 Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 22 Jul 2026 17:54:27 -0700 Subject: [PATCH 09/18] CI: run both VM engines across all frameworks and worlds; label jobs with the engine --- .github/workflows/tests.yml | 45 +++++++++++++++++++++++++--------- scripts/create-test-matrix.mjs | 29 +++++++++++----------- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bde5730946..c191c30a74 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -318,7 +318,7 @@ jobs: retention-days: 1 e2e-vercel-prod: - name: E2E Vercel Prod Tests (${{ matrix.app.name }}) + name: E2E Vercel Prod Tests (${{ matrix.app.name }} - ${{ matrix.vm }}) runs-on: ubuntu-latest timeout-minutes: 30 needs: ci-scope @@ -331,6 +331,12 @@ jobs: strategy: fail-fast: false matrix: + # Workflow VM engines: node:vm (default) and the opt-in QuickJS + # WASM engine. The env var is set on the e2e test runner, which is + # the client that starts runs against the deployed app — start() + # stamps executionContext.workflowVm so the deployed handler + # executes each run on the requested engine. + vm: [node, quickjs] app: - name: "example" project-id: "prj_xWq20Dd860HHAfzMjK2Mb6TPVxMa" @@ -431,12 +437,13 @@ jobs: run: echo "ms=$(($(date +%s) * 1000))" >> "$GITHUB_OUTPUT" - name: Run E2E Tests - run: pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-prod-$APP_NAME.json" + run: pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-prod-$APP_NAME-$WORKFLOW_VM.json" env: NODE_OPTIONS: "--enable-source-maps" DEPLOYMENT_URL: ${{ steps.waitForDeployment.outputs.deployment-url || steps.prodDeployment.outputs.deployment-url }} VERCEL_DEPLOYMENT_ID: ${{ steps.waitForDeployment.outputs.deployment-id || steps.prodDeployment.outputs.deployment-id }} APP_NAME: ${{ matrix.app.name }} + WORKFLOW_VM: ${{ matrix.vm }} # changeset-release PRs test main's production deployment, so they # must be treated as a production run everywhere downstream. WORKFLOW_VERCEL_ENV: ${{ (github.ref == 'refs/heads/main' || startsWith(github.head_ref, 'changeset-release/')) && 'production' || 'preview' }} @@ -475,15 +482,16 @@ jobs: if: always() env: APP_NAME: ${{ matrix.app.name }} - run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Vercel Prod ($APP_NAME)" >> $GITHUB_STEP_SUMMARY || true + WORKFLOW_VM: ${{ matrix.vm }} + run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Vercel Prod ($APP_NAME - $WORKFLOW_VM)" >> $GITHUB_STEP_SUMMARY || true - name: Upload E2E results if: always() uses: actions/upload-artifact@v4 with: - name: e2e-results-vercel-prod-${{ matrix.app.name }} + name: e2e-results-vercel-prod-${{ matrix.app.name }}-${{ matrix.vm }} path: | - e2e-vercel-prod-${{ matrix.app.name }}.json + e2e-vercel-prod-${{ matrix.app.name }}-${{ matrix.vm }}.json e2e-metadata-${{ matrix.app.name }}-vercel.json e2e-failures-${{ matrix.app.name }}-vercel.json e2e-diagnostics-${{ matrix.app.name }}-vercel.json @@ -906,11 +914,17 @@ jobs: if-no-files-found: ignore e2e-windows: - name: E2E Windows Tests + name: E2E Windows Tests (${{ matrix.vm }}) runs-on: windows-latest timeout-minutes: 30 needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} + strategy: + fail-fast: false + matrix: + # Workflow VM engines: node:vm (default) and the opt-in QuickJS + # WASM engine (WORKFLOW_VM=quickjs). + vm: [node, quickjs] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} @@ -948,7 +962,10 @@ jobs: cd workbench/nextjs-turbopack $logFile = "$env:GITHUB_WORKSPACE/nextjs-server.log" $env:DEV_SERVER_LOG_PATH = $logFile - $job = Start-Job -ScriptBlock { Set-Location $using:PWD; pnpm dev *>&1 | Tee-Object -FilePath $using:logFile } + # `$using:` only resolves PowerShell variables, not env vars, so + # copy MATRIX_VM into a session variable before Start-Job. + $matrixVm = $env:MATRIX_VM + $job = Start-Job -ScriptBlock { Set-Location $using:PWD; $env:WORKFLOW_VM = $using:matrixVm; pnpm dev *>&1 | Tee-Object -FilePath $using:logFile } Start-Sleep -Seconds 15 cd ../.. @@ -1002,7 +1019,7 @@ jobs: exit 1 } - pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-windows-nextjs-turbopack.json + pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-windows-nextjs-turbopack-$env:MATRIX_VM.json $e2eExit = $LASTEXITCODE Stop-Job $job -ErrorAction SilentlyContinue exit $e2eExit @@ -1018,6 +1035,8 @@ jobs: DEV_TEST_CONFIG: '{"generatedStepRegistrationPath":"app/.well-known/workflow/v1/flow/__step_registrations.js","generatedWorkflowPath":"app/.well-known/workflow/v1/flow/route.js","apiFilePath":"app/api/chat/route.ts","apiFileImportPath":"../../..","port":3000,"testWorkflowFile":"96_many_steps.ts"}' DEV_SERVER_LOG_PATH: "${{ github.workspace }}/nextjs-server.log" WORKFLOW_DEV_HMR_LOGS: "1" + WORKFLOW_VM: ${{ matrix.vm }} + MATRIX_VM: ${{ matrix.vm }} - name: Print Next.js server logs if: always() @@ -1035,14 +1054,16 @@ jobs: - name: Generate E2E summary if: always() shell: bash - run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Windows (nextjs-turbopack)" >> $GITHUB_STEP_SUMMARY || true + env: + MATRIX_VM: ${{ matrix.vm }} + run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Windows (nextjs-turbopack - $MATRIX_VM)" >> $GITHUB_STEP_SUMMARY || true - name: Upload E2E results if: always() uses: actions/upload-artifact@v4 with: - name: e2e-results-windows-nextjs-turbopack - path: e2e-windows-nextjs-turbopack.json + name: e2e-results-windows-nextjs-turbopack-${{ matrix.vm }} + path: e2e-windows-nextjs-turbopack-${{ matrix.vm }}.json retention-days: 7 if-no-files-found: ignore @@ -1050,7 +1071,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: nextjs-server-logs-windows + name: nextjs-server-logs-windows-${{ matrix.vm }} path: nextjs-server.log retention-days: 7 if-no-files-found: ignore diff --git a/scripts/create-test-matrix.mjs b/scripts/create-test-matrix.mjs index 284a27113e..3d056d912b 100644 --- a/scripts/create-test-matrix.mjs +++ b/scripts/create-test-matrix.mjs @@ -164,21 +164,20 @@ matrix.app.push({ ...DEV_TEST_CONFIGS['tanstack-start'], }); -// QuickJS WASM VM engine leg (opt-in via WORKFLOW_VM=quickjs). One app is -// enough while the engine is experimental — nextjs-turbopack is the most -// feature-complete workbench. The `vm` field is surfaced to the workflow -// dev server via the WORKFLOW_VM env var in tests.yml. -matrix.app.push( - createMatrixEntry( - 'nextjs-turbopack', - 'example-nextjs-workflow-turbopack', - DEV_TEST_CONFIGS['nextjs-turbopack'], - { - vm: 'quickjs', - runLabel: 'quickjs', - artifactSuffix: 'quickjs', - } - ) +// Cross-product with the workflow VM engine axis: every app is tested +// against both the default node:vm engine and the opt-in QuickJS WASM +// engine (WORKFLOW_VM=quickjs). Each engine gets its own artifactSuffix +// and runLabel so CI artifacts and job names are unique. The `vm` field +// is surfaced to the workflow dev server via the WORKFLOW_VM env var in +// tests.yml. +const VMS = ['node', 'quickjs']; +matrix.app = matrix.app.flatMap((app) => + VMS.map((vm) => ({ + ...app, + vm, + runLabel: [app.runLabel, vm].filter(Boolean).join(' '), + artifactSuffix: [app.artifactSuffix, vm].filter(Boolean).join('-'), + })) ); console.log(JSON.stringify(matrix)); From 2bd67b92c51fa68c6adc9a721a72993d81f9170e Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Thu, 23 Jul 2026 01:09:05 -0700 Subject: [PATCH 10/18] Fix stack overflow stripping inline source maps from webpack dev bundles; harden step-listing e2e assertions against eventually-consistent reads --- packages/core/e2e/e2e.test.ts | 60 +++++++++++++++++++++++----- packages/core/e2e/utils.ts | 36 +++++++++++++++++ packages/core/src/source-map.test.ts | 29 ++++++++++++++ packages/core/src/source-map.ts | 44 ++++++++++++++------ 4 files changed, 147 insertions(+), 22 deletions(-) diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index 53afa6de25..7412c9fcec 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -37,6 +37,7 @@ import { cliCancel, cliHealthJson, cliInspectJson, + cliInspectJsonUntil, fetchManifest, getCollectedRunIds, getWorkflowMetadata, @@ -1394,8 +1395,14 @@ describe('e2e', () => { expect(result.finalAttempt).toBe(3); - const { json: steps } = await cliInspectJson( - `steps --runId ${run.runId}` + const steps = await cliInspectJsonUntil( + `steps --runId ${run.runId}`, + (json) => + json.some( + (s: any) => + s.stepName.includes('retryUntilAttempt3') && + s.status === 'completed' + ) ); const step = steps.find((s: any) => s.stepName.includes('retryUntilAttempt3') @@ -1420,8 +1427,14 @@ describe('e2e', () => { // (which inspect the value inside the SWC-instrumented workflow). // Here we only assert step lifecycle behavior. - const { json: steps } = await cliInspectJson( - `steps --runId ${run.runId}` + const steps = await cliInspectJsonUntil( + `steps --runId ${run.runId}`, + (json) => + json.some( + (s: any) => + s.stepName.includes('throwFatalError') && + s.status === 'failed' + ) ); const step = steps.find((s: any) => s.stepName.includes('throwFatalError') @@ -1654,8 +1667,14 @@ describe('e2e', () => { expect(runData.status).toBe('completed'); // Verify the step itself failed - const { json: steps } = await cliInspectJson( - `steps --runId ${run.runId}` + const steps = await cliInspectJsonUntil( + `steps --runId ${run.runId}`, + (json) => + json.some( + (s: any) => + s.stepName.includes('nonExistentStep') && + s.status === 'failed' + ) ); const ghostStep = steps.find((s: any) => s.stepName.includes('nonExistentStep') @@ -2377,8 +2396,11 @@ describe('e2e', () => { // Verify that exactly 2 steps were executed: // 1. stepWithStepFunctionArg(doubleNumber) // (doubleNumber(10) is run inside the stepWithStepFunctionArg step) - const { json: eventsData } = await cliInspectJson( - `events --run ${run.runId} --json` + const eventsData = await cliInspectJsonUntil( + `events --run ${run.runId} --json`, + (json) => + json.filter((event: any) => event.eventType === 'step_completed') + .length >= 1 ); const stepCompletedEvents = eventsData.filter( (event) => event.eventType === 'step_completed' @@ -2839,8 +2861,26 @@ describe('e2e', () => { // - 2 lexical-`this` arrow steps from `makeAdder` (direct + via-step) // - 1 invokeAdderFromStep wrapper (which itself triggers another // makeAdder arrow step inside it) - const { json: steps } = await cliInspectJson( - `steps --runId ${run.runId}` + const steps = await cliInspectJsonUntil( + `steps --runId ${run.runId}`, + (json) => { + const byName = (needle: string) => + json.filter((s: any) => s.stepName.includes(needle)); + const counter = json.filter( + (s: any) => + s.stepName.includes('Counter#add') || + s.stepName.includes('Counter#multiply') || + s.stepName.includes('Counter#describe') + ); + return ( + counter.length === 4 && + counter.every((s: any) => s.status === 'completed') && + byName('_anonymousStep').length === 1 && + byName('_anonymousStep')[0].status === 'completed' && + byName('invokeAdderFromStep').length === 1 && + byName('invokeAdderFromStep')[0].status === 'completed' + ); + } ); // Filter to only Counter instance method steps const counterSteps = steps.filter( diff --git a/packages/core/e2e/utils.ts b/packages/core/e2e/utils.ts index 8c57ebed70..910dd1f94b 100644 --- a/packages/core/e2e/utils.ts +++ b/packages/core/e2e/utils.ts @@ -840,3 +840,39 @@ export const cliHealthJson = async (options?: { timeout?: number }) => { throw err; } }; + +/** + * Poll `cliInspectJson(args)` until `predicate(json)` holds, or the timeout + * elapses — in which case the LAST result is returned so the caller's + * assertions still run and produce a real failure message. + * + * Needed for step/event listing assertions made right after a run settles: + * on the vercel world these listings are served analytics-first from an + * eventually-consistent store, so rows for just-finished steps can be + * missing or carry stale pending/running statuses for a few seconds + * before converging on the durable state. + */ +export const cliInspectJsonUntil = async ( + args: string, + predicate: (json: any) => boolean, + { + timeoutMs = 30_000, + intervalMs = 2_000, + }: { timeoutMs?: number; intervalMs?: number } = {} +): Promise => { + const deadline = Date.now() + timeoutMs; + // biome-ignore lint/suspicious/noExplicitAny: raw CLI JSON + let json: any; + for (;;) { + ({ json } = await cliInspectJson(args)); + let satisfied = false; + try { + satisfied = predicate(json); + } catch { + // Malformed intermediate state (e.g. `.find()` returned undefined) + // counts as not-yet-converged. + } + if (satisfied || Date.now() >= deadline) return json; + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } +}; diff --git a/packages/core/src/source-map.test.ts b/packages/core/src/source-map.test.ts index 5c7e12361a..c2c66a9c4c 100644 --- a/packages/core/src/source-map.test.ts +++ b/packages/core/src/source-map.test.ts @@ -144,3 +144,32 @@ console.log(literal); expect(stripped).not.toMatch(/\/\/# sourceMappingURL/); }); }); + +describe('stripInlineSourceMap on webpack-dev-shaped bundles', () => { + it('handles huge bundles with many embedded per-module inline maps', () => { + // Webpack dev-server bundles embed one inline source map comment per + // module inside eval strings — hundreds of non-trailing occurrences + // across tens of MB. The previous regex implementation blew V8's + // call stack on such inputs ("Maximum call stack size exceeded"), + // wedging every QuickJS workflow invocation on webpack dev. + let code = ''; + for (let i = 0; i < 100; i++) { + code += `eval("var m${i} = 1;\\n//# sourceMappingURL=data:application/json;base64,${'A'.repeat(256 * 1024)}\\n");\n`; + } + const trailingPayload = 'B'.repeat(1024 * 1024); + code += `//# sourceMappingURL=data:application/json;base64,${trailingPayload}\n`; + + const stripped = stripInlineSourceMap(code); + // Only the trailing comment is stripped; the embedded ones stay. + expect(stripped).not.toContain(trailingPayload); + expect(stripped).toContain('m99'); + expect(stripped.length).toBeLessThan(code.length); + expect(stripped.match(/sourceMappingURL/g)?.length ?? 0).toBe(100); + }); + + it('leaves a non-trailing last occurrence untouched', () => { + const code = + 'a;\n//# sourceMappingURL=data:application/json;base64,Zm9v\nconst tail = 1;\n'; + expect(stripInlineSourceMap(code)).toBe(code); + }); +}); diff --git a/packages/core/src/source-map.ts b/packages/core/src/source-map.ts index aa8548b98c..4167996104 100644 --- a/packages/core/src/source-map.ts +++ b/packages/core/src/source-map.ts @@ -1,19 +1,13 @@ import { originalPositionFor, TraceMap } from '@jridgewell/trace-mapping'; -/** - * Pattern matching the trailing inline source map comment that bundlers - * (esbuild, etc.) emit. The comment is purely host-side metadata for - * `remapErrorStack` — the VM never needs it. Stripping it before - * passing the bundle to `vm.evalCode` materially reduces the QuickJS - * heap, because QuickJS retains source text for stack-trace line - * lookups. - */ -const INLINE_SOURCE_MAP_COMMENT_RE = - /\/\/# sourceMappingURL=data:application\/json;base64,[A-Za-z0-9+/=]+\s*$/; +/** Marker prefix of an inline source map comment emitted by bundlers. */ +const INLINE_SOURCE_MAP_MARKER = + '//# sourceMappingURL=data:application/json;base64,'; /** * Strip the trailing `//# sourceMappingURL=data:…` comment from a JS - * bundle. Returns the input unchanged if no inline map is present. + * bundle. Returns the input unchanged if no trailing inline map is + * present. * * Use this on the host side before evaluating workflow bundles inside * the QuickJS VM — the inline map can account for several MB of bundle @@ -21,9 +15,35 @@ const INLINE_SOURCE_MAP_COMMENT_RE = * bundle), and the VM never needs it; only host-side `remapErrorStack` * reads the map (and it can do so against the original, unstripped * string). + * + * Implemented as a linear `lastIndexOf` + character scan rather than a + * regex: on webpack dev-server bundles (tens of MB, with hundreds of + * per-module inline map comments embedded in eval strings) a + * `String.replace` regex over the bundle blows V8's call stack + * ("RangeError: Maximum call stack size exceeded"), wedging every + * workflow invocation on that framework. */ export function stripInlineSourceMap(workflowCode: string): string { - return workflowCode.replace(INLINE_SOURCE_MAP_COMMENT_RE, ''); + const idx = workflowCode.lastIndexOf(INLINE_SOURCE_MAP_MARKER); + if (idx === -1) return workflowCode; + // Only strip when the comment is the TRAILING content: everything + // after the marker must be base64 payload followed by optional + // whitespace. A mid-bundle occurrence (e.g. inside a string literal) + // is left untouched. + let i = idx + INLINE_SOURCE_MAP_MARKER.length; + const payloadStart = i; + const n = workflowCode.length; + while (i < n && isBase64Char(workflowCode.charCodeAt(i))) i++; + if (i === payloadStart) return workflowCode; + while (i < n) { + const c = workflowCode.charCodeAt(i); + // space, tab, newline, carriage return + if (c !== 0x20 && c !== 0x09 && c !== 0x0a && c !== 0x0d) { + return workflowCode; + } + i++; + } + return workflowCode.slice(0, idx); } function isBase64Char(code: number): boolean { From 2a082ddafebfa7e5f1caaca8be55d423f2172531 Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Thu, 23 Jul 2026 02:11:15 -0700 Subject: [PATCH 11/18] e2e: poll step listings until analytics rows include attempt (optional column can lag terminal status) --- packages/core/e2e/e2e.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index 7412c9fcec..ad212099c8 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -1395,13 +1395,18 @@ describe('e2e', () => { expect(result.finalAttempt).toBe(3); + // Poll on attempt too: the analytics-backed listing can serve the + // terminal status before the attempt column is ingested (attempt is + // optional in the analytics schema), so a completed row may briefly + // report attempt as undefined. const steps = await cliInspectJsonUntil( `steps --runId ${run.runId}`, (json) => json.some( (s: any) => s.stepName.includes('retryUntilAttempt3') && - s.status === 'completed' + s.status === 'completed' && + s.attempt === 3 ) ); const step = steps.find((s: any) => @@ -1427,13 +1432,15 @@ describe('e2e', () => { // (which inspect the value inside the SWC-instrumented workflow). // Here we only assert step lifecycle behavior. + // Poll on attempt too — see the retry-success test above. const steps = await cliInspectJsonUntil( `steps --runId ${run.runId}`, (json) => json.some( (s: any) => s.stepName.includes('throwFatalError') && - s.status === 'failed' + s.status === 'failed' && + s.attempt === 1 ) ); const step = steps.find((s: any) => From 0518e1ce1409cd89b0a8e453518dd995de2c21c6 Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Thu, 23 Jul 2026 12:24:32 -0700 Subject: [PATCH 12/18] e2e: use --withData to force storage-backed step listings for attempt assertions (analytics listing can omit attempt entirely) --- packages/core/e2e/e2e.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index ad212099c8..94df5ac1cd 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -1395,12 +1395,12 @@ describe('e2e', () => { expect(result.finalAttempt).toBe(3); - // Poll on attempt too: the analytics-backed listing can serve the - // terminal status before the attempt column is ingested (attempt is - // optional in the analytics schema), so a completed row may briefly - // report attempt as undefined. + // --withData forces the storage-backed listing: the analytics + // listing may omit the attempt column entirely (it is optional in + // the analytics schema), so only the durable step entity can be + // asserted on. Poll because rows for a just-finished run can lag. const steps = await cliInspectJsonUntil( - `steps --runId ${run.runId}`, + `steps --runId ${run.runId} --withData`, (json) => json.some( (s: any) => @@ -1432,9 +1432,10 @@ describe('e2e', () => { // (which inspect the value inside the SWC-instrumented workflow). // Here we only assert step lifecycle behavior. - // Poll on attempt too — see the retry-success test above. + // --withData forces the storage-backed listing — see the + // retry-success test above. const steps = await cliInspectJsonUntil( - `steps --runId ${run.runId}`, + `steps --runId ${run.runId} --withData`, (json) => json.some( (s: any) => From 6336b476d2baf69fcbac6acc513260e75eec9030 Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Thu, 30 Jul 2026 17:51:04 -0700 Subject: [PATCH 13/18] Sort imports in QuickJS serialization files (biome organizeImports) --- packages/core/src/serialization/compat.test.ts | 18 +++++++++--------- packages/core/src/serialization/workflow-vm.ts | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/core/src/serialization/compat.test.ts b/packages/core/src/serialization/compat.test.ts index b8e2580d1c..f012955c90 100644 --- a/packages/core/src/serialization/compat.test.ts +++ b/packages/core/src/serialization/compat.test.ts @@ -6,21 +6,21 @@ * during the migration period. */ -import { describe, it, expect } from 'vitest'; -import * as workflow from './workflow.js'; -import * as step from './step.js'; -import * as client from './client.js'; +import { describe, expect, it } from 'vitest'; +import { importKey } from '../encryption.js'; import { + dehydrateStepArguments, + dehydrateStepReturnValue, dehydrateWorkflowArguments, - hydrateWorkflowArguments, dehydrateWorkflowReturnValue, - hydrateWorkflowReturnValue, - dehydrateStepArguments, hydrateStepArguments, - dehydrateStepReturnValue, hydrateStepReturnValue, + hydrateWorkflowArguments, + hydrateWorkflowReturnValue, } from '../serialization.js'; -import { importKey } from '../encryption.js'; +import * as client from './client.js'; +import * as step from './step.js'; +import * as workflow from './workflow.js'; const testData = { primitives: [42, 'hello', true, null], diff --git a/packages/core/src/serialization/workflow-vm.ts b/packages/core/src/serialization/workflow-vm.ts index 61fc9d14a4..3aaf5bc730 100644 --- a/packages/core/src/serialization/workflow-vm.ts +++ b/packages/core/src/serialization/workflow-vm.ts @@ -9,7 +9,7 @@ */ import { devalueVmCodec } from './codec-devalue-vm.js'; -import { SerializationFormat, isFormatPrefix } from './types.js'; +import { isFormatPrefix, SerializationFormat } from './types.js'; const FORMAT_PREFIX_LENGTH = 4; let _encoder: { encode(s: string): Uint8Array }; From 2f3434d310bc530d61622fda0904f4fec74e712a Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Thu, 30 Jul 2026 19:11:24 -0700 Subject: [PATCH 14/18] QuickJS engine: resolve the run's full payload-key capability so sealed (encp) hook payloads open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main's sealed-box work (#3096) makes cross-deployment resumeHook() seal hook payloads to the target run's published X25519 public key. The shared start() path publishes that key regardless of engine, so QuickJS runs receive sealed payloads too — but the QuickJS entrypoint resolved only the bare symmetric key via importKey(), which cannot open encp envelopes. The first sealed hook payload wedged the run right after hook_received, timing out every hook/webhook e2e on Vercel prod (node:vm legs were fine — the node engine resolves the full capability via memoizeEncryptionKey). Resolve deriveRunPayloadKeys() in the entrypoint instead and widen the runtime's key types from CryptoKey to DecryptionKey. Writes stay symmetric (encrypt() with RunPayloadKeys takes the encr path). Regression test seals a payload exactly as resumeHook does and round-trips it through the VM. --- .../core/src/runtime/quickjs-entrypoint.ts | 19 +++- .../core/src/runtime/quickjs-runtime.test.ts | 88 +++++++++++++++++++ packages/core/src/runtime/quickjs-runtime.ts | 13 +-- 3 files changed, 111 insertions(+), 9 deletions(-) diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index 900c883e0a..c7e2924cab 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -29,9 +29,12 @@ import { type WorkflowRun, } from '@workflow/world'; import { classifyRunError } from '../classify-error.js'; -import { importKey } from '../encryption.js'; import { runtimeLogger } from '../logger.js'; -import { encrypt as encryptSerializedData } from '../serialization/encryption.js'; +import { + deriveRunPayloadKeys, + encrypt as encryptSerializedData, + type RunPayloadKeys, +} from '../serialization/encryption.js'; import { dehydrateRunError, hydrateRunError, @@ -103,7 +106,7 @@ async function dispatchPendingOps(params: { world: Awaited>; runId: string; workflowRun: WorkflowRun; - encryptionKey: Awaited> | undefined; + encryptionKey: RunPayloadKeys | undefined; pendingOperations: PendingOperation[]; queueSteps: boolean; wfdiag: (checkpoint: string, fields: Record) => void; @@ -521,8 +524,16 @@ export async function runWorkflowWithQuickJS(params: { // Resolve the encryption key up front — needed to decrypt event // payloads inside the VM and to encrypt event payloads written below. + // Resolve the FULL capability (symmetric AES key + X25519 keypair), not + // just `importKey(rawKey)`: a run reading its own event log can encounter + // sealed (`encp`) hook payloads that a cross-deployment `resumeHook()` + // wrote to it (sealing is presence-gated on the run's published + // encryptionPublicKey, which the shared start() path stamps regardless of + // engine). A bare symmetric key cannot open those and would wedge the run + // right after hook_received — the node:vm engine resolves the same full + // capability via memoizeEncryptionKey. const rawKey = await world.getEncryptionKeyForRun?.(workflowRun); - const encryptionKey = rawKey ? await importKey(rawKey) : undefined; + const encryptionKey = rawKey ? await deriveRunPayloadKeys(rawKey) : undefined; // Load the FULL event log for the run. On first invocation the // preloaded events from the run_started response are the complete log diff --git a/packages/core/src/runtime/quickjs-runtime.test.ts b/packages/core/src/runtime/quickjs-runtime.test.ts index 2f0ddc5696..dd3e41f66a 100644 --- a/packages/core/src/runtime/quickjs-runtime.test.ts +++ b/packages/core/src/runtime/quickjs-runtime.test.ts @@ -801,3 +801,91 @@ describe('hook payload buffering', () => { expect(unwrapResult(r2.completed!.result)).toEqual(trickyPayload); }); }); + +describe('sealed (encp) hook payloads', () => { + it('opens a payload sealed to the run public key, as cross-deployment resumeHook writes it', async () => { + // Regression: on Vercel, `resumeHook()` seals hook payloads to the + // target run's published X25519 public key (`encp`) instead of + // symmetric `encr`. The QuickJS engine resolved only the bare + // symmetric key, so the first sealed payload failed to open and the + // run wedged right after hook_received (every hook e2e timed out). + // The engine must resolve the run's FULL capability, like the + // node:vm engine's memoizeEncryptionKey does. + const { dehydrateStepReturnValue, sealTo } = await import( + '../serialization.js' + ); + const { deriveRunKeyPair } = await import('../sealed-box.js'); + const { deriveRunPayloadKeys } = await import( + '../serialization/encryption.js' + ); + + const code = ` + async function workflow() { + var hook = globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]({ token: "tok" }); + var payload = await hook; + return payload; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + const run = makeRun(); + + // First invocation: workflow suspends awaiting the hook. + const r1 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + const hookCid = r1.suspended!.pendingOperations.find( + (o) => o.type === 'hook' + )!.correlationId; + + // Seal the payload exactly as a cross-deployment resumeHook does: + // dehydrate with a SealTarget built from the run's public key. + const material = new Uint8Array(32).fill(7); + const { publicKey } = await deriveRunKeyPair(material); + const payload = { approved: true, note: 'sealed round-trip' }; + const sealedPayload = await dehydrateStepReturnValue( + payload, + run.runId, + sealTo(publicKey), + [], + globalThis, + false + ); + expect(sealedPayload).toBeInstanceOf(Uint8Array); + + // Replay with the hook_received carrying the sealed payload. The + // runtime holds the run's full capability derived from the same key + // material — it must open the sealed envelope. + const r2 = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + encryptionKey: await deriveRunPayloadKeys(material), + events: [ + runCreatedEvent(run), + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'hook_created', + correlationId: hookCid, + eventData: { token: 'tok', isWebhook: false }, + createdAt: new Date('2025-01-01T00:00:01Z'), + }, + { + eventId: 'evnt_002', + runId: run.runId, + eventType: 'hook_received', + correlationId: hookCid, + eventData: { payload: sealedPayload }, + createdAt: new Date('2025-01-01T00:00:02Z'), + }, + ], + }); + + expect(r2.completed).toBeDefined(); + expect(unwrapResult(r2.completed!.result)).toEqual(payload); + }); +}); diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index edbabb76d3..56f6162100 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -31,9 +31,9 @@ import type { Event, RunInput, WorkflowRun } from '@workflow/world'; import * as nanoid from 'nanoid'; import { JSException, QuickJS, type WasiOptions } from 'quickjs-wasi'; import seedrandom from 'seedrandom'; -import type { CryptoKey } from '../encryption.js'; import { runtimeLogger } from '../logger.js'; import { decompress } from '../serialization/compression.js'; +import type { DecryptionKey } from '../serialization/encryption.js'; import { decrypt } from '../serialization/encryption.js'; import { quickjsExtensions, quickjsWasm } from './quickjs-assets.generated.js'; import { runIdCreatedAt } from './run-id-time.js'; @@ -45,14 +45,17 @@ import { VM_SERDE_BUNDLE } from './vm-serde-bundle.generated.js'; * Prepare persisted payload bytes for consumption inside the VM: decrypt * (when an encryption key is configured) and decompress (specVersion >= 5 * payloads may be gzip/zstd-compressed). The VM only understands plain - * format-prefixed 'devl' bytes — it has neither the CryptoKey nor zlib. + * format-prefixed 'devl' bytes — it has neither the key material nor zlib. + * The key is the run's full DecryptionKey capability (symmetric AES key + + * X25519 keypair) so sealed `encp` hook payloads from cross-deployment + * resumeHook() calls open here too, not just symmetric `encr` ones. * Both stages are format-prefix dispatched, so plaintext/uncompressed * data passes through unchanged. Mirrors `prepareReplayPayload` in * serialization.ts (the node:vm engine's equivalent host-side stage). */ async function prepareBytesForVM( data: Uint8Array, - key?: CryptoKey + key?: DecryptionKey ): Promise { return (await decompress(await decrypt(data, key))) as Uint8Array; } @@ -187,7 +190,7 @@ export interface QuickJSRuntimeOptions { */ events: Event[]; /** Encryption key for decrypting event payloads (undefined if unencrypted) */ - encryptionKey?: CryptoKey; + encryptionKey?: DecryptionKey; /** * The local port the workflow server is listening on, used to populate * `workflowMetadata.url`. Resolved at call time on the host side so the @@ -1057,7 +1060,7 @@ async function processEvents( vm: QuickJS, events: Event[], advanceClock: (ms: number) => void, - encryptionKey?: CryptoKey + encryptionKey?: DecryptionKey ): Promise { let resolved = false; for (const event of events) { From 1ff0ed9ae6e1af15219d259da51c6b1321a81421 Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Fri, 31 Jul 2026 13:51:12 -0700 Subject: [PATCH 15/18] Address review: crypto/process parity, loud Intl guards, lazy engine import, VM-leak guard, telemetry namespace, eval-string escaping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Deterministic crypto.getRandomValues/randomUUID in the VM bootstrap, drawing from the seeded Math.random (identical sequences to the node engine's vm/index.ts implementations); all crypto.subtle methods throw with step-function guidance. process.env exposed as a frozen copy, matching node. - Intl: throwing constructors (no ICU in QuickJS), and toLocale*-family methods (incl. localeCompare) throw when given an explicit locale so cross-engine divergence is loud instead of silently writing different values into the event log. No-argument forms keep working. - runtime.ts lazy-imports the QuickJS entrypoint at dispatch, keeping the ~1.3MB embedded WASM assets out of node-engine deployments. - runQuickJSWorkflow wraps the per-run phase so an exceptional exit disposes the VM instead of leaking it in a reused compute instance; corrected the misleading fail-loud comment (run_failed, not retry); warn when the event drain loop exhausts its iteration bound. - Telemetry attributes renamed quickjs.* → workflow.vm.* to stay in the file's workflow.* namespace. - Eval-string correlation-id interpolation uses JSON.stringify instead of quote-only escaping. - common-vm.test.ts pins the reducer/reviver superset invariant against common.ts so the duplicated sets can't silently drift. - Docs enumerate the remaining global-surface differences (subtle.digest, Intl, WebAssembly, Atomics); quickjs-entrypoint documents the known precondition-guard gap. --- .../docs/v5/configuration/runtime-tuning.mdx | 7 +- packages/core/src/runtime.ts | 9 +- .../core/src/runtime/quickjs-entrypoint.ts | 10 + .../core/src/runtime/quickjs-runtime.test.ts | 91 ++++ packages/core/src/runtime/quickjs-runtime.ts | 505 ++++++++++++------ .../serialization/reducers/common-vm.test.ts | 67 +++ .../src/telemetry/semantic-conventions.ts | 10 +- 7 files changed, 518 insertions(+), 181 deletions(-) create mode 100644 packages/core/src/serialization/reducers/common-vm.test.ts diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 68f166ccd2..f4c94a3327 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -117,7 +117,12 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Values: `node` or `quickjs` - Selects the sandboxed VM engine that executes workflow functions (`"use workflow"`). Step functions are unaffected — they always run with full Node.js access. - `node` (default) runs workflow code in a [`node:vm`](https://nodejs.org/api/vm.html) context. -- `quickjs` (experimental) runs workflow code in a [QuickJS](https://github.com/quickjs-ng/quickjs) VM compiled to WebAssembly (via [`quickjs-wasi`](https://github.com/vercel-labs/quickjs-wasi)). Both engines implement the same event-replay execution model and are interchangeable per run. The QuickJS engine is intended for platforms that do not implement `node:vm`, and is the foundation for future VM-memory snapshotting. +- `quickjs` (experimental) runs workflow code in a [QuickJS](https://github.com/quickjs-ng/quickjs) VM compiled to WebAssembly (via [`quickjs-wasi`](https://github.com/vercel-labs/quickjs-wasi)). Both engines implement the same event-replay execution model (seeded PRNG, deterministic clock, and correlation-ID sequences are identical), but the **global surface is not identical** — see the differences below before switching an existing deployment. The QuickJS engine is intended for platforms that do not implement `node:vm`, and is the foundation for future VM-memory snapshotting. +- Global-surface differences under `quickjs` (workflow functions only — step functions always have full Node.js): + - `crypto.getRandomValues()` and `crypto.randomUUID()` are provided and deterministic (seeded like the node engine's). All `crypto.subtle.*` methods throw with guidance to move to a step function — including `digest`, which the node engine supports. + - `Intl` is not available (QuickJS has no ICU). The `Intl.*` constructors throw, and `toLocaleString`-family methods (including `localeCompare`) throw when called **with an explicit locale** — calling them without arguments keeps the engine default. Perform locale-sensitive formatting in a step function. + - `WebAssembly` and `Atomics` are not available. + - `process` exposes only a frozen copy of `env`, matching the node engine. - The engine choice is stamped into the run's `executionContext` when the run starts, so a run keeps executing on the engine it started on even if the deployment's `WORKFLOW_VM` changes. Runs without a stamped engine use the handler's `WORKFLOW_VM` value. - Unknown values throw at startup. diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 355ae48be4..a2d5c86aa1 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -70,7 +70,6 @@ import { queueMessage, withHealthCheck, } from './runtime/helpers.js'; -import { runWorkflowWithQuickJS } from './runtime/quickjs-entrypoint.js'; import { handleReplayBudgetExhausted, ReplayBudget, @@ -1628,6 +1627,14 @@ export function workflowEntrypoint( // first — it does not thread the turbo runReadyBarrier // the way handleSuspension does. await awaitRunReady(); + // Lazy import: the QuickJS entrypoint's import chain + // embeds the base64 WASM binary + extensions (~1.3 MB + // decoded at module scope). Loading it here keeps that + // out of node-engine deployments entirely — only the + // opt-in path pays, on first dispatch. + const { runWorkflowWithQuickJS } = await import( + './runtime/quickjs-entrypoint.js' + ); const quickjsResult = await runWorkflowWithQuickJS({ workflowCode, workflowName, diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index c7e2924cab..3a34514ab9 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -436,6 +436,16 @@ async function dispatchPendingOps(params: { * * This replaces the `node:vm` replay path (runWorkflow + EventsConsumer) * with a QuickJS VM invocation that performs the same full event replay. + * + * KNOWN GAP — precondition guard: unlike the node:vm path, no event write + * in this file participates in the optimistic-concurrency precondition + * guard (`withPreconditionRetry` + `stateUpdatedAtForCreate`), which + * protects a writer holding a stale event-log snapshot from clobbering a + * concurrent one. The engine currently relies on per-(runId, + * correlationId) event uniqueness (EntityConflictError dedup) alone. This + * is a deliberate simplification while the engine is experimental — wiring + * the guard is tracked follow-up work; anyone adding new write paths here + * should not assume parity with the node engine on this axis. */ export async function runWorkflowWithQuickJS(params: { workflowCode: string; diff --git a/packages/core/src/runtime/quickjs-runtime.test.ts b/packages/core/src/runtime/quickjs-runtime.test.ts index dd3e41f66a..e88179f858 100644 --- a/packages/core/src/runtime/quickjs-runtime.test.ts +++ b/packages/core/src/runtime/quickjs-runtime.test.ts @@ -889,3 +889,94 @@ describe('sealed (encp) hook payloads', () => { expect(unwrapResult(r2.completed!.result)).toEqual(payload); }); }); + +describe('global surface parity', () => { + const runToCompletion = async (body: string) => { + const code = ` + async function workflow() { ${body} } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + const run = makeRun(); + const result = await runQuickJSWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [runCreatedEvent(run)], + }); + return result; + }; + + it('crypto.randomUUID and getRandomValues are present and deterministic across invocations', async () => { + const body = ` + var bytes = crypto.getRandomValues(new Uint8Array(8)); + return { uuid: crypto.randomUUID(), bytes: Array.from(bytes) }; + `; + const r1 = await runToCompletion(body); + const r2 = await runToCompletion(body); + expect(r1.completed).toBeDefined(); + const v1 = unwrapResult(r1.completed!.result) as any; + const v2 = unwrapResult(r2.completed!.result) as any; + // Replay determinism: same seeded PRNG → identical values on every + // invocation of the same run. + expect(v1.uuid).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ + ); + expect(v2.uuid).toBe(v1.uuid); + expect(v2.bytes).toEqual(v1.bytes); + }); + + it('crypto.subtle methods throw with step-function guidance', async () => { + const result = await runToCompletion(` + try { + await crypto.subtle.digest("SHA-256", new Uint8Array(1)); + return { threw: false }; + } catch (e) { + return { threw: true, name: e.name, message: e.message }; + } + `); + const value = unwrapResult(result.completed!.result) as any; + expect(value.threw).toBe(true); + expect(value.message).toContain('step function'); + }); + + it('process.env is present (frozen copy, matching the node engine)', async () => { + const result = await runToCompletion(` + return { + hasProcess: typeof process === "object", + envIsObject: typeof process.env === "object", + frozen: Object.isFrozen(process.env), + }; + `); + expect(unwrapResult(result.completed!.result)).toEqual({ + hasProcess: true, + envIsObject: true, + frozen: true, + }); + }); + + it('Intl constructors and explicit-locale toLocale* calls throw loudly instead of diverging silently', async () => { + const result = await runToCompletion(` + var out = {}; + try { new Intl.NumberFormat("de-DE"); out.intl = "no-throw"; } + catch (e) { out.intl = e.message.indexOf("ICU") !== -1 ? "threw" : e.message; } + try { (1234.5).toLocaleString("de-DE"); out.number = "no-throw"; } + catch (e) { out.number = "threw"; } + try { new Date(0).toLocaleDateString("de-DE"); out.date = "no-throw"; } + catch (e) { out.date = "threw"; } + try { "a".localeCompare("b", "de-DE"); out.compare = "no-throw"; } + catch (e) { out.compare = "threw"; } + // No-argument forms keep working with the engine default. + out.plain = (1234.5).toLocaleString(); + out.plainCompare = "a".localeCompare("b"); + return out; + `); + const value = unwrapResult(result.completed!.result) as any; + expect(value.intl).toBe('threw'); + expect(value.number).toBe('threw'); + expect(value.date).toBe('threw'); + expect(value.compare).toBe('threw'); + expect(typeof value.plain).toBe('string'); + expect(value.plainCompare).toBeLessThan(0); + }); +}); diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index 56f6162100..464820d47a 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -264,6 +264,113 @@ globalThis.exports = {}; globalThis.module = { exports: globalThis.exports }; // NOTE: TextEncoder/TextDecoder are provided by the native encoding extension. +// ---- Deterministic \`crypto\` (parity with the node:vm engine) ---- +// getRandomValues / randomUUID draw from Math.random, which the host +// replaces with the run's seeded PRNG before any user code runs — so the +// values replay deterministically and match the node engine, whose +// implementations draw from the same seeded sequence (see vm/index.ts). +// Every crypto.subtle method throws with the same guidance as the node +// engine's non-replayable methods; unlike node, \`digest\` is also +// unavailable here (no native hash in the VM yet). +(function() { + function getRandomValues(array) { + for (var i = 0; i < array.length; i++) { + array[i] = Math.floor(Math.random() * 256); + } + return array; + } + // Mirrors vm/uuid.ts createRandomUUID: identical draw pattern from the + // seeded PRNG, so both engines produce the same UUID at the same point + // in a replay. + function randomUUID() { + var chars = "0123456789abcdef"; + var uuid = ""; + for (var i = 0; i < 36; i++) { + if (i === 8 || i === 13 || i === 18 || i === 23) { + uuid += "-"; + } else if (i === 14) { + uuid += "4"; + } else if (i === 19) { + uuid += chars[Math.floor(Math.random() * 4) + 8]; + } else { + uuid += chars[Math.floor(Math.random() * 16)]; + } + } + return uuid; + } + function subtleThrow(name) { + return function() { + var err = new Error("\`crypto.subtle." + name + "()\` is not available inside a workflow function. Move it to a step function where full Node.js crypto is available."); + err.name = "WorkflowRuntimeError"; + throw err; + }; + } + var subtle = {}; + ["encrypt","decrypt","sign","verify","digest","generateKey","deriveKey","deriveBits","importKey","exportKey","wrapKey","unwrapKey"].forEach(function(m) { + subtle[m] = subtleThrow(m); + }); + globalThis.crypto = { + getRandomValues: getRandomValues, + randomUUID: randomUUID, + subtle: subtle, + }; +})(); + +// ---- Loud Intl / locale guards ---- +// QuickJS has no ICU: \`Intl\` is absent and toLocaleString-family methods +// silently ignore their locale argument. Silent divergence from the node +// engine would write different values into a durable event log with no +// error anywhere — so make the gap loud instead: Intl constructors throw, +// and toLocale* methods throw ONLY when called with an explicit locale +// (the no-argument forms keep QuickJS's default behavior). +(function() { + function intlThrow(name) { + return function() { + var err = new Error("\`Intl." + name + "\` is not available in the QuickJS workflow engine (no ICU). Perform locale-sensitive formatting in a step function, or use WORKFLOW_VM=node."); + err.name = "WorkflowRuntimeError"; + throw err; + }; + } + if (typeof Intl === "undefined") { + var intl = {}; + ["Collator","DateTimeFormat","DisplayNames","DurationFormat","ListFormat","Locale","NumberFormat","PluralRules","RelativeTimeFormat","Segmenter"].forEach(function(n) { + intl[n] = intlThrow(n); + }); + intl.getCanonicalLocales = intlThrow("getCanonicalLocales"); + globalThis.Intl = intl; + } + function guardLocale(proto, method) { + var original = proto[method]; + if (typeof original !== "function") return; + proto[method] = function(locales) { + if (locales !== undefined) { + var err = new Error("\`" + method + "(locales, ...)\` with an explicit locale is not supported in the QuickJS workflow engine (no ICU) — it would silently ignore the locale. Format in a step function, or call without arguments for the engine default."); + err.name = "WorkflowRuntimeError"; + throw err; + } + return original.call(this); + }; + } + guardLocale(Number.prototype, "toLocaleString"); + guardLocale(Date.prototype, "toLocaleString"); + guardLocale(Date.prototype, "toLocaleDateString"); + guardLocale(Date.prototype, "toLocaleTimeString"); + guardLocale(String.prototype, "toLocaleLowerCase"); + guardLocale(String.prototype, "toLocaleUpperCase"); + // localeCompare's locales argument is the SECOND parameter. + (function() { + var original = String.prototype.localeCompare; + String.prototype.localeCompare = function(that, locales) { + if (locales !== undefined) { + var err = new Error("\`localeCompare(that, locales, ...)\` with an explicit locale is not supported in the QuickJS workflow engine (no ICU). Compare in a step function, or call without a locale."); + err.name = "WorkflowRuntimeError"; + throw err; + } + return original.call(this, that); + }; + })(); +})(); + globalThis[Symbol.for("WORKFLOW_USE_STEP")] = function(stepId, closureVarsFn) { var fn = function() { var args = Array.prototype.slice.call(arguments); @@ -882,120 +989,152 @@ export async function runQuickJSWorkflow( // ---- Phase 1: static initialization ---- const vm = await initWorkflowVM(() => vmNowMs); - // ---- Phase 2: per-run initialization ---- - - // Seeded Math.random - { - using randomFn = vm.newFunction('random', () => vm.newNumber(rng())); - using math = vm.global.getProp('Math'); - math.setProp('random', randomFn); - } - - // Seeded nanoid generator - { - using nanoidFn = vm.newFunction('__generateNanoid', () => - vm.newString(generateNanoid()) - ); - vm.setProp(vm.global, '__generateNanoid', nanoidFn); - } - - // Inject a deterministic timestamp for the VM's ULID factory. ULIDs - // produced inside the VM use this as their time prefix instead of - // Date.now(), so two concurrent workflow invocations of the same run - // produce IDENTICAL correlationIds (the random portion also matches - // because the PRNG is seeded the same way) and the world's - // EntityConflictError on `events.create` dedups one of each pair. - // Derived from the runId's embedded ULID (stable across invocations by - // construction — unlike `startedAt`, which differs between turbo's - // synthesized run object and the durably stored run). - vm.evalCode( - `globalThis.__ulidTimestamp = ${runIdCreatedAt(workflowRun.runId) ?? (+workflowRun.createdAt || startedAt)};` - ).dispose(); - - // Execute the workflow bundle — use the workflowId as the eval filename - // so QuickJS stack traces reference the workflow name, enabling source map - // remapping by remapErrorStack (which matches frames by filename). - // Evaluated in the per-run phase (after Math.random seeding) so that - // module-scope user code draws from the seeded PRNG, matching the - // node:vm engine's replay determinism. + // Any throw between here and the terminal paths (which dispose the VM + // inside checkWorkflowState / extractError before RETURNING) would leak + // a live QuickJS instance and its WASM linear memory for the lifetime + // of the compute instance — which is reused. Dispose on the way out of + // an exceptional exit and rethrow. try { - vm.evalCode(workflowCode, workflowId || 'workflow.js').dispose(); + return await runWorkflowInVM(); } catch (err) { - return extractError(vm, err, 'Workflow evaluation failed'); + try { + vm.dispose(); + } catch { + // Already disposed by a terminal path — ignore. + } + throw err; } - // Extract workflow arguments. Prefer the run_created event; fall back - // to the queue message's runInput if the event log is incomplete - // (eventually-consistent read after start()). Failing to find input - // for a first invocation is fatal — running the workflow function - // with no args would silently turn typed arguments into `undefined` - // and, for recursive workflows, produce exponential fan-out. - const runCreatedEvent = events.find((e) => e.eventType === 'run_created'); - const runCreatedInput = - runCreatedEvent && 'eventData' in runCreatedEvent - ? (runCreatedEvent.eventData as Record)?.input - : undefined; - const runInput: unknown = - runCreatedInput ?? (options.runInput?.input as unknown); - - if (runInput instanceof Uint8Array) { - const decryptedInput = await prepareBytesForVM( - runInput, - options.encryptionKey - ); - runtimeLogger.debug('QuickJS runtime: run input format', { - prefix: new TextDecoder().decode(decryptedInput.subarray(0, 4)), - byteLength: decryptedInput.byteLength, - source: runCreatedInput ? 'run_created' : 'queueMessage.runInput', - }); - const inputHandle = vm.newUint8Array(decryptedInput); - vm.setProp(vm.global, '__wdk_input', inputHandle); - inputHandle.dispose(); - } else if (runInput === undefined && events.length > 0) { - // The event log is non-empty (we got run_started or similar) but - // no run_created event was found and no queue-provided runInput is - // available. This is the race condition observed during the fib - // incident — silently dropping arguments would turn `n` into - // `undefined` and, for recursive workflows, cause exponential - // fan-out. Fail loud so the run goes to `run_failed` and the queue - // can retry. Empty `events` is allowed because tests that bootstrap - // a workflow with no arguments rely on the old permissive behavior. - throw new Error( - `Cannot start workflow run "${workflowRun.runId}": no run_created event found and no runInput in the queue payload, but other events are present (likely a read-after-write race during start()).` - ); - } + // ---- Phase 2: per-run initialization ---- + async function runWorkflowInVM(): Promise { + // Seeded Math.random + { + using randomFn = vm.newFunction('random', () => vm.newNumber(rng())); + using math = vm.global.getProp('Math'); + math.setProp('random', randomFn); + } - // Set workflow context metadata (for getWorkflowMetadata()). - // Must match the shape that the node:vm engine produces (see - // packages/core/src/workflow.ts: runWorkflow → ctx) so user code - // that compares `getWorkflowMetadata()` values between a step - // (server-side) and the workflow (VM-side) sees identical objects. - { - const metadata = { - workflowName: workflowRun.workflowName, - workflowRunId: workflowRun.runId, - workflowStartedAt: workflowRun.startedAt - ? new Date(+workflowRun.startedAt) - : new Date(), - url: process.env.VERCEL_URL - ? `https://${process.env.VERCEL_URL}` - : `http://localhost:${options.port ?? 3000}`, - features: { encryption: !!options.encryptionKey }, - }; + // Seeded nanoid generator + { + using nanoidFn = vm.newFunction('__generateNanoid', () => + vm.newString(generateNanoid()) + ); + vm.setProp(vm.global, '__generateNanoid', nanoidFn); + } + + // Inject a deterministic timestamp for the VM's ULID factory. ULIDs + // produced inside the VM use this as their time prefix instead of + // Date.now(), so two concurrent workflow invocations of the same run + // produce IDENTICAL correlationIds (the random portion also matches + // because the PRNG is seeded the same way) and the world's + // EntityConflictError on `events.create` dedups one of each pair. + // Derived from the runId's embedded ULID (stable across invocations by + // construction — unlike `startedAt`, which differs between turbo's + // synthesized run object and the durably stored run). vm.evalCode( - `globalThis[Symbol.for("WORKFLOW_CONTEXT")] = ${JSON.stringify(metadata)};` + - `globalThis[Symbol.for("WORKFLOW_CONTEXT")].workflowStartedAt = new Date(${JSON.stringify(metadata.workflowStartedAt.toISOString())});` + `globalThis.__ulidTimestamp = ${runIdCreatedAt(workflowRun.runId) ?? (+workflowRun.createdAt || startedAt)};` ).dispose(); - } - // Start the workflow function. If the workflow isn't registered, - // throw an error tagged with `name = "WorkflowNotRegisteredError"` - // so the host-side entrypoint can reconstruct a real - // WorkflowNotRegisteredError (a WorkflowRuntimeError subclass that - // classifies as RUNTIME_ERROR) rather than a generic user error. - // See quickjs-entrypoint.ts's run_failed branch. - try { - vm.evalCode(` + // `process.env` — parity with the node:vm engine, which exposes a frozen + // copy of the host env (vm/index.ts). Injected per run so the snapshot of + // the env is taken at invocation time, same as node. + { + const envHandle = vm.newString(JSON.stringify(process.env)); + vm.setProp(vm.global, '__wdk_env', envHandle); + envHandle.dispose(); + vm.evalCode( + 'globalThis.process = { env: Object.freeze(JSON.parse(globalThis.__wdk_env)) };' + + 'delete globalThis.__wdk_env;' + ).dispose(); + } + + // Execute the workflow bundle — use the workflowId as the eval filename + // so QuickJS stack traces reference the workflow name, enabling source map + // remapping by remapErrorStack (which matches frames by filename). + // Evaluated in the per-run phase (after Math.random seeding) so that + // module-scope user code draws from the seeded PRNG, matching the + // node:vm engine's replay determinism. + try { + vm.evalCode(workflowCode, workflowId || 'workflow.js').dispose(); + } catch (err) { + return extractError(vm, err, 'Workflow evaluation failed'); + } + + // Extract workflow arguments. Prefer the run_created event; fall back + // to the queue message's runInput if the event log is incomplete + // (eventually-consistent read after start()). Failing to find input + // for a first invocation is fatal — running the workflow function + // with no args would silently turn typed arguments into `undefined` + // and, for recursive workflows, produce exponential fan-out. + const runCreatedEvent = events.find((e) => e.eventType === 'run_created'); + const runCreatedInput = + runCreatedEvent && 'eventData' in runCreatedEvent + ? (runCreatedEvent.eventData as Record)?.input + : undefined; + const runInput: unknown = + runCreatedInput ?? (options.runInput?.input as unknown); + + if (runInput instanceof Uint8Array) { + const decryptedInput = await prepareBytesForVM( + runInput, + options.encryptionKey + ); + runtimeLogger.debug('QuickJS runtime: run input format', { + prefix: new TextDecoder().decode(decryptedInput.subarray(0, 4)), + byteLength: decryptedInput.byteLength, + source: runCreatedInput ? 'run_created' : 'queueMessage.runInput', + }); + const inputHandle = vm.newUint8Array(decryptedInput); + vm.setProp(vm.global, '__wdk_input', inputHandle); + inputHandle.dispose(); + } else if (runInput === undefined && events.length > 0) { + // The event log is non-empty (we got run_started or similar) but + // no run_created event was found and no queue-provided runInput is + // available. This is the race condition observed during the fib + // incident — silently dropping arguments would turn `n` into + // `undefined` and, for recursive workflows, cause exponential + // fan-out. Fail loud: the throw propagates to the entrypoint's + // catch, which records run_failed. A visible terminal failure is + // preferred over silently executing with undefined arguments — the + // queue-provided runInput fallback above makes this path rare. + // Empty `events` is allowed because tests that bootstrap a workflow + // with no arguments rely on the old permissive behavior. + throw new Error( + `Cannot start workflow run "${workflowRun.runId}": no run_created event found and no runInput in the queue payload, but other events are present (likely a read-after-write race during start()).` + ); + } + + // Set workflow context metadata (for getWorkflowMetadata()). + // Must match the shape that the node:vm engine produces (see + // packages/core/src/workflow.ts: runWorkflow → ctx) so user code + // that compares `getWorkflowMetadata()` values between a step + // (server-side) and the workflow (VM-side) sees identical objects. + { + const metadata = { + workflowName: workflowRun.workflowName, + workflowRunId: workflowRun.runId, + workflowStartedAt: workflowRun.startedAt + ? new Date(+workflowRun.startedAt) + : new Date(), + url: process.env.VERCEL_URL + ? `https://${process.env.VERCEL_URL}` + : `http://localhost:${options.port ?? 3000}`, + features: { encryption: !!options.encryptionKey }, + }; + vm.evalCode( + `globalThis[Symbol.for("WORKFLOW_CONTEXT")] = ${JSON.stringify(metadata)};` + + `globalThis[Symbol.for("WORKFLOW_CONTEXT")].workflowStartedAt = new Date(${JSON.stringify(metadata.workflowStartedAt.toISOString())});` + ).dispose(); + } + + // Start the workflow function. If the workflow isn't registered, + // throw an error tagged with `name = "WorkflowNotRegisteredError"` + // so the host-side entrypoint can reconstruct a real + // WorkflowNotRegisteredError (a WorkflowRuntimeError subclass that + // classifies as RUNTIME_ERROR) rather than a generic user error. + // See quickjs-entrypoint.ts's run_failed branch. + try { + vm.evalCode(` var __wfn = globalThis.__private_workflows.get(${JSON.stringify(workflowId)}); if (!__wfn) { var __wfnErr = new Error("Workflow \\"" + ${JSON.stringify(workflowId)} + "\\" is not registered in the current deployment."); @@ -1024,34 +1163,48 @@ export async function runQuickJSWorkflow( } ); `).dispose(); - } catch (err) { - return extractError(vm, err, 'Failed to start workflow'); - } + } catch (err) { + return extractError(vm, err, 'Failed to start workflow'); + } - // Process events and drain jobs in a loop. Events may resolve promises - // that unblock workflow code, which then creates NEW resolvers for - // subsequent events. Re-processing events matches these new resolvers - // against events that were already delivered. - { - let maxIterations = 100; - let madeProgress: boolean; - do { - madeProgress = await processEvents( - vm, - events, - advanceClock, - options.encryptionKey - ); - let batch: number; + // Process events and drain jobs in a loop. Events may resolve promises + // that unblock workflow code, which then creates NEW resolvers for + // subsequent events. Re-processing events matches these new resolvers + // against events that were already delivered. + { + let maxIterations = 100; + let madeProgress: boolean; do { - batch = vm.executePendingJobs(); - if (batch > 0) madeProgress = true; - } while (batch > 0); - } while (madeProgress && --maxIterations > 0); - } + madeProgress = await processEvents( + vm, + events, + advanceClock, + options.encryptionKey + ); + let batch: number; + do { + batch = vm.executePendingJobs(); + if (batch > 0) madeProgress = true; + } while (batch > 0); + } while (madeProgress && --maxIterations > 0); + if (madeProgress && maxIterations === 0) { + // The drain loop hit its bound while still making progress — + // proceeding as if it converged would present as a mysterious + // suspension or replay divergence. Make the giving-up visible so + // a wedge is attributable to this bound rather than a mystery. + runtimeLogger.warn( + 'QuickJS runtime: event drain loop hit its iteration bound before reaching a fixed point', + { + workflowRunId: workflowRun.runId, + eventCount: events.length, + } + ); + } + } - // ---- Check result ---- - return checkWorkflowState(vm); + // ---- Check result ---- + return checkWorkflowState(vm); + } } // ---- Event Processing ---- @@ -1074,7 +1227,10 @@ async function processEvents( const cid = event.correlationId; if (!cid) continue; - const escapedCid = cid.replace(/"/g, '\\"'); + // JSON.stringify handles quotes, backslashes and control characters; + // correlation ids are host-generated ULIDs today, but the eval-string + // safety shouldn't depend on that invariant being asserted nowhere. + const cidJs = JSON.stringify(cid); const eventData = 'eventData' in event ? (event.eventData as Record) @@ -1084,14 +1240,14 @@ async function processEvents( switch (event.eventType) { case 'step_completed': { const hasResolver = vm.dump( - vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + vm.evalCode(`!!globalThis.__resolvers[${cidJs}]`) ); const rawOutput = eventData?.result ?? eventData?.output; if (hasResolver) { if (rawOutput instanceof Uint8Array) { // Decrypt if encrypted — the VM only understands 'devl' format runtimeLogger.debug('QuickJS runtime: step result raw', { - correlationId: escapedCid, + correlationId: cid, rawPrefix: new TextDecoder().decode(rawOutput.subarray(0, 4)), rawByteLength: rawOutput.byteLength, isBuffer: Buffer.isBuffer(rawOutput), @@ -1101,7 +1257,7 @@ async function processEvents( encryptionKey ); runtimeLogger.debug('QuickJS runtime: step result decrypted', { - correlationId: escapedCid, + correlationId: cid, prefix: new TextDecoder().decode(decryptedOutput.subarray(0, 4)), byteLength: decryptedOutput.byteLength, }); @@ -1109,13 +1265,13 @@ async function processEvents( vm.setProp(vm.global, '__tmp_result', bytesHandle); bytesHandle.dispose(); vm.evalCode( - `globalThis.__resolvers["${escapedCid}"].resolve(globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result));` + - `delete globalThis.__resolvers["${escapedCid}"];` + + `globalThis.__resolvers[${cidJs}].resolve(globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result));` + + `delete globalThis.__resolvers[${cidJs}];` + `delete globalThis.__tmp_result;` ).dispose(); } else { runtimeLogger.debug('QuickJS runtime: step result non-binary', { - correlationId: escapedCid, + correlationId: cid, type: typeof rawOutput, isNull: rawOutput === null, isUndefined: rawOutput === undefined, @@ -1124,8 +1280,8 @@ async function processEvents( const serialized = rawOutput !== undefined ? JSON.stringify(rawOutput) : 'undefined'; vm.evalCode( - `globalThis.__resolvers["${escapedCid}"].resolve(${serialized});` + - `delete globalThis.__resolvers["${escapedCid}"];` + `globalThis.__resolvers[${cidJs}].resolve(${serialized});` + + `delete globalThis.__resolvers[${cidJs}];` ).dispose(); } // Drain ALL microtasks after resolve @@ -1137,12 +1293,12 @@ async function processEvents( } while (b > 0); } } - markCreated(vm, escapedCid); + markCreated(vm, cidJs); break; } case 'step_failed': { const hasResolver = vm.dump( - vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + vm.evalCode(`!!globalThis.__resolvers[${cidJs}]`) ); if (hasResolver) { const errorData = eventData?.error; @@ -1160,8 +1316,8 @@ async function processEvents( vm.evalCode( `(function(){` + `var e=globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_error);` + - `globalThis.__resolvers["${escapedCid}"].reject(e);` + - `delete globalThis.__resolvers["${escapedCid}"];` + + `globalThis.__resolvers[${cidJs}].reject(e);` + + `delete globalThis.__resolvers[${cidJs}];` + `delete globalThis.__tmp_error;` + `})()` ).dispose(); @@ -1187,8 +1343,8 @@ async function processEvents( : ''; vm.evalCode( `(function(){var e=new Error(${JSON.stringify(msg)});e.name="FatalError";e.fatal=true;${stackAssignment}` + - `globalThis.__resolvers["${escapedCid}"].reject(e);` + - `delete globalThis.__resolvers["${escapedCid}"];})()` + `globalThis.__resolvers[${cidJs}].reject(e);` + + `delete globalThis.__resolvers[${cidJs}];})()` ).dispose(); } { @@ -1199,17 +1355,17 @@ async function processEvents( } while (b > 0); } } - markCreated(vm, escapedCid); + markCreated(vm, cidJs); break; } case 'wait_completed': { const hasResolver = vm.dump( - vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + vm.evalCode(`!!globalThis.__resolvers[${cidJs}]`) ); if (hasResolver) { vm.evalCode( - `globalThis.__resolvers["${escapedCid}"].resolve();` + - `delete globalThis.__resolvers["${escapedCid}"];` + `globalThis.__resolvers[${cidJs}].resolve();` + + `delete globalThis.__resolvers[${cidJs}];` ).dispose(); { resolved = true; @@ -1219,7 +1375,7 @@ async function processEvents( } while (b > 0); } } - markCreated(vm, escapedCid); + markCreated(vm, cidJs); break; } case 'attr_set': { @@ -1230,12 +1386,12 @@ async function processEvents( ?.type; if (writer !== 'workflow') break; const hasResolver = vm.dump( - vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + vm.evalCode(`!!globalThis.__resolvers[${cidJs}]`) ); if (hasResolver) { vm.evalCode( - `globalThis.__resolvers["${escapedCid}"].resolve();` + - `delete globalThis.__resolvers["${escapedCid}"];` + `globalThis.__resolvers[${cidJs}].resolve();` + + `delete globalThis.__resolvers[${cidJs}];` ).dispose(); { resolved = true; @@ -1245,7 +1401,7 @@ async function processEvents( } while (b > 0); } } - markCreated(vm, escapedCid); + markCreated(vm, cidJs); break; } case 'hook_received': { @@ -1267,7 +1423,7 @@ async function processEvents( eventId: event.eventId, } ); - markCreated(vm, escapedCid); + markCreated(vm, cidJs); break; } @@ -1276,7 +1432,7 @@ async function processEvents( // The payload is the dehydrated `{ aborted: true, reason }` object. const isAbortHook = vm.dump( vm.evalCode( - `!!(globalThis.__abortSignals && globalThis.__abortSignals["${escapedCid}"])` + `!!(globalThis.__abortSignals && globalThis.__abortSignals[${cidJs}])` ) ); if (isAbortHook) { @@ -1293,12 +1449,12 @@ async function processEvents( `(function(){` + `var p=globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_abort);` + `delete globalThis.__tmp_abort;` + - `globalThis.__abortSignals["${escapedCid}"]._setAborted(p&&typeof p==="object"?p.reason:undefined);` + + `globalThis.__abortSignals[${cidJs}]._setAborted(p&&typeof p==="object"?p.reason:undefined);` + `})()` ).dispose(); } else { vm.evalCode( - `globalThis.__abortSignals["${escapedCid}"]._setAborted(undefined);` + `globalThis.__abortSignals[${cidJs}]._setAborted(undefined);` ).dispose(); } if (event.eventId) { @@ -1313,12 +1469,12 @@ async function processEvents( b = vm.executePendingJobs(); } while (b > 0); } - markCreated(vm, escapedCid); + markCreated(vm, cidJs); break; } const hasResolver = vm.dump( - vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + vm.evalCode(`!!globalThis.__resolvers[${cidJs}]`) ); const rawPayload = eventData?.payload ?? eventData?.result; runtimeLogger.debug('QuickJS runtime: processing hook_received', { @@ -1343,8 +1499,8 @@ async function processEvents( vm.setProp(vm.global, '__tmp_result', bytesHandle); bytesHandle.dispose(); vm.evalCode( - `globalThis.__resolvers["${escapedCid}"].resolve(globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result));` + - `delete globalThis.__resolvers["${escapedCid}"];` + + `globalThis.__resolvers[${cidJs}].resolve(globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result));` + + `delete globalThis.__resolvers[${cidJs}];` + `delete globalThis.__tmp_result;` ).dispose(); } else { @@ -1353,8 +1509,8 @@ async function processEvents( ? JSON.stringify(rawPayload) : 'undefined'; vm.evalCode( - `globalThis.__resolvers["${escapedCid}"].resolve(${serialized});` + - `delete globalThis.__resolvers["${escapedCid}"];` + `globalThis.__resolvers[${cidJs}].resolve(${serialized});` + + `delete globalThis.__resolvers[${cidJs}];` ).dispose(); } // Mark this event as processed in the VM heap to prevent @@ -1379,7 +1535,7 @@ async function processEvents( ? JSON.stringify(event.eventId) : 'null'; const bufferAndTrack = - `(globalThis.__hookPayloadBuffer["${escapedCid}"] = globalThis.__hookPayloadBuffer["${escapedCid}"] || [])` + + `(globalThis.__hookPayloadBuffer[${cidJs}] = globalThis.__hookPayloadBuffer[${cidJs}] || [])` + `.push(%PAYLOAD%);` + (event.eventId ? `(globalThis.__hookPayloadBuffer.__processedEventIds = globalThis.__hookPayloadBuffer.__processedEventIds || {})[${eventIdJs}] = true;` @@ -1416,7 +1572,7 @@ async function processEvents( ).dispose(); } } - markCreated(vm, escapedCid); + markCreated(vm, cidJs); break; } case 'hook_conflict': { @@ -1482,14 +1638,14 @@ async function processEvents( b = vm.executePendingJobs(); } while (b > 0); } - markCreated(vm, escapedCid); + markCreated(vm, cidJs); break; } case 'step_created': case 'step_started': case 'step_retrying': case 'wait_created': { - markCreated(vm, escapedCid); + markCreated(vm, cidJs); break; } case 'hook_created': { @@ -1515,13 +1671,13 @@ async function processEvents( b = vm.executePendingJobs(); } while (b > 0); } - markCreated(vm, escapedCid); + markCreated(vm, cidJs); break; } case 'hook_disposed': { // Disambiguate from the `hook` pending op with the same // correlationId — we want to mark the `hook_dispose` entry. - markCreated(vm, escapedCid, 'hook_dispose'); + markCreated(vm, cidJs, 'hook_dispose'); break; } } @@ -1529,15 +1685,16 @@ async function processEvents( return resolved; } -function markCreated(vm: QuickJS, escapedCid: string, opType?: string): void { +function markCreated(vm: QuickJS, cidJs: string, opType?: string): void { + // `cidJs` is the JSON.stringify-quoted correlation id (see processEvents). // `hook` and `hook_dispose` pending ops share the same correlationId, // so when processing `hook_disposed` events we must disambiguate by // type — otherwise `.find()` returns the original `hook` op and the // `hook_dispose` op is never marked, causing the entrypoint to keep // retrying a hook_disposed for an already-deleted entity. const predicate = opType - ? `function(p){return p.correlationId==="${escapedCid}"&&p.type==="${opType}";}` - : `function(p){return p.correlationId==="${escapedCid}";}`; + ? `function(p){return p.correlationId===${cidJs}&&p.type===${JSON.stringify(opType)};}` + : `function(p){return p.correlationId===${cidJs};}`; vm.evalCode( `var __p=globalThis.__pending.find(${predicate});` + `if(__p)__p.hasCreatedEvent=true;` diff --git a/packages/core/src/serialization/reducers/common-vm.test.ts b/packages/core/src/serialization/reducers/common-vm.test.ts new file mode 100644 index 0000000000..a0607353a1 --- /dev/null +++ b/packages/core/src/serialization/reducers/common-vm.test.ts @@ -0,0 +1,67 @@ +/** + * Drift guard for the duplicated reducer/reviver sets. + * + * `common-vm.ts` intentionally duplicates `common.ts` without Node.js + * dependencies so it can run inside the QuickJS VM. Nothing else keeps the + * two in sync: a reducer added to `common.ts` but not here means values + * serialize on one side of the VM boundary and fail to revive on the other, + * at runtime, for whichever type was added. + * + * These tests pin the invariant that held at review time: the VM set is a + * strict superset of the node set, adding exactly the stream/fetch types + * that the node side handles elsewhere (workflow.ts's context-specific + * reducers). + */ + +import { describe, expect, it } from 'vitest'; +import { + getCommonReducers as getNodeReducers, + getCommonRevivers as getNodeRevivers, +} from './common.js'; +import { + getCommonReducers as getVmReducers, + getCommonRevivers as getVmRevivers, +} from './common-vm.js'; + +/** + * Types the VM set adds on top of the node set. The node engine handles + * these with workflow-context-specific reducers in serialization.ts + * instead of the common set; the VM codec needs them in its common set + * because it has no other layer. + */ +const VM_ONLY_TYPES = [ + 'ReadableStream', + 'Request', + 'Response', + 'WritableStream', +]; + +describe('common-vm reducer/reviver drift guard', () => { + it('VM reducers ⊇ node reducers', () => { + const nodeKeys = Object.keys(getNodeReducers()); + const vmKeys = new Set(Object.keys(getVmReducers())); + const missing = nodeKeys.filter((key) => !vmKeys.has(key)); + expect( + missing, + 'reducer(s) exist in common.ts but not common-vm.ts — values of these types will serialize on the node side and fail to revive in the VM' + ).toEqual([]); + }); + + it('VM revivers ⊇ node revivers', () => { + const nodeKeys = Object.keys(getNodeRevivers()); + const vmKeys = new Set(Object.keys(getVmRevivers())); + const missing = nodeKeys.filter((key) => !vmKeys.has(key)); + expect( + missing, + 'reviver(s) exist in common.ts but not common-vm.ts — wire payloads of these types will fail to revive in the VM' + ).toEqual([]); + }); + + it('VM-only additions are exactly the known stream/fetch types', () => { + const nodeKeys = new Set(Object.keys(getNodeReducers())); + const extras = Object.keys(getVmReducers()) + .filter((key) => !nodeKeys.has(key)) + .sort(); + expect(extras).toEqual(VM_ONLY_TYPES); + }); +}); diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index f0e0c532e7..f13c6a8ee8 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -100,26 +100,26 @@ export const WorkflowVm = SemanticConvention<'node' | 'quickjs'>('workflow.vm'); /** Outcome of a QuickJS VM workflow invocation */ export const QuickJSOutcome = SemanticConvention< 'completed' | 'suspended' | 'failed' ->('quickjs.outcome'); +>('workflow.vm.outcome'); /** Whether preloaded events from `events.create('run_started')` were used */ export const QuickJSEventsPreloaded = SemanticConvention( - 'quickjs.events.preloaded' + 'workflow.vm.events.preloaded' ); /** Total number of events fetched from the world for this invocation */ export const QuickJSEventsFetchedCount = SemanticConvention( - 'quickjs.events.fetched_count' + 'workflow.vm.events.fetched_count' ); /** Number of pages required to fetch all events */ export const QuickJSEventsFetchedPages = SemanticConvention( - 'quickjs.events.fetched_pages' + 'workflow.vm.events.fetched_pages' ); /** Number of pending VM operations captured at suspension */ export const QuickJSPendingOpsCount = SemanticConvention( - 'quickjs.pending_ops_count' + 'workflow.vm.pending_ops_count' ); /** Active trace-correlation mode for this invocation (linked or continuous) */ From fe96bca424883e935ff1f552ae3ea1cd3f8b5abb Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Fri, 31 Jul 2026 15:44:28 -0700 Subject: [PATCH 16/18] QuickJS engine: implement resilient resumeHook (hookInput materialization + resumeId dedup) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1834 made resumeHook() fall back to enqueueing the run with a hookInput payload when the direct hook_received write fails transiently, with the runtime materializing the missing event on delivery. Only the node:vm path implemented it — the QuickJS dispatch returned before the node block, so the resilient payload was silently dropped and the new e2e timed out on every quickjs leg. - runtime.ts threads hookInput into runWorkflowWithQuickJS; the entrypoint materializes the missing hook_received after loading the event log (resumeId-keyed dedup, occurredAt from the resumeId ULID, local eventData substitution for lazy/ref responses, EntityConflict / HookNotFound handling) — mirroring the node block. - processEvents drops duplicate hook_received rows sharing a resumeId (first-in-log wins), matching the node engine's EventsConsumer dedup; the seen-set lives in the VM heap so it is deterministic per replay. Verified against the dev server with WORKFLOW_VM=quickjs: the resilient resume e2e passes and the materialization is observable in the logs; all 27 hook e2e tests green. --- .changeset/quickjs-inline-steps.md | 6 + packages/core/src/runtime.ts | 4 + .../core/src/runtime/quickjs-entrypoint.ts | 109 ++++++++++++++++++ packages/core/src/runtime/quickjs-runtime.ts | 38 ++++++ 4 files changed, 157 insertions(+) create mode 100644 .changeset/quickjs-inline-steps.md diff --git a/.changeset/quickjs-inline-steps.md b/.changeset/quickjs-inline-steps.md new file mode 100644 index 0000000000..5142da21ac --- /dev/null +++ b/.changeset/quickjs-inline-steps.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +QuickJS engine performance: cache compiled WebAssembly modules process-wide, and execute steps inline in a live-VM continuation loop (no queue round-trip per step, cheap events fed before step bodies, delayed wait-continuation dispatch for racing timers). diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index a2d5c86aa1..4acaed151a 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1643,6 +1643,10 @@ export function workflowEntrypoint( runInput, parentSpan: span, maxEventsLimit, + // Resilient resume (see the node block below): the + // QuickJS entrypoint materializes the missing + // hook_received from this payload itself. + hookInput, }); if (quickjsResult?.timeoutSeconds !== undefined) { // Use `reinvoke` rather than returning diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index 3a34514ab9..ab90b3daa7 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -24,10 +24,12 @@ import { import { parseWorkflowName } from '@workflow/utils/parse-name'; import { type Event, + type HookInput, type RunInput, SPEC_VERSION_CURRENT, type WorkflowRun, } from '@workflow/world'; +import { decodeTime } from 'ulid'; import { classifyRunError } from '../classify-error.js'; import { runtimeLogger } from '../logger.js'; import { @@ -478,6 +480,14 @@ export async function runWorkflowWithQuickJS(params: { * with MAX_EVENTS_EXCEEDED. */ maxEventsLimit?: number; + /** + * Resilient-resume payload from the queue message (present only on the + * delivery triggered by a `resumeHook()` whose direct `hook_received` + * write failed transiently). Mirrors the node:vm engine: the missing + * event is materialized from this payload before replay so the workflow + * still receives it. See the same-named field in `QueueMessageSchema`. + */ + hookInput?: HookInput; }): Promise<{ timeoutSeconds?: number } | void> { const { workflowCode, @@ -487,6 +497,7 @@ export async function runWorkflowWithQuickJS(params: { runInput, parentSpan, maxEventsLimit, + hookInput, } = params; const world = await getWorld(); const runId = workflowRun.runId; @@ -581,6 +592,104 @@ export async function runWorkflowWithQuickJS(params: { events = allEvents; } + // --- Resilient resume: materialize missing hook_received --- + // `resumeHook()` writes `hook_received` first and only enqueues a resume + // carrying `hookInput` if that direct write fails with a retryable error + // (transient 429/5xx). In that recovery path, materialize the event here + // so replay can see the payload — the exact mirror of the node:vm + // engine's block in runtime.ts (see there for the full dedup / conflict + // semantics rationale; the replay-side resumeId dedup lives in + // processEvents' hook_received handling for this engine). + if (hookInput) { + const alreadyMaterialized = events.some( + (e) => + e.eventType === 'hook_received' && + e.correlationId === hookInput.hookId && + (e.eventData as { resumeId?: string } | undefined)?.resumeId === + hookInput.resumeId + ); + if (!alreadyMaterialized) { + // The resumeId is a ULID minted in resumeHook() at resume time, so + // its embedded timestamp dates the materialized event to when the + // resume actually happened rather than after the queue round-trip. + let occurredAt: Date | undefined; + try { + occurredAt = new Date(decodeTime(hookInput.resumeId)); + } catch { + occurredAt = undefined; + } + try { + const result = await world.events.create( + runId, + { + eventType: 'hook_received', + specVersion: workflowRun.specVersion ?? SPEC_VERSION_CURRENT, + correlationId: hookInput.hookId, + eventData: { + ...(hookInput.token ? { token: hookInput.token } : {}), + payload: hookInput.payload as never, + resumeId: hookInput.resumeId, + }, + }, + { occurredAt } + ); + if (result.event) { + // The server returns a "lazy" response for hook_received — the + // payload on result.event.eventData may be a RefDescriptor when + // offloaded to blob storage. Substitute the eventData we already + // have locally so the in-memory event matches what + // getWorkflowRunEvents would return after ref hydration. + events.push({ + ...result.event, + eventData: { + ...(hookInput.token ? { token: hookInput.token } : {}), + payload: hookInput.payload as never, + resumeId: hookInput.resumeId, + }, + } as Event); + } + runtimeLogger.warn( + 'Materialized hook_received event from queue payload (resilient resume)', + { + workflowRunId: runId, + hookId: hookInput.hookId, + resumeId: hookInput.resumeId, + } + ); + parentSpan?.setAttributes({ + ...Attribute.HookResilientResumeMaterialized(true), + }); + } catch (err) { + if (EntityConflictError.is(err)) { + // Defensive only today — no World enforces uniqueness on + // hook_received; the duplicate-row case is neutralized by the + // replay-side resumeId dedup. Same contract as the node engine. + runtimeLogger.info( + 'Hook resilient-resume materialization skipped (already exists)', + { + workflowRunId: runId, + hookId: hookInput.hookId, + resumeId: hookInput.resumeId, + } + ); + } else if (HookNotFoundError.is(err)) { + // The hook was disposed between resumeHook() and this delivery — + // no active awaiter to deliver to. Drop the resume. + runtimeLogger.warn( + 'Hook was disposed before resilient resume could materialize — dropping payload', + { + workflowRunId: runId, + hookId: hookInput.hookId, + resumeId: hookInput.resumeId, + } + ); + } else { + throw err; + } + } + } + } + // Event-limit guard: fail a runaway run once its log reaches the // server-supplied ceiling — same enforcement point as the node:vm // engine's replay loop. diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index 464820d47a..46b3a05a35 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -1427,6 +1427,44 @@ async function processEvents( break; } + // Resilient-resume dedup (parity with the node engine's + // EventsConsumer in workflow/hook.ts): two hook_received rows for + // ONE resume attempt share a client-minted `resumeId` (a duplicate + // can be committed when the materialization fallback races a + // delayed direct write — hook_received has no storage uniqueness + // constraint). Deliver only the first-in-log occurrence. The seen + // set lives in the VM heap so it is deterministic per replay and + // survives event re-scans within the invocation. Events without a + // resumeId (older SDKs) are never deduped. + { + const resumeId = (eventData as { resumeId?: unknown } | undefined) + ?.resumeId; + if (typeof resumeId === 'string') { + const resumeIdJs = JSON.stringify(resumeId); + const duplicate = vm.dump( + vm.evalCode( + `(globalThis.__hookSeenResumeIds = globalThis.__hookSeenResumeIds || {})[${resumeIdJs}] === true` + ) + ); + if (duplicate) { + runtimeLogger.debug( + 'QuickJS runtime: duplicate hook_received for the same resume attempt, dropping', + { correlationId: cid, eventId: event.eventId, resumeId } + ); + if (event.eventId) { + vm.evalCode( + `(globalThis.__hookPayloadBuffer.__processedEventIds = globalThis.__hookPayloadBuffer.__processedEventIds || {})[${JSON.stringify(event.eventId)}] = true;` + ).dispose(); + } + markCreated(vm, cidJs); + break; + } + vm.evalCode( + `globalThis.__hookSeenResumeIds[${resumeIdJs}] = true;` + ).dispose(); + } + } + // Abort delivery: hook_received for an AbortController's system // hook flips the registered signal instead of resolving a promise. // The payload is the dehydrated `{ aborted: true, reason }` object. From 4506623812ff4f3abf78a5b058bc6f9babb037af Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 22 Jul 2026 03:19:04 -0700 Subject: [PATCH 17/18] QuickJS engine: inline step execution via live-VM continuation loop + WASM module caching --- packages/core/src/runtime.ts | 1 + .../core/src/runtime/quickjs-entrypoint.ts | 517 ++++++++++++++---- packages/core/src/runtime/quickjs-runtime.ts | 244 ++++++++- .../src/telemetry/semantic-conventions.ts | 5 + 4 files changed, 651 insertions(+), 116 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 4acaed151a..62fd067379 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1643,6 +1643,7 @@ export function workflowEntrypoint( runInput, parentSpan: span, maxEventsLimit, + deliveryAttempt: metadata.attempt, // Resilient resume (see the node block below): the // QuickJS entrypoint materializes the missing // hook_received from this payload itself. diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index ab90b3daa7..4cdd977293 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -25,6 +25,7 @@ import { parseWorkflowName } from '@workflow/utils/parse-name'; import { type Event, type HookInput, + ROOT_RUN_ID_ATTRIBUTE, type RunInput, SPEC_VERSION_CURRENT, type WorkflowRun, @@ -45,6 +46,7 @@ import { import { remapErrorStack, stripInlineSourceMap } from '../source-map.js'; import * as Attribute from '../telemetry/semantic-conventions.js'; import { serializeTraceCarrier } from '../telemetry.js'; +import { getMaxInlineSteps } from './constants.js'; import { getPortLazy } from './get-port-lazy.js'; import { getWorkflowQueueName, queueMessage } from './helpers.js'; import { @@ -54,8 +56,12 @@ import { type PendingOperation, type PendingStep, type PendingWait, - runQuickJSWorkflow, + startQuickJSWorkflow, } from './quickjs-runtime.js'; +import { ReplayBudget } from './replay-budget.js'; +import { executeStep, type StepExecutionResult } from './step-executor.js'; +import { runStepSingleFlight } from './step-single-flight.js'; +import { getWaitContinuationDispatch } from './wait-continuation.js'; import { getWorld } from './world.js'; /** Tiny ms timer using performance.now() — already monotonic on Node. */ @@ -91,18 +97,54 @@ export function isFirstInvocation( ); } +/** + * Queue a step for background execution via the unified workflow queue + * (V2 architecture). The combined handler in runtime.ts dispatches + * messages with `stepId` to executeStep, which works for both VM engines. + * `delaySeconds` supports retry/throttle backoff. + */ +async function queueStepMessage(params: { + world: Awaited>; + runId: string; + workflowRun: WorkflowRun; + step: PendingStep; + delaySeconds?: number; + wfdiag: (checkpoint: string, fields: Record) => void; +}): Promise { + const { world, runId, workflowRun, step, delaySeconds, wfdiag } = params; + const traceCarrier = await serializeTraceCarrier(); + await queueMessage( + world, + getWorkflowQueueName(workflowRun.workflowName), + { + runId, + stepId: step.correlationId, + stepName: step.stepId, + traceCarrier, + requestedAt: new Date(), + }, + { + idempotencyKey: step.correlationId, + ...(delaySeconds && delaySeconds > 0 ? { delaySeconds } : {}), + } + ); + wfdiag('step_queued', { + stepId: step.stepId, + correlationId: step.correlationId, + delaySeconds: delaySeconds ?? 0, + }); +} + /** * Dispatch durable side effects for a set of pending VM operations: * step_created (+ optional queueing), hook_created / hook_received (aborts), * attr_set, hook_disposed, and wait_created events. * - * Used in two modes: - * - suspension (queueSteps: true): normal suspension processing; new steps - * are queued for execution. - * - terminal drain (queueSteps: false): flush leftover side effects when - * the workflow completed or failed — mirrors the node:vm engine's - * drainPendingQueueItems. Steps are created but NOT queued, and the run - * is never requeued. + * Steps are created but never queued here — queueing (or inline + * execution) is the caller's decision. Used both for suspension + * processing (the inline loop) and for the terminal drain (flushing + * leftover side effects when the workflow completed or failed, mirroring + * the node:vm engine's drainPendingQueueItems). */ async function dispatchPendingOps(params: { world: Awaited>; @@ -110,7 +152,6 @@ async function dispatchPendingOps(params: { workflowRun: WorkflowRun; encryptionKey: RunPayloadKeys | undefined; pendingOperations: PendingOperation[]; - queueSteps: boolean; wfdiag: (checkpoint: string, fields: Record) => void; }): Promise<{ createdAttributeEvent: boolean; @@ -345,35 +386,9 @@ async function dispatchPendingOps(params: { throw err; } - // Queue the step execution via the unified workflow queue - // (V2 architecture). The combined handler in runtime.ts - // dispatches messages with `stepId` to executeStep, which - // works for both VM engines — so the QuickJS engine reuses - // the same step execution path as the node:vm engine - // instead of needing a separate step route. Skipped in - // terminal-drain mode (the workflow already finished; the - // event is the durable record, matching the node:vm drain). - if (params.queueSteps) { - const traceCarrier = await serializeTraceCarrier(); - await queueMessage( - world, - getWorkflowQueueName(workflowRun.workflowName), - { - runId, - stepId: step.correlationId, - stepName: step.stepId, - traceCarrier, - requestedAt: new Date(), - }, - { - idempotencyKey: step.correlationId, - } - ); - wfdiag('step_queued', { - stepId: step.stepId, - correlationId: step.correlationId, - }); - } + // NOTE: step queueing is the caller's decision — the inline + // loop executes fresh steps in the live VM and only queues the + // overflow / retry / backstop cases (see queueStepMessage). })() ); } else if (op.type === 'attribute' && !op.hasCreatedEvent) { @@ -488,6 +503,13 @@ export async function runWorkflowWithQuickJS(params: { * still receives it. See the same-named field in `QueueMessageSchema`. */ hookInput?: HookInput; + /** + * Queue delivery attempt of the message driving this invocation (from + * the queue handler's metadata; 1 = first delivery). Redeliveries + * (attempt > 1) trigger backstop queue messages for pending steps that + * an earlier crashed invocation may have orphaned mid-inline-execution. + */ + deliveryAttempt?: number; }): Promise<{ timeoutSeconds?: number } | void> { const { workflowCode, @@ -498,6 +520,7 @@ export async function runWorkflowWithQuickJS(params: { parentSpan, maxEventsLimit, hookInput, + deliveryAttempt, } = params; const world = await getWorld(); const runId = workflowRun.runId; @@ -760,7 +783,7 @@ export async function runWorkflowWithQuickJS(params: { eventCount: events.length, }); - const result = await runQuickJSWorkflow({ + const session = await startQuickJSWorkflow({ // Pass the STRIPPED bundle to the VM so the inline source map // doesn't end up in the QuickJS heap. The original (unstripped) // `workflowCode` is still kept in this host-side scope and is used @@ -773,6 +796,7 @@ export async function runWorkflowWithQuickJS(params: { port, runInput, }); + let result = session.result; runtimeLogger.debug('QuickJS runtime: VM returned', { workflowRunId: runId, @@ -801,6 +825,332 @@ export async function runWorkflowWithQuickJS(params: { failureName: result.failed?.name, }); + // ---- Inline continuation loop ---- + // + // While the workflow is suspended, this loop keeps the VM alive and + // makes as much forward progress as possible within one invocation: + // + // 1. Dispatch durable side effects for the suspension's pending ops + // (step_created / hook_created / attr_set / wait_created / + // hook_received for aborts) and complete elapsed waits. + // 2. Feed all newly recorded events (attr_set, hook_created, elapsed + // wait_completed, terminals written by concurrent invocations, ...) + // into the LIVE VM via session.continueWithEvents — resuming + // execution exactly where it left off, no fresh-VM re-replay. + // Cheap progress is fed BEFORE running step bodies so promise + // chains that are not gated on steps (hook.getConflict(), + // setAttributes(), racing sleeps) advance first and can surface + // additional pending steps for the same inline batch. + // 3. Once no cheap progress remains, execute up to + // getMaxInlineSteps() steps created by THIS invocation inline (no + // queue round-trip), in parallel, with the replay budget paused + // during step bodies — mirroring the node:vm engine's inline + // replay loop. Overflow and retry/throttled steps are queued for + // background execution. A delayed wait-continuation message is + // enqueued for the soonest pending wait first, so racing timers + // fire on time (in a separate invocation) while step bodies block + // this one. + // + // The loop exits when the workflow settles, no forward progress is + // possible in-process, the replay budget is exhausted, or the run is + // gone. + const seenEventIds = new Set(); + for (const e of events) { + if (e.eventId) seenEventIds.add(e.eventId); + } + // Step cids whose step_created THIS invocation wrote — these are safe + // inline candidates (no other invocation can own them; a concurrent + // creator would have lost the events.create race). + const stepsCreatedByUs = new Set(); + // Step cids already executed inline by this invocation. + const executedStepIds = new Set(); + // Steps for which THIS invocation already sent a queue message. + const queuedStepIds = new Set(); + // Aborts THIS invocation already recorded (hook_received written) — + // guards against re-recording when the VM-side flag has not been + // cleared yet within the same iteration. + const recordedAbortIds = new Set(); + // Waits for which THIS invocation already completed/scheduled work. + const completedWaitIds2 = new Set(); + const scheduledWaitContinuations = new Set(); + const maxInlineSteps = getMaxInlineSteps(); + const budget = new ReplayBudget(); + const workflowStartedAt = workflowRun.startedAt + ? +workflowRun.startedAt + : Date.now(); + const rootRunId = + (workflowRun.attributes as Record | undefined)?.[ + ROOT_RUN_ID_ATTRIBUTE + ] ?? runId; + let inlineStepsExecuted = 0; + let runGone = false; + + /** Fetch all events not yet processed by the live VM (log order). */ + const fetchUnseenEvents = async (): Promise => { + const unseen: Event[] = []; + let cursor: string | null = null; + let hasMore = true; + while (hasMore) { + const response = await world.events.list({ + runId, + pagination: { + sortOrder: 'asc', + cursor: cursor ?? undefined, + limit: 1000, + }, + }); + for (const e of response.data) { + if (e.eventId && seenEventIds.has(e.eventId)) continue; + if (e.eventId) seenEventIds.add(e.eventId); + unseen.push(e); + } + if (response.cursor) cursor = response.cursor; + hasMore = response.data.length > 0 && response.cursor != null; + } + return unseen; + }; + + try { + let iteration = 0; + while (result.suspended && !runGone && !budget.isExhausted()) { + iteration++; + const pendingOperations = result.suspended.pendingOperations; + + // 1. Durable side effects for this suspension's pending ops. Record + // which steps we created (before dispatch marks are fed back). + for (const op of pendingOperations) { + if (op.type === 'step' && !op.hasCreatedEvent) { + stepsCreatedByUs.add(op.correlationId); + } + } + const opsToDispatch = pendingOperations.map((op) => + op.type === 'hook' && + (op as PendingHook).abortRequested && + recordedAbortIds.has(op.correlationId) + ? ({ ...op, abortRequested: false } as PendingOperation) + : op + ); + for (const op of pendingOperations) { + if (op.type === 'hook' && (op as PendingHook).abortRequested) { + recordedAbortIds.add(op.correlationId); + } + } + await dispatchPendingOps({ + world, + runId, + workflowRun, + encryptionKey, + pendingOperations: opsToDispatch, + wfdiag, + }); + + // Complete elapsed waits so their wait_completed events are picked + // up by the feed below (instead of a queue re-invocation). + const waitCompletePromises: Promise[] = []; + for (const op of pendingOperations) { + if (op.type !== 'wait') continue; + const wait = op as PendingWait; + if (completedWaitIds2.has(wait.correlationId)) continue; + if (new Date(wait.resumeAt).getTime() - Date.now() > 0) continue; + completedWaitIds2.add(wait.correlationId); + waitCompletePromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'wait_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: wait.correlationId, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } + if (waitCompletePromises.length > 0) { + await Promise.all(waitCompletePromises); + } + + // 2. Cheap progress first: feed newly recorded events into the live + // VM before blocking on step bodies. + { + const newEvents = await fetchUnseenEvents(); + if (newEvents.length > 0) { + result = await session.continueWithEvents(newEvents); + wfdiag('inline_iteration', { + iteration, + phase: 'feed', + fedEvents: newEvents.length, + outcome: result.completed + ? 'completed' + : result.failed + ? 'failed' + : 'suspended', + }); + continue; + } + } + + // 3. No cheap progress left — execute steps inline. + const stepOps = pendingOperations.filter( + (op): op is PendingStep => op.type === 'step' + ); + const ourSteps = stepOps.filter( + (op) => + stepsCreatedByUs.has(op.correlationId) && + !executedStepIds.has(op.correlationId) && + !queuedStepIds.has(op.correlationId) + ); + // Steps created by an EARLIER invocation that are still pending. + // On a redelivery (attempt > 1) the original invocation may have + // crashed mid-inline-execution, orphaning the step (no queue + // message exists on the inline path) — send a backstop message. + // The queue's idempotency key dedups repeats and executeStep + // resolves already-completed steps as 'skipped'. First deliveries + // skip this: the step is most likely executing in a live + // invocation, and a backstop would routinely double-run bodies. + if ((deliveryAttempt ?? 1) > 1) { + for (const step of stepOps) { + if (stepsCreatedByUs.has(step.correlationId)) continue; + if (queuedStepIds.has(step.correlationId)) continue; + queuedStepIds.add(step.correlationId); + await queueStepMessage({ world, runId, workflowRun, step, wfdiag }); + } + } + + const inlineCandidates = + maxInlineSteps <= 0 ? [] : ourSteps.slice(0, maxInlineSteps); + const overflowSteps = ourSteps.slice(inlineCandidates.length); + for (const step of overflowSteps) { + queuedStepIds.add(step.correlationId); + await queueStepMessage({ world, runId, workflowRun, step, wfdiag }); + } + + if (inlineCandidates.length === 0) { + // No in-process progress possible — the run awaits an external + // stimulus (hook payload, queued step, wait timer). + break; + } + + // Racing timers must fire on time while step bodies block this + // invocation: enqueue a delayed continuation for the soonest + // pending wait (a separate invocation writes its wait_completed at + // the right log position — same mechanism as the node:vm engine's + // wait-continuation dispatch). + let soonestWait: { correlationId: string; seconds: number } | undefined; + for (const op of pendingOperations) { + if (op.type !== 'wait') continue; + const wait = op as PendingWait; + if (scheduledWaitContinuations.has(wait.correlationId)) continue; + const resumeMs = new Date(wait.resumeAt).getTime() - Date.now(); + if (resumeMs <= 0) continue; + const seconds = Math.max(1, Math.ceil(resumeMs / 1000)); + if (!soonestWait || seconds < soonestWait.seconds) { + soonestWait = { correlationId: wait.correlationId, seconds }; + } + } + if (soonestWait) { + scheduledWaitContinuations.add(soonestWait.correlationId); + await queueMessage( + world, + getWorkflowQueueName(workflowRun.workflowName), + { + runId, + traceCarrier: await serializeTraceCarrier(), + requestedAt: new Date(), + }, + getWaitContinuationDispatch( + soonestWait.seconds, + soonestWait.correlationId + ) + ); + wfdiag('wait_continuation_scheduled', { + correlationId: soonestWait.correlationId, + delaySeconds: soonestWait.seconds, + }); + } + + // Execute the inline batch in parallel. The replay budget is + // paused while step bodies run — step duration is bounded by the + // platform function duration, not the replay timeout. + budget.pause(); + let outcomes: StepExecutionResult[]; + try { + outcomes = await Promise.all( + inlineCandidates.map((step) => + runStepSingleFlight(runId, step.correlationId, () => + executeStep({ + world, + workflowRunId: runId, + workflowDeploymentId: workflowRun.deploymentId, + workflowName: workflowRun.workflowName, + workflowStartedAt, + rootRunId, + stepId: step.correlationId, + stepName: step.stepId, + encryptionKey, + runSpecVersion: workflowRun.specVersion, + }) + ) + ) + ); + } finally { + budget.resume(); + } + inlineStepsExecuted += inlineCandidates.length; + + for (let i = 0; i < inlineCandidates.length; i++) { + const step = inlineCandidates[i]; + const outcome = outcomes[i]; + executedStepIds.add(step.correlationId); + if (outcome.type === 'retry' || outcome.type === 'throttled') { + // Hand the step to the queue with the requested backoff — + // background delivery drives the retry from here. + queuedStepIds.add(step.correlationId); + await queueStepMessage({ + world, + runId, + workflowRun, + step, + delaySeconds: outcome.timeoutSeconds, + wfdiag, + }); + } else if (outcome.type === 'gone') { + runGone = true; + } + } + wfdiag('inline_steps_executed', { + iteration, + count: inlineCandidates.length, + outcomes: outcomes.map((o) => o.type), + }); + + // Feed the inline batch's terminal events into the live VM. + const newEvents = await fetchUnseenEvents(); + if (newEvents.length === 0) break; + result = await session.continueWithEvents(newEvents); + + wfdiag('inline_iteration', { + iteration, + phase: 'steps', + fedEvents: newEvents.length, + outcome: result.completed + ? 'completed' + : result.failed + ? 'failed' + : 'suspended', + budgetExhausted: budget.isExhausted(), + }); + } + } finally { + session.dispose(); + } + + parentSpan?.setAttributes({ + ...Attribute.QuickJSInlineSteps(inlineStepsExecuted), + }); + if (result.completed) { // Workflow completed runtimeLogger.info('QuickJS runtime: workflow completed', { @@ -823,7 +1173,6 @@ export async function runWorkflowWithQuickJS(params: { workflowRun, encryptionKey, pendingOperations: result.completed.drainOperations, - queueSteps: false, wfdiag, }); } catch (err) { @@ -868,24 +1217,21 @@ export async function runWorkflowWithQuickJS(params: { throw err; } } else if (result.suspended) { - // Workflow suspended + // Workflow still suspended after the inline loop. All durable side + // effects for the final suspension state were already dispatched by + // the loop; what remains is deciding how the run gets re-invoked. const { pendingOperations } = result.suspended; runtimeLogger.info('QuickJS runtime: workflow suspended', { workflowRunId: runId, + inlineStepsExecuted, pendingSteps: pendingOperations.filter((p) => p.type === 'step').length, pendingWaits: pendingOperations.filter((p) => p.type === 'wait').length, pendingOps: pendingOperations.map((p) => ({ type: p.type, correlationId: p.correlationId, hasCreatedEvent: p.hasCreatedEvent, - ...(p.type === 'step' - ? { - stepId: (p as PendingStep).stepId, - inputType: typeof (p as PendingStep).input, - inputIsUint8Array: (p as PendingStep).input instanceof Uint8Array, - } - : {}), + ...(p.type === 'step' ? { stepId: (p as PendingStep).stepId } : {}), })), }); @@ -894,53 +1240,35 @@ export async function runWorkflowWithQuickJS(params: { ...Attribute.QuickJSPendingOpsCount(pendingOperations.length), }); - // Build per-pending-op promises so events.create + queueMessage - // calls fan out in parallel rather than serially. This mirrors - // the node:vm engine's `Promise.all(ops)` pattern in - // suspension-handler.ts and significantly reduces wall-clock time - // on cloud worlds (e.g. Vercel) where each storage call is a - // network round-trip. - let minTimeoutSeconds: number | undefined; - const { createdAttributeEvent, createdGetConflictHook } = - await dispatchPendingOps({ - world, - runId, - workflowRun, - encryptionKey, - pendingOperations, - queueSteps: true, - wfdiag, + if (runGone) { + // The run no longer exists (expired / deleted) — nothing to drive. + wfdiag('exit_suspended', { action: 'run_gone' }); + return; + } + + if (budget.isExhausted()) { + // The loop stopped on the replay budget with progress still + // possible — continue in a fresh invocation. + wfdiag('exit_suspended', { + action: 'budget_exhausted_requeue', + timeoutSeconds: 0, }); + return { timeoutSeconds: 0 }; + } - // Handle pending waits — both newly created and still-pending from - // earlier invocations. For each wait, either create a wait_completed - // event (if elapsed) or schedule a timeout for re-queuing. - let needsRequeue = false; - const waitCompletePromises: Promise[] = []; + // Schedule a timer for the earliest pending wait. A wait that elapsed + // in the window since the loop's last check requeues immediately (its + // wait_completed is written by the next invocation's elapsed-wait + // sweep). + let minTimeoutSeconds: number | undefined; + let hasElapsedWait = false; for (const op of pendingOperations) { if (op.type !== 'wait') continue; const wait = op as PendingWait; const resumeMs = new Date(wait.resumeAt).getTime() - Date.now(); - if (resumeMs <= 0) { - // Wait has elapsed — create wait_completed and re-queue. - waitCompletePromises.push( - (async () => { - try { - await world.events.create(runId, { - eventType: 'wait_completed', - specVersion: SPEC_VERSION_CURRENT, - correlationId: wait.correlationId, - }); - needsRequeue = true; - } catch (err) { - if (EntityConflictError.is(err)) return; - throw err; - } - })() - ); + hasElapsedWait = true; } else { - // Wait hasn't elapsed yet — schedule a timeout const timeoutSeconds = Math.max(1, Math.ceil(resumeMs / 1000)); if ( minTimeoutSeconds === undefined || @@ -950,20 +1278,10 @@ export async function runWorkflowWithQuickJS(params: { } } } - if (waitCompletePromises.length > 0) { - await Promise.all(waitCompletePromises); - } - if (needsRequeue || createdAttributeEvent || createdGetConflictHook) { - // An elapsed wait was completed, a new attr_set event was written, - // or a getConflict()-awaited hook was created — re-queue immediately - // so the next invocation can process the new event. + if (hasElapsedWait) { wfdiag('exit_suspended', { - action: needsRequeue - ? 'wait_elapsed_requeue' - : createdAttributeEvent - ? 'attr_set_requeue' - : 'get_conflict_requeue', + action: 'wait_elapsed_requeue', timeoutSeconds: 0, }); return { timeoutSeconds: 0 }; @@ -1029,7 +1347,6 @@ export async function runWorkflowWithQuickJS(params: { workflowRun, encryptionKey, pendingOperations: result.failed.drainOperations, - queueSteps: false, wfdiag, }); } catch (err) { diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index 46b3a05a35..9371aff338 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -29,7 +29,12 @@ import type { Event, RunInput, WorkflowRun } from '@workflow/world'; import * as nanoid from 'nanoid'; -import { JSException, QuickJS, type WasiOptions } from 'quickjs-wasi'; +import { + type ExtensionDescriptor, + JSException, + QuickJS, + type WasiOptions, +} from 'quickjs-wasi'; import seedrandom from 'seedrandom'; import { runtimeLogger } from '../logger.js'; import { decompress } from '../serialization/compression.js'; @@ -906,7 +911,65 @@ globalThis[Symbol.for("WORKFLOW_GET_STREAM_ID")] = function(namespace) { * new runs can restore from it instead of paying VM creation + eval cost * (`QuickJS.restore` accepts the same wasi override). */ -async function initWorkflowVM(getNowMs: () => number): Promise { +/** + * Loosely-typed accessor for the `WebAssembly` global. The package + * tsconfig's `lib: ["es2022"]` does not include the DOM lib where the + * `WebAssembly` namespace types live; the runtime global is available on + * every WASM-capable platform this engine targets. + */ +const WebAssemblyGlobal = (globalThis as any).WebAssembly as { + compile(bytes: Uint8Array): Promise; +}; + +type CompiledExtension = Omit & { + wasm: ExtensionDescriptor['wasm']; +}; + +/** + * Process-wide cache of the compiled `WebAssembly.Module`s for the main + * QuickJS runtime and its native extensions. `WebAssembly.compile` of the + * ~600 KB runtime binary is the most expensive part of VM creation and is + * pure (no per-VM state — instantiation binds the per-VM memory), so it + * only needs to happen once per process. The promise is cached (not the + * result) so concurrent first invocations share a single compilation. + */ +let compiledAssetsPromise: + | Promise<{ + wasm: object; + extensions: CompiledExtension[]; + }> + | undefined; + +function getCompiledAssets() { + if (!compiledAssetsPromise) { + compiledAssetsPromise = (async () => { + const [wasm, ...extensionModules] = await Promise.all([ + WebAssemblyGlobal.compile(quickjsWasm), + ...quickjsExtensions.map((ext) => + WebAssemblyGlobal.compile(ext.wasm as Uint8Array) + ), + ]); + return { + wasm, + extensions: quickjsExtensions.map((ext, i) => ({ + ...ext, + wasm: extensionModules[i] as ExtensionDescriptor['wasm'], + })), + }; + })(); + // On failure, clear the cache so a later invocation can retry rather + // than being stuck with a rejected promise forever. + compiledAssetsPromise.catch(() => { + compiledAssetsPromise = undefined; + }); + } + return compiledAssetsPromise; +} + +async function initWorkflowVM( + getNowMs: () => number, + interruptBudget: InterruptBudget +): Promise { // Deterministic replay clock: Date.now() / new Date() inside the VM // read the host-controlled clock instead of wall time. Replay // re-executes the workflow from the top on every invocation, so the @@ -920,11 +983,12 @@ async function initWorkflowVM(getNowMs: () => number): Promise { }, }); + const assets = await getCompiledAssets(); const vm = await QuickJS.create({ - wasm: quickjsWasm, + wasm: assets.wasm as never, memoryLimit: 256 * 1024 * 1024, - interruptHandler: createInterruptHandler(), - extensions: quickjsExtensions, + interruptHandler: createInterruptHandler(interruptBudget), + extensions: assets.extensions, wasi, }); @@ -937,9 +1001,43 @@ async function initWorkflowVM(getNowMs: () => number): Promise { return vm; } +/** + * A live QuickJS workflow invocation. When the initial `result` is + * `suspended`, the VM is kept alive so the caller can feed newly recorded + * events (e.g. terminal events of inline-executed steps) into the SAME VM + * via `continueWithEvents` — resuming execution exactly where it left off + * without a fresh-VM re-replay. Terminal results dispose the VM + * automatically; `dispose()` must be called when abandoning a suspended + * session (idempotent). + */ +export interface QuickJSWorkflowSession { + result: QuickJSRuntimeResult; + /** + * Process newly recorded events in the live VM and re-evaluate the + * workflow state. Only valid while the last result was `suspended`. + * Resets the VM's interrupt budget for the new execution burst. + */ + continueWithEvents(newEvents: Event[]): Promise; + /** Dispose the VM if it is still alive. Safe to call multiple times. */ + dispose(): void; +} + +/** + * Run a workflow invocation to its first settled state and dispose the + * VM. Convenience wrapper over {@link startQuickJSWorkflow} for callers + * (and tests) that don't use live-VM continuation. + */ export async function runQuickJSWorkflow( options: QuickJSRuntimeOptions ): Promise { + const session = await startQuickJSWorkflow(options); + session.dispose(); + return session.result; +} + +export async function startQuickJSWorkflow( + options: QuickJSRuntimeOptions +): Promise { const { workflowCode, workflowId, workflowRun, events } = options; const startedAt = workflowRun.startedAt ? +workflowRun.startedAt : Date.now(); @@ -987,7 +1085,8 @@ export async function runQuickJSWorkflow( }; // ---- Phase 1: static initialization ---- - const vm = await initWorkflowVM(() => vmNowMs); + const interruptBudget: InterruptBudget = { start: Date.now() }; + const vm = await initWorkflowVM(() => vmNowMs, interruptBudget); // Any throw between here and the terminal paths (which dispose the VM // inside checkWorkflowState / extractError before RETURNING) would leak @@ -1006,7 +1105,7 @@ export async function runQuickJSWorkflow( } // ---- Phase 2: per-run initialization ---- - async function runWorkflowInVM(): Promise { + async function runWorkflowInVM(): Promise { // Seeded Math.random { using randomFn = vm.newFunction('random', () => vm.newNumber(rng())); @@ -1057,7 +1156,9 @@ export async function runQuickJSWorkflow( try { vm.evalCode(workflowCode, workflowId || 'workflow.js').dispose(); } catch (err) { - return extractError(vm, err, 'Workflow evaluation failed'); + return makeSettledSession( + extractError(vm, err, 'Workflow evaluation failed') + ); } // Extract workflow arguments. Prefer the run_created event; fall back @@ -1164,7 +1265,9 @@ export async function runQuickJSWorkflow( ); `).dispose(); } catch (err) { - return extractError(vm, err, 'Failed to start workflow'); + return makeSettledSession( + extractError(vm, err, 'Failed to start workflow') + ); } // Process events and drain jobs in a loop. Events may resolve promises @@ -1203,10 +1306,93 @@ export async function runQuickJSWorkflow( } // ---- Check result ---- - return checkWorkflowState(vm); + return makeLiveSession( + vm, + interruptBudget, + advanceClock, + options.encryptionKey + ); } } +/** Session wrapper for a result whose VM is already settled/disposed. */ +function makeSettledSession( + result: QuickJSRuntimeResult +): QuickJSWorkflowSession { + return { + result, + continueWithEvents: () => { + throw new Error( + 'QuickJS workflow session is settled — continueWithEvents is only valid while suspended' + ); + }, + dispose: () => {}, + }; +} + +/** + * Evaluate the VM's state and wrap it in a live session. While suspended, + * the VM stays alive so `continueWithEvents` can resume it in place; + * terminal states dispose the VM immediately (inside checkWorkflowState). + */ +function makeLiveSession( + vm: QuickJS, + interruptBudget: InterruptBudget, + advanceClock: (ms: number) => void, + encryptionKey?: DecryptionKey +): QuickJSWorkflowSession { + const result = checkWorkflowState(vm, { keepAliveOnSuspend: true }); + let alive = !!result.suspended; + + const session: QuickJSWorkflowSession = { + result, + async continueWithEvents( + newEvents: Event[] + ): Promise { + if (!alive) { + throw new Error( + 'QuickJS workflow session is not alive — continueWithEvents is only valid while suspended' + ); + } + // Fresh execution burst — the interrupt budget bounds VM compute, + // not wall time spent waiting on inline steps between bursts. + interruptBudget.start = Date.now(); + + let maxIterations = 100; + let madeProgress: boolean; + do { + madeProgress = await processEvents( + vm, + newEvents, + advanceClock, + encryptionKey + ); + let batch: number; + do { + batch = vm.executePendingJobs(); + if (batch > 0) madeProgress = true; + } while (batch > 0); + } while (madeProgress && --maxIterations > 0); + + const next = checkWorkflowState(vm, { keepAliveOnSuspend: true }); + if (!next.suspended) alive = false; + session.result = next; + return next; + }, + dispose(): void { + if (alive) { + alive = false; + try { + vm.dispose(); + } catch { + // Already disposed — ignore. + } + } + }, + }; + return session; +} + // ---- Event Processing ---- async function processEvents( @@ -1495,6 +1681,17 @@ async function processEvents( `globalThis.__abortSignals[${cidJs}]._setAborted(undefined);` ).dispose(); } + // The abort is durably recorded — clear the pending op's + // abortRequested marker so the host doesn't re-record it (the + // workflow's own abort() call can set the flag before this + // event is processed when it happens later in replay order, + // and hook_received events are not unique per correlationId). + vm.evalCode( + `(function(){` + + `var p=globalThis.__pending.find(function(q){return q.correlationId===${JSON.stringify(cid)}&&q.type==="hook";});` + + `if(p)p.abortRequested=false;` + + `})()` + ).dispose(); if (event.eventId) { vm.evalCode( `(globalThis.__hookPayloadBuffer.__processedEventIds = globalThis.__hookPayloadBuffer.__processedEventIds || {})[${JSON.stringify(event.eventId)}] = true;` @@ -1779,7 +1976,10 @@ function collectDrainOperations(vm: QuickJS): PendingOperation[] { return vm.dump(h) as PendingOperation[]; } -function checkWorkflowState(vm: QuickJS): QuickJSRuntimeResult { +function checkWorkflowState( + vm: QuickJS, + opts: { keepAliveOnSuspend?: boolean } = {} +): QuickJSRuntimeResult { // Check completed — __workflowResult is a format-prefixed Uint8Array { using h = vm.evalCode('globalThis.__workflowResult'); @@ -1848,7 +2048,7 @@ function checkWorkflowState(vm: QuickJS): QuickJSRuntimeResult { `globalThis.__pending.filter(function(p){return!!globalThis.__resolvers[p.correlationId] || !p.hasCreatedEvent || p.abortRequested;})` ); const pendingOps = vm.dump(pendingH) as PendingOperation[]; - vm.dispose(); + if (!opts.keepAliveOnSuspend) vm.dispose(); return { suspended: { @@ -1891,8 +2091,20 @@ function extractError( }; } -function createInterruptHandler(): () => boolean { - const start = Date.now(); - const timeout = 30_000; - return () => Date.now() - start > timeout; +/** + * Mutable interrupt budget for a VM. QuickJS polls the interrupt handler + * during JS execution; when it returns true, execution aborts. The budget + * bounds a single host->VM execution burst (bundle eval + event + * processing), not total VM lifetime — the inline-step loop keeps a VM + * alive across step executions that can legitimately take minutes, so the + * host resets the budget before each re-entry (see resetBudget calls). + */ +interface InterruptBudget { + start: number; +} + +const INTERRUPT_BUDGET_MS = 30_000; + +function createInterruptHandler(budget: InterruptBudget): () => boolean { + return () => Date.now() - budget.start > INTERRUPT_BUDGET_MS; } diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index f13c6a8ee8..7456380768 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -122,6 +122,11 @@ export const QuickJSPendingOpsCount = SemanticConvention( 'workflow.vm.pending_ops_count' ); +/** Number of steps executed inline (live-VM continuation) this invocation */ +export const QuickJSInlineSteps = SemanticConvention( + 'quickjs.inline_steps' +); + /** Active trace-correlation mode for this invocation (linked or continuous) */ export const WorkflowTraceMode = SemanticConvention<'linked' | 'continuous'>( 'workflow.trace.mode' From b232778a1294dc2b6ca83acd152a450dead9d199 Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Fri, 31 Jul 2026 14:12:41 -0700 Subject: [PATCH 18/18] Address review: exclusive inline step claims, self-write requeue, in-loop event ceiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Inline steps now claim via a lazy step_started carrying the input (step_created deferred, atomic create-claim in the world), with ownerMessageId stamped and authoritativeAttempt=1 — a concurrent invocation racing on the same fresh step loses with EntityConflictError and skips instead of both bare-starting the step and double-running the body. This also removes the stepsCreatedByUs set, whose 'created by us' invariant didn't survive the swallowed create-race conflict; redelivery backstops now key on hasCreatedEvent. - dispatchPendingOps' createdAttributeEvent/createdGetConflictHook signals are consumed again: when the loop exits suspended without ever reading back a self-written attr_set / getConflict hook_created (eventually-consistent listing lag), the entrypoint requeues immediately instead of parking the run awaiting_external with its unblocking event already written. - The server-supplied event ceiling is re-checked at the top of every continuation-loop turn (seenEventIds.size), so a single invocation fanning out inline can no longer grow the log arbitrarily past the operator's limit. The quickjs dispatch in runtime.ts converts MaxEventsExceededError into run_failed / MAX_EVENTS_EXCEEDED — the guard's throw previously nacked forever, parking runaway runs in 'running'. - Documented the deliberate decision that the platform function timeout is the only bound on inline chaining (budget parked per batch), matching the node engine. --- packages/core/src/runtime.ts | 57 ++++-- .../core/src/runtime/quickjs-entrypoint.ts | 189 ++++++++++++++---- 2 files changed, 188 insertions(+), 58 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 62fd067379..f0279e7c8c 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1635,20 +1635,49 @@ export function workflowEntrypoint( const { runWorkflowWithQuickJS } = await import( './runtime/quickjs-entrypoint.js' ); - const quickjsResult = await runWorkflowWithQuickJS({ - workflowCode, - workflowName, - workflowRun, - preloadedEvents, - runInput, - parentSpan: span, - maxEventsLimit, - deliveryAttempt: metadata.attempt, - // Resilient resume (see the node block below): the - // QuickJS entrypoint materializes the missing - // hook_received from this payload itself. - hookInput, - }); + let quickjsResult: Awaited< + ReturnType + >; + try { + quickjsResult = await runWorkflowWithQuickJS({ + workflowCode, + workflowName, + workflowRun, + preloadedEvents, + runInput, + parentSpan: span, + maxEventsLimit, + deliveryAttempt: metadata.attempt, + ownerMessageId: metadata.messageId, + // Resilient resume (see the node block below): the + // QuickJS entrypoint materializes the missing + // hook_received from this payload itself. + hookInput, + }); + } catch (err) { + // The event-ceiling guard (initial and per-loop-turn) + // throws MaxEventsExceededError — terminal by + // definition: redelivering would re-read the same + // oversized log and throw again forever, leaving the + // run parked in `running`. Record run_failed / + // MAX_EVENTS_EXCEEDED and consume the message. All + // other errors keep propagating so the queue's + // redelivery semantics drive the retry. + if (MaxEventsExceededError.is(err)) { + await recordFatalRunError({ + world, + workflowRun, + runId, + requestId, + err, + errorCode: RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED, + logMessage: + 'Workflow run exceeded the configured max events limit', + }); + return; + } + throw err; + } if (quickjsResult?.timeoutSeconds !== undefined) { // Use `reinvoke` rather than returning // `{ timeoutSeconds }` directly: under turbo the diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index 4cdd977293..480b927f76 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -152,6 +152,13 @@ async function dispatchPendingOps(params: { workflowRun: WorkflowRun; encryptionKey: RunPayloadKeys | undefined; pendingOperations: PendingOperation[]; + /** + * Step cids whose `step_created` must NOT be written here: the inline + * loop claims these atomically via a lazy `step_started` (carrying the + * input), so a concurrent claimant loses with EntityConflictError + * instead of both invocations bare-starting the same step. + */ + skipStepCreation?: Set; wfdiag: (checkpoint: string, fields: Record) => void; }): Promise<{ createdAttributeEvent: boolean; @@ -159,6 +166,7 @@ async function dispatchPendingOps(params: { }> { const { world, runId, workflowRun, encryptionKey, pendingOperations } = params; + const skipStepCreation = params.skipStepCreation; const wfdiag = params.wfdiag; // Set when a hook with a parked getConflict() awaiter had its // hook_created written this invocation. The workflow must be re-invoked @@ -360,7 +368,11 @@ async function dispatchPendingOps(params: { } for (const op of pendingOperations) { - if (op.type === 'step' && !op.hasCreatedEvent) { + if ( + op.type === 'step' && + !op.hasCreatedEvent && + !skipStepCreation?.has(op.correlationId) + ) { const step = op as PendingStep; opsPromises.push( (async () => { @@ -510,6 +522,12 @@ export async function runWorkflowWithQuickJS(params: { * an earlier crashed invocation may have orphaned mid-inline-execution. */ deliveryAttempt?: number; + /** + * Queue message ID of the delivery driving this invocation, stamped as + * `ownerMessageId` on inline lazy step claims so wake replays defer to + * the in-flight body instead of requeueing the step. + */ + ownerMessageId?: string; }): Promise<{ timeoutSeconds?: number } | void> { const { workflowCode, @@ -521,6 +539,7 @@ export async function runWorkflowWithQuickJS(params: { maxEventsLimit, hookInput, deliveryAttempt, + ownerMessageId, } = params; const world = await getWorld(); const runId = workflowRun.runId; @@ -858,10 +877,6 @@ export async function runWorkflowWithQuickJS(params: { for (const e of events) { if (e.eventId) seenEventIds.add(e.eventId); } - // Step cids whose step_created THIS invocation wrote — these are safe - // inline candidates (no other invocation can own them; a concurrent - // creator would have lost the events.create race). - const stepsCreatedByUs = new Set(); // Step cids already executed inline by this invocation. const executedStepIds = new Set(); // Steps for which THIS invocation already sent a queue message. @@ -884,6 +899,14 @@ export async function runWorkflowWithQuickJS(params: { ] ?? runId; let inlineStepsExecuted = 0; let runGone = false; + // Set when this invocation wrote an event the workflow must consume to + // make progress (attr_set, getConflict-awaited hook_created) and the + // loop has not yet read it back — eventually-consistent listings can + // return 0 new events right after a write. If it is still set when the + // loop exits suspended, the entrypoint requeues immediately instead of + // exiting awaiting_external with the unblocking event already written + // and nothing scheduled to read it. + let pendingRequeueSignal = false; /** Fetch all events not yet processed by the live VM (log order). */ const fetchUnseenEvents = async (): Promise => { @@ -914,15 +937,40 @@ export async function runWorkflowWithQuickJS(params: { let iteration = 0; while (result.suspended && !runGone && !budget.isExhausted()) { iteration++; + // Re-check the event ceiling every turn: the loop appends events on + // each continueWithEvents, so a single invocation can otherwise grow + // the log arbitrarily far past the operator's limit (the node engine + // re-checks per replay for the same reason). `seenEventIds` counts + // every event this invocation has observed — initial log + all + // feeds. + if (maxEventsLimit !== undefined && seenEventIds.size >= maxEventsLimit) { + throw new MaxEventsExceededError(seenEventIds.size, maxEventsLimit); + } const pendingOperations = result.suspended.pendingOperations; - // 1. Durable side effects for this suspension's pending ops. Record - // which steps we created (before dispatch marks are fed back). - for (const op of pendingOperations) { - if (op.type === 'step' && !op.hasCreatedEvent) { - stepsCreatedByUs.add(op.correlationId); - } - } + // Select this turn's inline candidates BEFORE dispatch: fresh steps + // (no step_created yet) that this invocation hasn't already handled. + // Their step_created is deliberately NOT written by dispatch — the + // inline claim below is a lazy step_started carrying the input, + // which the world applies as an atomic create-claim. A concurrent + // invocation racing on the same fresh step loses that claim with + // EntityConflictError and skips, so step bodies cannot double-run + // (previously both invocations bare-started the step after one lost + // the swallowed step_created race). + const freshSteps = pendingOperations.filter( + (op): op is PendingStep => + op.type === 'step' && + !op.hasCreatedEvent && + !executedStepIds.has(op.correlationId) && + !queuedStepIds.has(op.correlationId) + ); + const inlineCandidates = + maxInlineSteps <= 0 ? [] : freshSteps.slice(0, maxInlineSteps); + const inlineClaimCids = new Set( + inlineCandidates.map((step) => step.correlationId) + ); + + // 1. Durable side effects for this suspension's pending ops. const opsToDispatch = pendingOperations.map((op) => op.type === 'hook' && (op as PendingHook).abortRequested && @@ -935,14 +983,21 @@ export async function runWorkflowWithQuickJS(params: { recordedAbortIds.add(op.correlationId); } } - await dispatchPendingOps({ + const dispatched = await dispatchPendingOps({ world, runId, workflowRun, encryptionKey, pendingOperations: opsToDispatch, + skipStepCreation: inlineClaimCids, wfdiag, }); + if ( + dispatched.createdAttributeEvent || + dispatched.createdGetConflictHook + ) { + pendingRequeueSignal = true; + } // Complete elapsed waits so their wait_completed events are picked // up by the feed below (instead of a queue re-invocation). @@ -977,6 +1032,10 @@ export async function runWorkflowWithQuickJS(params: { { const newEvents = await fetchUnseenEvents(); if (newEvents.length > 0) { + // The listing caught up with this invocation's writes — any + // attr_set / getConflict hook_created has been (or is being) + // consumed by the live VM, so no external requeue is needed. + pendingRequeueSignal = false; result = await session.continueWithEvents(newEvents); wfdiag('inline_iteration', { iteration, @@ -996,32 +1055,29 @@ export async function runWorkflowWithQuickJS(params: { const stepOps = pendingOperations.filter( (op): op is PendingStep => op.type === 'step' ); - const ourSteps = stepOps.filter( - (op) => - stepsCreatedByUs.has(op.correlationId) && - !executedStepIds.has(op.correlationId) && - !queuedStepIds.has(op.correlationId) - ); - // Steps created by an EARLIER invocation that are still pending. - // On a redelivery (attempt > 1) the original invocation may have - // crashed mid-inline-execution, orphaning the step (no queue - // message exists on the inline path) — send a backstop message. - // The queue's idempotency key dedups repeats and executeStep - // resolves already-completed steps as 'skipped'. First deliveries - // skip this: the step is most likely executing in a live - // invocation, and a backstop would routinely double-run bodies. + // Steps created by an EARLIER invocation that are still pending + // (their step_created came back through the event feed). On a + // redelivery (attempt > 1) the original invocation may have crashed + // mid-inline-execution, orphaning the step (no queue message exists + // on the inline path) — send a backstop message. The queue's + // idempotency key dedups repeats and executeStep resolves + // already-completed steps as 'skipped'. First deliveries skip this: + // the step is most likely executing in a live invocation, and a + // backstop would routinely double-run bodies. if ((deliveryAttempt ?? 1) > 1) { for (const step of stepOps) { - if (stepsCreatedByUs.has(step.correlationId)) continue; + if (!step.hasCreatedEvent) continue; + if (executedStepIds.has(step.correlationId)) continue; if (queuedStepIds.has(step.correlationId)) continue; queuedStepIds.add(step.correlationId); await queueStepMessage({ world, runId, workflowRun, step, wfdiag }); } } - const inlineCandidates = - maxInlineSteps <= 0 ? [] : ourSteps.slice(0, maxInlineSteps); - const overflowSteps = ourSteps.slice(inlineCandidates.length); + // Steps beyond the inline cap: their step_created was written by + // dispatch above (they are not in the lazy-claim set), so hand them + // to the queue. + const overflowSteps = freshSteps.slice(inlineCandidates.length); for (const step of overflowSteps) { queuedStepIds.add(step.correlationId); await queueStepMessage({ world, runId, workflowRun, step, wfdiag }); @@ -1073,25 +1129,49 @@ export async function runWorkflowWithQuickJS(params: { // Execute the inline batch in parallel. The replay budget is // paused while step bodies run — step duration is bounded by the - // platform function duration, not the replay timeout. + // platform function duration, not the replay timeout. NOTE (by + // design): with the budget parked per batch, the only bound on how + // many inline steps one invocation can chain is the platform's + // function timeout — the SDK deliberately imposes no cap of its + // own, matching the node:vm engine, where a long sequential + // workflow likewise runs step-by-step until the platform reclaims + // the invocation and a redelivery resumes from the log. budget.pause(); let outcomes: StepExecutionResult[]; try { outcomes = await Promise.all( inlineCandidates.map((step) => runStepSingleFlight(runId, step.correlationId, () => - executeStep({ - world, - workflowRunId: runId, - workflowDeploymentId: workflowRun.deploymentId, - workflowName: workflowRun.workflowName, - workflowStartedAt, - rootRunId, - stepId: step.correlationId, - stepName: step.stepId, - encryptionKey, - runSpecVersion: workflowRun.specVersion, - }) + (async () => + executeStep({ + world, + workflowRunId: runId, + workflowDeploymentId: workflowRun.deploymentId, + workflowName: workflowRun.workflowName, + workflowStartedAt, + rootRunId, + stepId: step.correlationId, + stepName: step.stepId, + encryptionKey, + runSpecVersion: workflowRun.specVersion, + // Lazy inline claim: step_created is deferred (dispatch + // skipped it) and this step_started carries the input, + // so the world creates the step atomically — + // exactly-one-owner. A concurrent claimant gets + // EntityConflictError → { type: 'skipped' } and never + // runs the body. Mirrors the node engine's inline path. + lazyStepInput: await encryptSerializedData( + step.input, + encryptionKey + ), + // Ownership stamp: wake replays see the body as in + // flight in this invocation and arm a delayed backstop + // instead of immediately requeueing the step. + ownerMessageId, + // A lazy step is brand-new by construction — first + // attempt. + authoritativeAttempt: 1, + }))() ) ) ); @@ -1119,6 +1199,10 @@ export async function runWorkflowWithQuickJS(params: { } else if (outcome.type === 'gone') { runGone = true; } + // 'skipped': a concurrent invocation won the lazy create-claim and + // owns the body. Marked executed above so this invocation never + // re-claims it; the winner's terminal events arrive via the feed + // (or drive a separate invocation). } wfdiag('inline_steps_executed', { iteration, @@ -1287,6 +1371,23 @@ export async function runWorkflowWithQuickJS(params: { return { timeoutSeconds: 0 }; } + if (pendingRequeueSignal) { + // This invocation wrote an event the workflow needs to consume + // (attr_set / getConflict-awaited hook_created) but the + // eventually-consistent listing never returned it before the loop + // exited. Without a requeue the run would park awaiting_external + // with its unblocking event already durably written and no future + // invocation coming — requeue immediately so a fresh read picks it + // up. In the common case the loop's own feed observes the write and + // clears this flag, so this only fires when the read actually + // lagged. + wfdiag('exit_suspended', { + action: 'unread_self_write_requeue', + timeoutSeconds: 0, + }); + return { timeoutSeconds: 0 }; + } + if (minTimeoutSeconds !== undefined) { wfdiag('exit_suspended', { action: 'schedule_wait_timeout',