From 096ea5bd179762b38c9ee4f9e1918fff7bfdc43e Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 23 Jun 2026 16:06:29 +0200 Subject: [PATCH 01/46] feat: per-run workerd sandbox that runs MCP Code Mode programs --- .actor/actor.json | 38 ++++++ .dockerignore | 4 + .gitignore | 4 + Dockerfile | 31 +++++ README.md | 75 ++++++++++++ package.json | 13 ++ pnpm-lock.yaml | 75 ++++++++++++ worker/config.capnp | 39 ++++++ worker/entrypoint.sh | 54 ++++++++ worker/guard.js | 45 +++++++ worker/runner.js | 286 +++++++++++++++++++++++++++++++++++++++++++ 11 files changed, 664 insertions(+) create mode 100644 .actor/actor.json create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 package.json create mode 100644 pnpm-lock.yaml create mode 100644 worker/config.capnp create mode 100755 worker/entrypoint.sh create mode 100644 worker/guard.js create mode 100644 worker/runner.js diff --git a/.actor/actor.json b/.actor/actor.json new file mode 100644 index 0000000..3b0434e --- /dev/null +++ b/.actor/actor.json @@ -0,0 +1,38 @@ +{ + "actorSpecification": 1, + "name": "code-runtime", + "title": "Code Runtime", + "description": "Runs an LLM-submitted TypeScript/JavaScript program in a sandboxed workerd V8 isolate with Apify bindings. One program per run; the captured { stdout, stderr } is pushed as a single item to the default dataset.", + "version": "0.1", + "buildTag": "latest", + "usesStandbyMode": false, + "input": { + "title": "Code Runtime Input", + "description": "The program to run inside the sandbox.", + "type": "object", + "schemaVersion": 1, + "properties": { + "code": { + "title": "Code", + "type": "string", + "description": "TypeScript/JavaScript program executed inside the sandbox. It receives an `apify` binding and `console`; stdout and stderr are captured separately and pushed to the default dataset as { stdout, stderr }.", + "editor": "javascript" + } + }, + "required": ["code"] + }, + "output": { + "actorOutputSchemaVersion": 1, + "title": "Code Runtime Output", + "description": "One dataset item { stdout, stderr } with the program's captured output.", + "type": "object", + "properties": { + "output": { + "type": "string", + "title": "Execution output", + "template": "{{links.apiDefaultDatasetUrl}}/items" + } + } + }, + "dockerfile": "../Dockerfile" +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ec15a3b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +node_modules +.git +*.log +.DS_Store diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1e13533 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +worker/usercode.js +*.log +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b675bf7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# Two-stage build: drop the Node runtime entirely. workerd is a standalone +# glibc binary; only libc + libm are needed at runtime (verified via `ldd`). +# +# Stage 1: pull the workerd binary via a Node base. pnpm keeps it in the virtual +# store (not hoisted), so resolve the path through `require('workerd')`. +FROM node:24-bookworm-slim AS builder +WORKDIR /build +COPY package.json pnpm-lock.yaml ./ +# --ignore-scripts skips workerd's postinstall (a binary-download fallback we +# don't need — the binary ships in the @cloudflare/workerd-linux-64 optional dep) +# and avoids pnpm's hard error on unapproved dependency build scripts. +RUN corepack enable \ + && pnpm install --prod --frozen-lockfile --ignore-scripts \ + && BIN="$(node -e "process.stdout.write(require('workerd').default)")" \ + && cp "$BIN" /workerd \ + && chmod +x /workerd + +# Stage 2: minimal runtime — debian + ca-certificates + the workerd binary. +FROM debian:bookworm-slim + +# curl: loopback HTTP client + Actor-input fetch; jq: extract `code` from the input JSON. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl jq \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /workerd /usr/local/bin/workerd + +WORKDIR /app +COPY worker/ ./worker/ + +ENTRYPOINT ["sh", "/app/worker/entrypoint.sh"] diff --git a/README.md b/README.md index e1d04ba..ca5c9e3 100644 --- a/README.md +++ b/README.md @@ -1 +1,76 @@ # Code Runtime (experimental) + +> ⚠️ **Experimental infrastructure Actor.** It powers **Code Mode** on +> [mcp.apify.com](https://mcp.apify.com) and is normally invoked by the Apify +> MCP Server, not run by hand. Its behaviour and API may change without notice. + +## What it does + +This Actor executes a single TypeScript/JavaScript program that an AI agent +submits through the Apify MCP Server's **Code Mode**, then returns whatever the +program printed. + +Code Mode exists so an agent can do many Apify operations in **one** program — +search the Store, run an Actor, read its dataset, filter and aggregate the +results — instead of sending every intermediate result back through the model. +This Actor is the sandbox that runs that program. + +## Enabling Code Mode on the MCP Server + +Code Mode is opt-in. Add the Code Mode tools to the `tools` query parameter of +your mcp.apify.com connection URL: + +``` +https://mcp.apify.com/?tools=run-code,get-code-docs +``` + +For full configuration options, use the configurator at +[mcp.apify.com](https://mcp.apify.com). + +## How it works + +- **One program per run.** The Actor reads your `code`, runs it once, writes the + result, and exits. +- The code runs inside a [`workerd`](https://github.com/cloudflare/workerd) V8 + isolate: **no filesystem, no package imports**, and outbound network is + restricted to `*.apify.com`. +- Inside the program a global **`apify`** object exposes a small, typed subset of + the Apify API — run Actors, read/write datasets and key-value stores — using + the current run's token. +- `console.log` / `console.info` go to **stdout**; `console.error` / + `console.warn` go to **stderr**. The two streams are captured separately. + +## Input + +```json +{ + "code": "const { items } = await apify.actor.runAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" +} +``` + +| Field | Type | Description | +|---|---|---| +| `code` | string | The TypeScript/JavaScript program to run. It receives the `apify` binding and `console`. | + +## Output + +A single **dataset item** with the captured streams: + +```json +{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "" } +``` + +If the program throws, the error lands in `stderr`; `stdout` keeps whatever was +printed before the failure. + +## Permissions & safety + +- Runs with **limited permissions**: the sandbox has no filesystem and can reach + only the Apify API (`*.apify.com`). +- It uses the **run's own token**, so the program can access only what you can. +- Each run is an isolated, single-use container — nothing persists between runs. + +## Learn more + +- Apify MCP Server: +- Code Mode design: [apify/apify-mcp-server#794](https://github.com/apify/apify-mcp-server/pull/794) diff --git a/package.json b/package.json new file mode 100644 index 0000000..33190c6 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "code-runtime", + "version": "0.1.0", + "description": "workerd as a normal (per-run) Apify Actor; one V8 isolate per run via the Worker Loader API.", + "private": true, + "packageManager": "pnpm@11.1.3", + "dependencies": { + "workerd": "1.20260402.1" + }, + "engines": { + "node": ">=24" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..95ed99b --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,75 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + workerd: + specifier: 1.20260402.1 + version: 1.20260402.1 + +packages: + + '@cloudflare/workerd-darwin-64@1.20260402.1': + resolution: {integrity: sha512-/kyyZ5HjOPT202Vsw3P+vICsVulldC9ym+W5UyKl9dufxfHgcdeT4EYtT+tkk+3SlkB1RmoVxgr3VRlqP2ojjw==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@cloudflare/workerd-darwin-arm64@1.20260402.1': + resolution: {integrity: sha512-nr+AgUonmepiuD4utn4KQl2fIn/aDuWEO+/B2fzjMnjLSgLpN1IQADUW1uD4FVJdd0ss9ugK1dztFJJWpImCaQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@cloudflare/workerd-linux-64@1.20260402.1': + resolution: {integrity: sha512-0vJj0pO6ARpCmvyLgbQk204dogL72geEyC+rOnkFgwcDJI4e8oxKrTQZWjPHra3BHUknEomUd37i3K4L4JfU4w==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@cloudflare/workerd-linux-arm64@1.20260402.1': + resolution: {integrity: sha512-aRsxuv1bmwkkX4sG8igutkwVHgsJ1I6mH4/u/1Jfhpb7Bj3xr9p9N2fjDwax+Dhy7xt2tC8TS/ZWLeFFjK2NMA==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@cloudflare/workerd-windows-64@1.20260402.1': + resolution: {integrity: sha512-aFKYAuIYTPsuWyHxv39yOFi9bmIvAsIuNmHGYkYltHZUoN4by6tJVCOQUp6PDlZO7TjHSD2XjjM8JBwK/D6tVA==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + + workerd@1.20260402.1: + resolution: {integrity: sha512-Cg+OUlukdcCHrTTg0MBCIMFRE6XO3yGVGiWCnJPvfffy2Ga2girrEq3qF/YlHSTmbIyEE5ebCFxBYYYZueQ/Mg==} + engines: {node: '>=16'} + hasBin: true + +snapshots: + + '@cloudflare/workerd-darwin-64@1.20260402.1': + optional: true + + '@cloudflare/workerd-darwin-arm64@1.20260402.1': + optional: true + + '@cloudflare/workerd-linux-64@1.20260402.1': + optional: true + + '@cloudflare/workerd-linux-arm64@1.20260402.1': + optional: true + + '@cloudflare/workerd-windows-64@1.20260402.1': + optional: true + + workerd@1.20260402.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260402.1 + '@cloudflare/workerd-darwin-arm64': 1.20260402.1 + '@cloudflare/workerd-linux-64': 1.20260402.1 + '@cloudflare/workerd-linux-arm64': 1.20260402.1 + '@cloudflare/workerd-windows-64': 1.20260402.1 diff --git a/worker/config.capnp b/worker/config.capnp new file mode 100644 index 0000000..e7288de --- /dev/null +++ b/worker/config.capnp @@ -0,0 +1,39 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + (name = "main", worker = .codeRuntime), + # Outbound for fetch(). Apify's api.apify.com may resolve to a private + # address inside the platform network, so allow private/local too. + # tlsOptions enables HTTPS egress (trust workerd's built-in CA set); + # the hostname allowlist is enforced by guard.js, not here. + (name = "internet", network = (allow = ["public", "private", "local"], tlsOptions = (trustBrowserCas = true))), + ], + sockets = [ + ( + name = "http", + address = "127.0.0.1:8787", + http = (), + service = "main", + ), + ], +); + +# runner.js is the entrypoint module; usercode.js is generated at container +# start by entrypoint.sh (the Actor input wrapped as `export async function run`). +const codeRuntime :Workerd.Worker = ( + modules = [ + (name = "runner.js", esModule = embed "runner.js"), + (name = "guard.js", esModule = embed "guard.js"), + (name = "usercode.js", esModule = embed "usercode.js"), + ], + bindings = [ + (name = "APIFY_TOKEN", fromEnvironment = "APIFY_TOKEN"), + (name = "DEFAULT_DATASET_ID", fromEnvironment = "ACTOR_DEFAULT_DATASET_ID"), + (name = "DEFAULT_DATASET_ID_LEGACY", fromEnvironment = "APIFY_DEFAULT_DATASET_ID"), + (name = "API_BASE_URL", fromEnvironment = "APIFY_API_BASE_URL"), + ], + globalOutbound = "internet", + compatibilityDate = "2026-01-15", + compatibilityFlags = ["nodejs_compat"], +); diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh new file mode 100755 index 0000000..27892bd --- /dev/null +++ b/worker/entrypoint.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# Normal-mode (non-standby) Apify Actor entrypoint. Single-tenant per run: +# 1. read the Actor input and wrap its `code` into the runnable usercode.js module +# 2. boot workerd on loopback (it embeds runner.js + usercode.js) +# 3. trigger /run once, then exit +# workerd hosts the sandboxed worker and is reached only over loopback. +set -eu + +PORT=8787 +READINESS_ATTEMPTS=100 # 100 * 0.1s = 10s budget for workerd to bind the socket + +API_BASE="${APIFY_API_BASE_URL:-https://api.apify.com}" +API_BASE="${API_BASE%/}" # APIFY_API_BASE_URL ships with a trailing slash +STORE_ID="${ACTOR_DEFAULT_KEY_VALUE_STORE_ID:-${APIFY_DEFAULT_KEY_VALUE_STORE_ID:-}}" +INPUT_KEY="${APIFY_INPUT_KEY:-INPUT}" + +if [ -z "${APIFY_TOKEN:-}" ] || [ -z "$STORE_ID" ]; then + echo "[code-runtime] missing APIFY_TOKEN or default key-value store ID" >&2 + exit 1 +fi + +# Fetch the Actor input and wrap its `code` into the runnable module. The `code` +# is inserted as code between the wrapper lines (not as a string) — no escaping. +INPUT_URL="${API_BASE}/v2/key-value-stores/${STORE_ID}/records/${INPUT_KEY}" +input_status=$(curl -sS -o /tmp/input.json -w '%{http_code}' \ + -H "Authorization: Bearer ${APIFY_TOKEN}" "$INPUT_URL") +if [ "$input_status" != "200" ]; then + echo "[code-runtime] failed to read Actor input from ${INPUT_URL} (HTTP ${input_status})" >&2 + exit 1 +fi + +{ + printf 'export async function run(apify, console) {\n' + jq -r '.code // ""' < /tmp/input.json + printf '\n}\n' +} > /app/worker/usercode.js + +/usr/local/bin/workerd serve --experimental /app/worker/config.capnp & +workerd_pid=$! +trap 'kill "$workerd_pid" 2>/dev/null || true' EXIT + +attempt=0 +until curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge "$READINESS_ATTEMPTS" ]; then + echo "[code-runtime] workerd did not become ready in time" >&2 + exit 1 + fi + sleep 0.1 +done + +# Trigger the single run. The worker runs the program and pushes { stdout, stderr } +# to the default dataset. A non-2xx response fails the Actor run (curl -f). +curl -fsS -X POST "http://127.0.0.1:${PORT}/run" diff --git a/worker/guard.js b/worker/guard.js new file mode 100644 index 0000000..426a170 --- /dev/null +++ b/worker/guard.js @@ -0,0 +1,45 @@ +// Restrict the user program's global fetch() to apify.com and its subdomains. +// Imported before usercode.js so the override is in place even for code that +// runs at module-evaluation time. Our own Apify API calls use the exported +// realFetch (the internal API is a private IP, not *.apify.com), so they are +// unaffected by this guard. +// +// NOTE: this guards the fetch() API only. It is not a complete egress boundary — +// the airtight control is workerd's globalOutbound. See SECURITY notes in the repo. + +const realFetch = globalThis.fetch.bind(globalThis); + +// Match apify.com exactly or any subdomain. The leading dot in the suffix is +// what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` +// (ends with `.evil.com`) both fail. +function isAllowedHost(hostname) { + const host = hostname.toLowerCase().replace(/\.$/, ''); // strip FQDN trailing dot + return host === 'apify.com' || host.endsWith('.apify.com'); +} + +function requestUrl(input) { + if (typeof input === 'string') return input; + if (input instanceof URL) return input.href; + if (input && typeof input.url === 'string') return input.url; // Request + return String(input); +} + +globalThis.fetch = (input, init) => { + let url; + try { + // Parse to the real host — defeats userinfo (`apify.com@evil.com`), + // path/query/fragment (`evil.com/apify.com`) and similar tricks. + url = new URL(requestUrl(input)); + } catch { + throw new Error('Blocked fetch: only absolute http(s) URLs to apify.com are allowed'); + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') { + throw new Error(`Blocked fetch: protocol "${url.protocol}" is not allowed`); + } + if (!isAllowedHost(url.hostname)) { + throw new Error(`Blocked fetch to "${url.hostname}": only apify.com and its subdomains are allowed`); + } + return realFetch(input, init); +}; + +export { realFetch }; diff --git a/worker/runner.js b/worker/runner.js new file mode 100644 index 0000000..18b57ac --- /dev/null +++ b/worker/runner.js @@ -0,0 +1,286 @@ +// Single worker for the per-run code-runtime Actor. It runs the user's program +// (imported from the generated `usercode.js` module) with the `apify` REST +// binding and a captured `console`, then pushes `{ stdout, stderr }` to the +// run's default dataset. The container entrypoint generates `usercode.js`, +// boots workerd, and triggers `/run` once. +// +// Single-tenant: one run = one container = one program = one token. No Worker +// Loader / per-request isolate is needed — the program runs in this worker, +// which is itself the sandbox (no filesystem, restricted outbound network). +// guard.js must be imported before usercode.js: it overrides globalThis.fetch +// to allow only apify.com, and exports realFetch for our own (internal) API calls. +import { realFetch } from './guard.js'; +import { run } from './usercode.js'; + +const DEFAULT_ITERATE_BATCH = 1000; +const DEFAULT_GET_SCHEMA_SAMPLE = 5; + +function stringify(x) { + if (typeof x === 'string') return x; + try { return JSON.stringify(x); } catch { return String(x); } +} + +function makeApifyBinding(token, apiV2) { + const baseHeaders = { Authorization: `Bearer ${token}` }; + + // Build a URL with optional query params; null/undefined values are dropped. + const buildUrl = (path, searchParams) => { + const url = new URL(`${apiV2}${path}`); + if (searchParams) { + for (const [k, v] of Object.entries(searchParams)) { + if (v !== undefined && v !== null) url.searchParams.set(k, String(v)); + } + } + return url; + }; + + // Single-source HTTP wrapper. Throws on non-2xx with the response body in the message. + // `body`: string / Uint8Array passed through; objects are JSON.stringify'd. + const apiCall = async (method, path, { searchParams, body, contentType } = {}) => { + const init = { method, headers: { ...baseHeaders } }; + if (body !== undefined) { + const isRaw = typeof body === 'string' || body instanceof Uint8Array || body instanceof ArrayBuffer; + init.body = isRaw ? body : JSON.stringify(body); + init.headers['content-type'] = contentType ?? (isRaw ? 'application/octet-stream' : 'application/json'); + } + const r = await realFetch(buildUrl(path, searchParams), init); + if (!r.ok) throw new Error(`${method} ${path} failed: ${r.status} ${await r.text()}`); + return r; + }; + + const apiJson = async (...args) => (await apiCall(...args)).json(); + const apiData = async (...args) => (await apiJson(...args)).data; + + const actor = { + // GET /v2/store — Apify Store search. Returns the items array directly. + search: ({ query, limit, category }) => + apiData('GET', '/store', { searchParams: { search: query, limit, category } }) + .then((d) => d.items), + + getDetails: ({ actorId }) => + apiData('GET', `/acts/${encodeURIComponent(actorId)}`), + + // POST /runs with waitForFinish blocks until the run completes (max 60s per the + // Apify API; for longer runs the caller should use start() + apify.run.wait()). + // Returns the run record so the caller can read defaultDatasetId / defaultKeyValueStoreId. + // Intentionally does NOT use /run-sync, which returns the OUTPUT KVS record (a pattern + // only some Actors follow) rather than the structured run record. + run: ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs = 60, maxTotalChargeUsd, maxItems }) => + apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { + searchParams: { + waitForFinish: waitForFinishSecs, + memory: memoryMbytes, + timeout: timeoutSecs, + maxTotalChargeUsd, + maxItems, + }, + body: input ?? {}, + }), + + // Async kickoff. Returns immediately with a run record in READY/RUNNING state. + start: ({ actorId, input, memoryMbytes, timeoutSecs, maxTotalChargeUsd, maxItems }) => + apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { + searchParams: { + memory: memoryMbytes, + timeout: timeoutSecs, + maxTotalChargeUsd, + maxItems, + }, + body: input ?? {}, + }), + // runAndGetItems is added below once `dataset.listItems` is defined. + }; + + const run = { + get: ({ runId }) => + apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`), + + // Block until the run terminates or `waitForFinishSecs` elapses (whichever comes first). + // The Apify API caps this at 60s per request; longer waits require a polling loop. + wait: ({ runId, waitForFinishSecs = 60 }) => + apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { + searchParams: { waitForFinish: waitForFinishSecs }, + }), + + abort: ({ runId }) => + apiData('POST', `/actor-runs/${encodeURIComponent(runId)}/abort`), + + // Returns the full run log as text. `limit` tails the last N characters; the Apify API + // does not paginate logs, so this is a client-side slice (the full body is fetched). + getLog: async ({ runId, limit }) => { + const r = await apiCall('GET', `/logs/${encodeURIComponent(runId)}`); + const text = await r.text(); + return limit && text.length > limit ? text.slice(-limit) : text; + }, + }; + + const dataset = { + // Returns the items array directly (no wrapper). The Apify API's + // `x-apify-pagination-total` header is unreliable for freshly-created datasets + // (eventually consistent), so we don't surface a `total`. Use `getSchema` if you + // need an item count, or iterate to consume the whole dataset. + listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }) => { + const r = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { + searchParams: { + fields: fields?.join(','), + omit: omit?.join(','), + limit, + offset, + clean: clean ? '1' : undefined, + desc: desc ? '1' : undefined, + }, + }); + return r.json(); + }, + + // Async generator over the entire dataset. Pages internally in `batchSize` chunks + // so the user can `for await (const item of apify.dataset.iterate({...}))` without + // worrying about offsets. Stops when a page returns fewer items than `batchSize` + // (the natural end-of-data signal — pagination total is not used, see listItems). + iterate: async function* ({ datasetId, fields, omit, clean, desc, batchSize = DEFAULT_ITERATE_BATCH }) { + let offset = 0; + while (true) { + const items = await dataset.listItems({ + datasetId, fields, omit, clean, desc, + limit: batchSize, offset, + }); + if (items.length === 0) break; + for (const item of items) yield item; + if (items.length < batchSize) break; + offset += items.length; + } + }, + + // Apify has no dedicated schema endpoint; we infer one from a small sample of items. + // Returns { itemCount, sampleSize, fields: [{ name, types, nullable }] }. + getSchema: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }) => { + const meta = await apiData('GET', `/datasets/${encodeURIComponent(datasetId)}`); + const items = await dataset.listItems({ datasetId, limit: sample }); + const fields = new Map(); + for (const item of items) { + for (const [name, value] of Object.entries(item ?? {})) { + const type = value === null ? 'null' : Array.isArray(value) ? 'array' : typeof value; + if (!fields.has(name)) fields.set(name, new Set()); + fields.get(name).add(type); + } + } + return { + itemCount: meta?.itemCount, + sampleSize: items.length, + fields: [...fields.entries()].map(([name, types]) => ({ + name, + types: [...types], + nullable: types.has('null'), + })), + }; + }, + + create: ({ name } = {}) => + apiData('POST', '/datasets', { searchParams: { name } }), + + pushItems: async ({ datasetId, items }) => { + await apiCall('POST', `/datasets/${encodeURIComponent(datasetId)}/items`, { body: items }); + }, + }; + + actor.runAndGetItems = async ({ actorId, input, fields, limit, ...runOpts }) => { + const runRecord = await actor.run({ actorId, input, ...runOpts }); + const items = await dataset.listItems({ + datasetId: runRecord.defaultDatasetId, fields, limit, + }); + return { run: runRecord, items }; + }; + + const kvs = { + // Returns the value directly (parsed when JSON, string when text/*, Uint8Array otherwise). + // Returns null when the key does not exist (404), not an error — this matches the common + // "lookup or default" pattern in code. + get: async ({ storeId, key }) => { + const r = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { + headers: baseHeaders, + }); + if (r.status === 404) return null; + if (!r.ok) throw new Error(`GET kvs.get failed: ${r.status} ${await r.text()}`); + const ct = r.headers.get('content-type') ?? ''; + if (ct.includes('application/json')) return r.json(); + if (ct.startsWith('text/')) return r.text(); + return new Uint8Array(await r.arrayBuffer()); + }, + + // `value`: object → application/json; string → text/plain; Uint8Array/ArrayBuffer → + // application/octet-stream (or whatever the caller passed via `contentType`). + set: async ({ storeId, key, value, contentType }) => { + let body; + let ct = contentType; + if (value instanceof Uint8Array || value instanceof ArrayBuffer) { + body = value; + ct = ct ?? 'application/octet-stream'; + } else if (typeof value === 'string') { + body = value; + ct = ct ?? 'text/plain; charset=utf-8'; + } else { + body = JSON.stringify(value); + ct = ct ?? 'application/json; charset=utf-8'; + } + await apiCall('PUT', `/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`, { + body, contentType: ct, + }); + }, + + list: ({ storeId, limit, exclusiveStartKey }) => + apiData('GET', `/key-value-stores/${encodeURIComponent(storeId)}/keys`, { + searchParams: { limit, exclusiveStartKey }, + }), + + create: ({ name } = {}) => + apiData('POST', '/key-value-stores', { searchParams: { name } }), + }; + + return { actor, run, dataset, kvs }; +} + +// Push the captured streams as a single item to the run's default dataset. +async function pushOutput(apiV2, token, env, item) { + const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; + if (!datasetId) throw new Error('Default dataset ID missing from Actor run environment.'); + const r = await realFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json; charset=utf-8' }, + body: JSON.stringify(item), + }); + if (!r.ok) throw new Error(`Failed to push dataset item: ${r.status} ${await r.text()}`); +} + +export default { + async fetch(request, env) { + const url = new URL(request.url); + if (url.pathname === '/health') return new Response('ok'); + if (url.pathname !== '/run') return new Response('Not found', { status: 404 }); + + const token = env.APIFY_TOKEN; + if (!token) throw new Error('APIFY_TOKEN missing from Actor run environment.'); + // APIFY_API_BASE_URL is the platform-internal API (may have a trailing slash). + const apiV2 = `${(env.API_BASE_URL || 'https://api.apify.com').replace(/\/+$/, '')}/v2`; + + const stdout = []; + const stderr = []; + const captureConsole = { + log: (...args) => stdout.push(args.map(stringify).join(' ')), + error: (...args) => stderr.push(args.map(stringify).join(' ')), + warn: (...args) => stderr.push(args.map(stringify).join(' ')), + info: (...args) => stdout.push(args.map(stringify).join(' ')), + }; + + // A thrown program is a user-level failure: capture it in stderr and still + // push the output, so the run SUCCEEDS with diagnostics. Infra failures + // (missing env, dataset push) throw and fail the run. + try { + await run(makeApifyBinding(token, apiV2), captureConsole); + } catch (err) { + stderr.push(err?.stack ?? err?.message ?? String(err)); + } + + await pushOutput(apiV2, token, env, { stdout: stdout.join('\n'), stderr: stderr.join('\n') }); + return Response.json({ ok: true }); + }, +}; From 6abfe193e0dd0a31203514658c297d89ee6d594a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 17:45:27 +0200 Subject: [PATCH 02/46] test: add apify binding smoke test (tests/binding-smoke.js + test.sh) --- test.sh | 33 ++++++++++++ tests/binding-smoke.js | 111 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100755 test.sh create mode 100644 tests/binding-smoke.js diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..6d72c0d --- /dev/null +++ b/test.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# Deploy the Actor (apify push) and run the binding smoke test on the freshly +# built version via `apify call`. Exits non-zero if any binding check fails. +# +# Usage: ./test.sh +set -eu + +cd "$(dirname "$0")" + +TEST_JS="tests/binding-smoke.js" + +command -v apify >/dev/null 2>&1 || { echo "apify CLI not found" >&2; exit 1; } +command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 1; } + +input_json="$(mktemp)" +trap 'rm -f "$input_json"' EXIT + +echo "==> apify push" +apify push + +echo "==> building input from ${TEST_JS}" +jq -n --arg code "$(cat "$TEST_JS")" '{ code: $code }' > "$input_json" + +echo "==> apify call (running the test on the built Actor)" +output="$(apify call -f "$input_json" -o 2>&1)" +echo "$output" + +if printf '%s' "$output" | grep -q 'ALL_TESTS_PASSED'; then + echo "==> all binding tests passed" +else + echo "==> binding tests FAILED" >&2 + exit 1 +fi diff --git a/tests/binding-smoke.js b/tests/binding-smoke.js new file mode 100644 index 0000000..98b5b55 --- /dev/null +++ b/tests/binding-smoke.js @@ -0,0 +1,111 @@ +// Smoke test for the `apify` binding exposed to Code Mode programs. Submitted as +// the Actor's `code` input by test.sh and executed on the built Actor via +// `apify call`. Exercises every binding method and prints a sentinel line +// (ALL_TESTS_PASSED) that test.sh greps for. +const results = []; +async function check(name, fn) { + try { + const out = await fn(); + console.log(`PASS ${name}: ${out ?? ''}`); + results.push(true); + } catch (e) { + console.error(`FAIL ${name}: ${e.message}`); + results.push(false); + } +} + +const ACTOR = 'apify/hello-world'; + +// ---- actor (read) ---- +await check('actor.search', async () => { + const items = await apify.actor.search({ query: 'hello world', limit: 3 }); + if (!Array.isArray(items)) throw new Error('expected array'); + return `${items.length} actors`; +}); +await check('actor.getDetails', async () => { + const d = await apify.actor.getDetails({ actorId: ACTOR }); + return `${d.username}/${d.name}`; +}); + +// ---- dataset ---- +let datasetId; +await check('dataset.create', async () => { + datasetId = (await apify.dataset.create()).id; + return datasetId; +}); +await check('dataset.pushItems', async () => { + await apify.dataset.pushItems({ datasetId, items: [{ a: 1, b: 'x' }, { a: 2, b: 'y' }] }); + return '2 pushed'; +}); +await check('dataset.listItems', async () => { + const items = await apify.dataset.listItems({ datasetId }); + return `${items.length} items`; +}); +await check('dataset.getSchema', async () => { + const s = await apify.dataset.getSchema({ datasetId }); + return `itemCount=${s.itemCount} fields=${s.fields.map((f) => f.name).join(',')}`; +}); +await check('dataset.iterate', async () => { + let n = 0; + for await (const _ of apify.dataset.iterate({ datasetId, batchSize: 1 })) n++; + return `${n} iterated`; +}); + +// ---- key-value store ---- +let storeId; +await check('kvs.create', async () => { + storeId = (await apify.kvs.create()).id; + return storeId; +}); +await check('kvs.set', async () => { + await apify.kvs.set({ storeId, key: 'obj', value: { hello: 'world' } }); + await apify.kvs.set({ storeId, key: 'txt', value: 'plain' }); + return 'set obj + txt'; +}); +await check('kvs.get', async () => { + const obj = await apify.kvs.get({ storeId, key: 'obj' }); + const txt = await apify.kvs.get({ storeId, key: 'txt' }); + const missing = await apify.kvs.get({ storeId, key: 'nope' }); + return `obj.hello=${obj.hello} txt=${txt} missing=${missing}`; +}); +await check('kvs.list', async () => { + const l = await apify.kvs.list({ storeId }); + return `${l.items.length} keys`; +}); + +// ---- run lifecycle ---- +let runId; +await check('actor.start', async () => { + const run = await apify.actor.start({ actorId: ACTOR }); + runId = run.id; + return `runId=${runId} status=${run.status}`; +}); +await check('run.get', async () => { + return `status=${(await apify.run.get({ runId })).status}`; +}); +await check('run.wait', async () => { + return `status=${(await apify.run.wait({ runId, waitForFinishSecs: 60 })).status}`; +}); +await check('run.getLog', async () => { + return `${(await apify.run.getLog({ runId, limit: 200 })).length} chars`; +}); + +// ---- run + get items (sync) ---- +await check('actor.run', async () => { + return `status=${(await apify.actor.run({ actorId: ACTOR, waitForFinishSecs: 60 })).status}`; +}); +await check('actor.runAndGetItems', async () => { + const { run, items } = await apify.actor.runAndGetItems({ actorId: ACTOR, limit: 5, waitForFinishSecs: 60 }); + return `status=${run.status} items=${items.length}`; +}); + +// ---- abort ---- +await check('run.abort', async () => { + const run = await apify.actor.start({ actorId: ACTOR }); + return `status=${(await apify.run.abort({ runId: run.id })).status}`; +}); + +const passed = results.filter(Boolean).length; +console.log(`\n=== SUMMARY: ${passed}/${results.length} passed ===`); +if (passed === results.length) console.log('ALL_TESTS_PASSED'); +else console.error(`SOME_TESTS_FAILED (${results.length - passed} failed)`); From b669c05d1eed22b18b1fa547e5d5fa993af9ced0 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 17:51:46 +0200 Subject: [PATCH 03/46] docs: tighten README copy and add compact apify binding reference --- README.md | 51 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ca5c9e3..fac65a6 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ ## What it does -This Actor executes a single TypeScript/JavaScript program that an AI agent +This Actor executes a single TypeScript/JavaScript script that an AI agent submits through the Apify MCP Server's **Code Mode**, then returns whatever the -program printed. +script printed. -Code Mode exists so an agent can do many Apify operations in **one** program — +Code Mode exists so an agent can do many Apify operations in **one go** — search the Store, run an Actor, read its dataset, filter and aggregate the results — instead of sending every intermediate result back through the model. -This Actor is the sandbox that runs that program. +This Actor is the sandbox that runs that script. ## Enabling Code Mode on the MCP Server @@ -29,17 +29,50 @@ For full configuration options, use the configurator at ## How it works -- **One program per run.** The Actor reads your `code`, runs it once, writes the +- **One script per run.** The Actor reads your `code`, runs it once, writes the result, and exits. - The code runs inside a [`workerd`](https://github.com/cloudflare/workerd) V8 isolate: **no filesystem, no package imports**, and outbound network is restricted to `*.apify.com`. -- Inside the program a global **`apify`** object exposes a small, typed subset of +- Inside the script a global **`apify`** object exposes a small, typed subset of the Apify API — run Actors, read/write datasets and key-value stores — using - the current run's token. + the current run's token (see below). - `console.log` / `console.info` go to **stdout**; `console.error` / `console.warn` go to **stderr**. The two streams are captured separately. +## The `apify` binding + +Every method takes one options object and returns parsed JSON +(`?` = optional, `= x` = default): + +```js +// Actors +apify.actor.search({ query, limit?, category? }) // → actors[] +apify.actor.getDetails({ actorId }) // → actor +apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run +apify.actor.run({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) +apify.actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } + +// Runs +apify.run.get({ runId }) // → run +apify.run.wait({ runId, waitForFinishSecs = 60 }) // → run +apify.run.abort({ runId }) // → run +apify.run.getLog({ runId, limit? }) // → string + +// Datasets +apify.dataset.create({ name? }) // → dataset +apify.dataset.pushItems({ datasetId, items }) // → void +apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) // → items[] +apify.dataset.iterate({ datasetId, batchSize = 1000, ...filters }) // → async iterable +apify.dataset.getSchema({ datasetId, sample = 5 }) // → { itemCount, fields[] } + +// Key-value stores +apify.kvs.create({ name? }) // → store +apify.kvs.set({ storeId, key, value, contentType? }) // → void +apify.kvs.get({ storeId, key }) // → value | null +apify.kvs.list({ storeId, limit?, exclusiveStartKey? }) // → { items } +``` + ## Input ```json @@ -50,7 +83,7 @@ For full configuration options, use the configurator at | Field | Type | Description | |---|---|---| -| `code` | string | The TypeScript/JavaScript program to run. It receives the `apify` binding and `console`. | +| `code` | string | The TypeScript/JavaScript script to run. It receives the `apify` binding and `console`. | ## Output @@ -60,7 +93,7 @@ A single **dataset item** with the captured streams: { "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "" } ``` -If the program throws, the error lands in `stderr`; `stdout` keeps whatever was +If the script throws, the error lands in `stderr`; `stdout` keeps whatever was printed before the failure. ## Permissions & safety From f20b4896cdcae673ae617bd16bdd02ee3f300ffe Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 17:55:08 +0200 Subject: [PATCH 04/46] docs: move apify binding section above Learn more; trim intro copy --- README.md | 69 +++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index fac65a6..99f71bb 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,13 @@ ## What it does -This Actor executes a single TypeScript/JavaScript script that an AI agent -submits through the Apify MCP Server's **Code Mode**, then returns whatever the -script printed. +This Actor executes TypeScript/JavaScript that an AI agent submits through the +Apify MCP Server's **Code Mode**, then returns whatever the script printed. Code Mode exists so an agent can do many Apify operations in **one go** — search the Store, run an Actor, read its dataset, filter and aggregate the -results — instead of sending every intermediate result back through the model. -This Actor is the sandbox that runs that script. +results — instead of sending every intermediate result back through the model +and wasting tokens. This Actor is the sandbox that runs that script. ## Enabling Code Mode on the MCP Server @@ -40,6 +39,36 @@ For full configuration options, use the configurator at - `console.log` / `console.info` go to **stdout**; `console.error` / `console.warn` go to **stderr**. The two streams are captured separately. +## Input + +```json +{ + "code": "const { items } = await apify.actor.runAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" +} +``` + +| Field | Type | Description | +|---|---|---| +| `code` | string | The TypeScript/JavaScript script to run. It receives the `apify` binding and `console`. | + +## Output + +A single **dataset item** with the captured streams: + +```json +{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "" } +``` + +If the script throws, the error lands in `stderr`; `stdout` keeps whatever was +printed before the failure. + +## Permissions & safety + +- Runs with **limited permissions**: the sandbox has no filesystem and can reach + only the Apify API (`*.apify.com`). +- It uses the **run's own token**, so the program can access only what you can. +- Each run is an isolated, single-use container — nothing persists between runs. + ## The `apify` binding Every method takes one options object and returns parsed JSON @@ -73,36 +102,6 @@ apify.kvs.get({ storeId, key }) // → value | null apify.kvs.list({ storeId, limit?, exclusiveStartKey? }) // → { items } ``` -## Input - -```json -{ - "code": "const { items } = await apify.actor.runAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" -} -``` - -| Field | Type | Description | -|---|---|---| -| `code` | string | The TypeScript/JavaScript script to run. It receives the `apify` binding and `console`. | - -## Output - -A single **dataset item** with the captured streams: - -```json -{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "" } -``` - -If the script throws, the error lands in `stderr`; `stdout` keeps whatever was -printed before the failure. - -## Permissions & safety - -- Runs with **limited permissions**: the sandbox has no filesystem and can reach - only the Apify API (`*.apify.com`). -- It uses the **run's own token**, so the program can access only what you can. -- Each run is an isolated, single-use container — nothing persists between runs. - ## Learn more - Apify MCP Server: From 63590fdc4ce2f51b64eba39767e2e3d9a7108b39 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 18:13:57 +0200 Subject: [PATCH 05/46] docs: drop run-token sentence from Permissions & safety --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 99f71bb..b64b77b 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,6 @@ printed before the failure. - Runs with **limited permissions**: the sandbox has no filesystem and can reach only the Apify API (`*.apify.com`). -- It uses the **run's own token**, so the program can access only what you can. - Each run is an isolated, single-use container — nothing persists between runs. ## The `apify` binding From 8d0c7ffa23b7fdf40c6324050684834f478d9d5f Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 18:21:03 +0200 Subject: [PATCH 06/46] docs: add docs/API.md detailed reference and link it from README --- README.md | 3 +- docs/API.md | 307 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 docs/API.md diff --git a/README.md b/README.md index b64b77b..085eb7e 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,8 @@ printed before the failure. ## The `apify` binding Every method takes one options object and returns parsed JSON -(`?` = optional, `= x` = default): +(`?` = optional, `= x` = default). Full API documentation is available +[here](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). ```js // Actors diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..718be3f --- /dev/null +++ b/docs/API.md @@ -0,0 +1,307 @@ +# The `apify` binding — API reference + +Inside a Code Mode script a global **`apify`** object exposes a small, typed +subset of the Apify API, authenticated with the current run's token. This +document describes every method in detail. + +## Conventions + +- **Every method is `async`** — `await` the result (or `for await` for + `dataset.iterate`). +- **One options object.** Each method takes a single object argument; there are + no positional parameters. +- **`actorId`** accepts either `username/name` (e.g. `apify/rag-web-browser`) or + the Actor's ID. +- **Errors.** A non-2xx API response throws an `Error` whose message is + ` failed: `. The one exception is + [`kvs.get`](#kvsget--value--null), which returns `null` for a missing key + instead of throwing. +- **Network.** Outbound `fetch` from your script is restricted to `apify.com` + and its subdomains. + +--- + +## `apify.actor` + +### `actor.search({ query, limit?, category? })` → `Actor[]` + +Search the Apify Store. + +| Param | Type | Required | Description | +|---|---|---|---| +| `query` | `string` | yes | Full-text search query. | +| `limit` | `number` | no | Maximum number of results. | +| `category` | `string` | no | Restrict to a Store category. | + +Returns the array of matching Store Actor records. + +```js +const actors = await apify.actor.search({ query: 'web scraper', limit: 5 }); +console.log(actors.map((a) => `${a.username}/${a.name}`).join('\n')); +``` + +### `actor.getDetails({ actorId })` → `Actor` + +Fetch the full record for one Actor. + +| Param | Type | Required | Description | +|---|---|---|---| +| `actorId` | `string` | yes | `username/name` or Actor ID. | + +### `actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` + +Start an Actor **asynchronously** and return immediately with a run record in +`READY`/`RUNNING` state. Use [`run.wait`](#runwait--run) to block for the result. + +| Param | Type | Required | Description | +|---|---|---|---| +| `actorId` | `string` | yes | `username/name` or Actor ID. | +| `input` | `object` | no | Actor input (defaults to `{}`). | +| `memoryMbytes` | `number` | no | Memory limit for the run. | +| `timeoutSecs` | `number` | no | Run timeout in seconds. | +| `maxTotalChargeUsd` | `number` | no | Hard cap on the run's cost. | +| `maxItems` | `number` | no | Max dataset items for pay-per-result Actors. | + +### `actor.run({ actorId, input?, waitForFinishSecs?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` + +Start an Actor and **wait** for it to finish (or until `waitForFinishSecs` +elapses), then return the run record. + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `actorId` | `string` | yes | | `username/name` or Actor ID. | +| `input` | `object` | no | `{}` | Actor input. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait. **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.wait()` loop. | +| `memoryMbytes` | `number` | no | | Memory limit. | +| `timeoutSecs` | `number` | no | | Run timeout. | +| `maxTotalChargeUsd` | `number` | no | | Cost cap. | +| `maxItems` | `number` | no | | Max items (pay-per-result). | + +The run record exposes `defaultDatasetId` and `defaultKeyValueStoreId` for +reading results. + +### `actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` + +Convenience wrapper: `actor.run(...)` followed by reading the run's default +dataset. + +| Param | Type | Required | Description | +|---|---|---|---| +| `actorId` | `string` | yes | `username/name` or Actor ID. | +| `input` | `object` | no | Actor input. | +| `fields` | `string[]` | no | Restrict returned item fields. | +| `limit` | `number` | no | Max items to fetch. | +| `...runOpts` | | no | Any `actor.run` option (`waitForFinishSecs`, `memoryMbytes`, `timeoutSecs`, `maxTotalChargeUsd`, `maxItems`). | + +Returns `{ run: Run, items: object[] }`. + +```js +const { run, items } = await apify.actor.runAndGetItems({ + actorId: 'apify/rag-web-browser', + input: { query: 'apify' }, + limit: 3, +}); +console.log(run.status, items.length); +``` + +--- + +## `apify.run` + +### `run.get({ runId })` → `Run` + +Fetch the current run record (status, stats, default storage IDs). + +| Param | Type | Required | Description | +|---|---|---|---| +| `runId` | `string` | yes | The run ID. | + +### `run.wait({ runId, waitForFinishSecs? })` → `Run` + +Block until the run terminates or `waitForFinishSecs` elapses, whichever comes +first, then return the run record. + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `runId` | `string` | yes | | The run ID. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait. **Capped at 60s by the API**; poll in a loop for longer runs. | + +### `run.abort({ runId })` → `Run` + +Abort a running run. Returns the run record (status transitions to `ABORTING`). + +| Param | Type | Required | Description | +|---|---|---|---| +| `runId` | `string` | yes | The run ID. | + +### `run.getLog({ runId, limit? })` → `string` + +Return the run's log as text. + +| Param | Type | Required | Description | +|---|---|---|---| +| `runId` | `string` | yes | The run ID. | +| `limit` | `number` | no | If set, return only the **last** `limit` characters (client-side tail; the full log is fetched). | + +--- + +## `apify.dataset` + +### `dataset.create({ name? })` → `Dataset` + +Create a dataset and return its record (use `.id` for subsequent calls). + +| Param | Type | Required | Description | +|---|---|---|---| +| `name` | `string` | no | Named (persistent) dataset; omit for an unnamed (temporary) one. | + +### `dataset.pushItems({ datasetId, items })` → `void` + +Append one or more items to a dataset. + +| Param | Type | Required | Description | +|---|---|---|---| +| `datasetId` | `string` | yes | Target dataset ID. | +| `items` | `object \| object[]` | yes | A single item or an array of items. | + +### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `object[]` + +Read a page of items. Returns the items array directly (no pagination wrapper). + +| Param | Type | Required | Description | +|---|---|---|---| +| `datasetId` | `string` | yes | Dataset ID. | +| `fields` | `string[]` | no | Only include these fields. | +| `omit` | `string[]` | no | Exclude these fields. | +| `limit` | `number` | no | Page size. | +| `offset` | `number` | no | Starting offset. | +| `clean` | `boolean` | no | Skip empty items / hidden fields. | +| `desc` | `boolean` | no | Reverse (newest first). | + +> A dataset's pagination total is eventually consistent right after creation, so +> no `total` is surfaced. Use [`getSchema`](#datasetgetschema--schema) for an +> item count, or [`iterate`](#datasetiterate--asyncgeneratorobject) to consume +> everything. + +### `dataset.iterate({ datasetId, fields?, omit?, clean?, desc?, batchSize? })` → `AsyncGenerator` + +Async-iterate the **entire** dataset, paging internally so you don't manage +offsets. + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `datasetId` | `string` | yes | | Dataset ID. | +| `fields` | `string[]` | no | | Only include these fields. | +| `omit` | `string[]` | no | | Exclude these fields. | +| `clean` | `boolean` | no | | Skip empty items / hidden fields. | +| `desc` | `boolean` | no | | Reverse order. | +| `batchSize` | `number` | no | `1000` | Items fetched per page. | + +```js +let count = 0; +for await (const item of apify.dataset.iterate({ datasetId })) count++; +console.log('total items:', count); +``` + +### `dataset.getSchema({ datasetId, sample? })` → `Schema` + +Infer a lightweight schema from a sample of items (Apify has no schema endpoint). + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `datasetId` | `string` | yes | | Dataset ID. | +| `sample` | `number` | no | `5` | Number of items to inspect. | + +Returns: + +```js +{ + itemCount, // number | undefined (dataset metadata; eventually consistent) + sampleSize, // number of items actually inspected + fields: [ // one entry per field seen in the sample + { name, types, nullable } // types: string[] (e.g. ['string','null']); nullable: boolean + ] +} +``` + +--- + +## `apify.kvs` + +### `kvs.create({ name? })` → `Store` + +Create a key-value store and return its record. + +| Param | Type | Required | Description | +|---|---|---|---| +| `name` | `string` | no | Named (persistent) store; omit for an unnamed (temporary) one. | + +### `kvs.set({ storeId, key, value, contentType? })` → `void` + +Write a record. The content type is inferred from `value`: + +| `value` type | Stored as | +|---|---| +| `object` | `application/json` | +| `string` | `text/plain; charset=utf-8` | +| `Uint8Array` / `ArrayBuffer` | `application/octet-stream` | + +| Param | Type | Required | Description | +|---|---|---|---| +| `storeId` | `string` | yes | Store ID. | +| `key` | `string` | yes | Record key. | +| `value` | `object \| string \| Uint8Array \| ArrayBuffer` | yes | Value to store. | +| `contentType` | `string` | no | Override the inferred content type. | + +### `kvs.get({ storeId, key })` → `value` \| `null` + +Read a record. The return type follows the stored content type: + +- `application/json` → parsed **object** +- `text/*` → **string** +- anything else → **`Uint8Array`** + +Returns **`null`** when the key does not exist (404), so you can do +lookup-or-default without a `try/catch`. + +| Param | Type | Required | Description | +|---|---|---|---| +| `storeId` | `string` | yes | Store ID. | +| `key` | `string` | yes | Record key. | + +### `kvs.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, ... }` + +List keys in a store. + +| Param | Type | Required | Description | +|---|---|---|---| +| `storeId` | `string` | yes | Store ID. | +| `limit` | `number` | no | Max keys to return. | +| `exclusiveStartKey` | `string` | no | Continue listing after this key (pagination). | + +Returns the store-keys listing, whose `items` is an array of `{ key, size }`. + +--- + +## `console` + +`console` is captured, not printed live: + +| Method | Stream | +|---|---| +| `console.log`, `console.info` | **stdout** | +| `console.error`, `console.warn` | **stderr** | + +Non-string arguments are `JSON.stringify`'d. When the script finishes, both +streams are written to the run's default dataset as a single item: + +```json +{ "stdout": "...", "stderr": "..." } +``` + +## Error handling + +- A non-2xx API response throws `Error: failed: `. +- If your script throws, the error (stack/message) is appended to **stderr** and + the run still **succeeds** with whatever was printed beforehand — so failures + are observable in the output rather than crashing the run. From 4c3a430b3da5cbc63dc964b6e1d0846b78817c24 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 19:49:34 +0200 Subject: [PATCH 07/46] docs: document exact output shapes and link each method to its Apify API v2 endpoint --- docs/API.md | 140 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 105 insertions(+), 35 deletions(-) diff --git a/docs/API.md b/docs/API.md index 718be3f..3347393 100644 --- a/docs/API.md +++ b/docs/API.md @@ -12,10 +12,17 @@ document describes every method in detail. no positional parameters. - **`actorId`** accepts either `username/name` (e.g. `apify/rag-web-browser`) or the Actor's ID. +- **Return values.** The Apify API wraps most responses in a `{ "data": … }` + envelope. These methods **unwrap it for you** and return the inner value; the + linked API page describes that inner `data` shape. Where a method transforms + the response further (extracts an array, parses by content type, infers a + schema, returns plain text), the exact output is spelled out below. +- **`Apify API:`** each method links to its underlying endpoint on + [docs.apify.com/api/v2](https://docs.apify.com/api/v2) so you can inspect the + live request/response schema. - **Errors.** A non-2xx API response throws an `Error` whose message is ` failed: `. The one exception is - [`kvs.get`](#kvsget--value--null), which returns `null` for a missing key - instead of throwing. + [`kvs.get`](#kvsget--value--null), which returns `null` for a missing key. - **Network.** Outbound `fetch` from your script is restricted to `apify.com` and its subdomains. @@ -33,7 +40,9 @@ Search the Apify Store. | `limit` | `number` | no | Maximum number of results. | | `category` | `string` | no | Restrict to a Store category. | -Returns the array of matching Store Actor records. +**Output:** the `data.items` array of the Store listing (the pagination wrapper +is dropped) — i.e. an `Actor[]`. +**Apify API:** [`GET /v2/store`](https://docs.apify.com/api/v2/store-get) ```js const actors = await apify.actor.search({ query: 'web scraper', limit: 5 }); @@ -48,6 +57,9 @@ Fetch the full record for one Actor. |---|---|---|---| | `actorId` | `string` | yes | `username/name` or Actor ID. | +**Output:** the Actor object (unwrapped `data`). +**Apify API:** [`GET /v2/acts/{actorId}`](https://docs.apify.com/api/v2/act-get) + ### `actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` Start an Actor **asynchronously** and return immediately with a run record in @@ -57,11 +69,14 @@ Start an Actor **asynchronously** and return immediately with a run record in |---|---|---|---| | `actorId` | `string` | yes | `username/name` or Actor ID. | | `input` | `object` | no | Actor input (defaults to `{}`). | -| `memoryMbytes` | `number` | no | Memory limit for the run. | -| `timeoutSecs` | `number` | no | Run timeout in seconds. | +| `memoryMbytes` | `number` | no | Memory limit for the run (`memory` query param). | +| `timeoutSecs` | `number` | no | Run timeout in seconds (`timeout`). | | `maxTotalChargeUsd` | `number` | no | Hard cap on the run's cost. | | `maxItems` | `number` | no | Max dataset items for pay-per-result Actors. | +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) + ### `actor.run({ actorId, input?, waitForFinishSecs?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` Start an Actor and **wait** for it to finish (or until `waitForFinishSecs` @@ -71,19 +86,21 @@ elapses), then return the run record. |---|---|---|---|---| | `actorId` | `string` | yes | | `username/name` or Actor ID. | | `input` | `object` | no | `{}` | Actor input. | -| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait. **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.wait()` loop. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.wait()` loop. | | `memoryMbytes` | `number` | no | | Memory limit. | | `timeoutSecs` | `number` | no | | Run timeout. | | `maxTotalChargeUsd` | `number` | no | | Cost cap. | | `maxItems` | `number` | no | | Max items (pay-per-result). | -The run record exposes `defaultDatasetId` and `defaultKeyValueStoreId` for -reading results. +**Output:** the Run object (unwrapped `data`), exposing `defaultDatasetId` and +`defaultKeyValueStoreId` for reading results. Uses the standard run endpoint +(not `/run-sync`, which returns the output record instead of the run object). +**Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) ### `actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` Convenience wrapper: `actor.run(...)` followed by reading the run's default -dataset. +dataset via `dataset.listItems`. | Param | Type | Required | Description | |---|---|---|---| @@ -93,7 +110,17 @@ dataset. | `limit` | `number` | no | Max items to fetch. | | `...runOpts` | | no | Any `actor.run` option (`waitForFinishSecs`, `memoryMbytes`, `timeoutSecs`, `maxTotalChargeUsd`, `maxItems`). | -Returns `{ run: Run, items: object[] }`. +**Output (custom):** + +```js +{ + run: Run, // the run object, as actor.run returns + items: object[] // items from run.defaultDatasetId +} +``` + +**Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) +then [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) ```js const { run, items } = await apify.actor.runAndGetItems({ @@ -116,6 +143,9 @@ Fetch the current run record (status, stats, default storage IDs). |---|---|---|---| | `runId` | `string` | yes | The run ID. | +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`GET /v2/actor-runs/{runId}`](https://docs.apify.com/api/v2/actor-run-get) + ### `run.wait({ runId, waitForFinishSecs? })` → `Run` Block until the run terminates or `waitForFinishSecs` elapses, whichever comes @@ -124,16 +154,23 @@ first, then return the run record. | Param | Type | Required | Default | Description | |---|---|---|---|---| | `runId` | `string` | yes | | The run ID. | -| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait. **Capped at 60s by the API**; poll in a loop for longer runs. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **Capped at 60s by the API**; poll in a loop for longer runs. | + +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`GET /v2/actor-runs/{runId}`](https://docs.apify.com/api/v2/actor-run-get) ### `run.abort({ runId })` → `Run` Abort a running run. Returns the run record (status transitions to `ABORTING`). +Aborting a finished run is an API error. | Param | Type | Required | Description | |---|---|---|---| | `runId` | `string` | yes | The run ID. | +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`POST /v2/actor-runs/{runId}/abort`](https://docs.apify.com/api/v2/act-run-abort-post) + ### `run.getLog({ runId, limit? })` → `string` Return the run's log as text. @@ -143,6 +180,10 @@ Return the run's log as text. | `runId` | `string` | yes | The run ID. | | `limit` | `number` | no | If set, return only the **last** `limit` characters (client-side tail; the full log is fetched). | +**Output (custom):** the raw log **text** (not JSON). With `limit`, the last +`limit` characters. +**Apify API:** [`GET /v2/logs/{runId}`](https://docs.apify.com/api/v2/log-get) + --- ## `apify.dataset` @@ -155,6 +196,9 @@ Create a dataset and return its record (use `.id` for subsequent calls). |---|---|---|---| | `name` | `string` | no | Named (persistent) dataset; omit for an unnamed (temporary) one. | +**Output:** the Dataset object (unwrapped `data`). +**Apify API:** [`POST /v2/datasets`](https://docs.apify.com/api/v2/datasets-post) + ### `dataset.pushItems({ datasetId, items })` → `void` Append one or more items to a dataset. @@ -164,29 +208,34 @@ Append one or more items to a dataset. | `datasetId` | `string` | yes | Target dataset ID. | | `items` | `object \| object[]` | yes | A single item or an array of items. | +**Output:** none (resolves once the items are stored). +**Apify API:** [`POST /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-post) + ### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `object[]` -Read a page of items. Returns the items array directly (no pagination wrapper). +Read a page of items. | Param | Type | Required | Description | |---|---|---|---| | `datasetId` | `string` | yes | Dataset ID. | -| `fields` | `string[]` | no | Only include these fields. | +| `fields` | `string[]` | no | Only include these fields (joined into `fields`). | | `omit` | `string[]` | no | Exclude these fields. | | `limit` | `number` | no | Page size. | | `offset` | `number` | no | Starting offset. | -| `clean` | `boolean` | no | Skip empty items / hidden fields. | -| `desc` | `boolean` | no | Reverse (newest first). | +| `clean` | `boolean` | no | Skip empty items / hidden fields (`clean=1`). | +| `desc` | `boolean` | no | Reverse (newest first, `desc=1`). | -> A dataset's pagination total is eventually consistent right after creation, so -> no `total` is surfaced. Use [`getSchema`](#datasetgetschema--schema) for an -> item count, or [`iterate`](#datasetiterate--asyncgeneratorobject) to consume -> everything. +**Output (custom):** the **items array directly** — this endpoint already +returns a bare array (no `data`/pagination wrapper). A dataset's pagination +total is eventually consistent right after creation, so no `total` is surfaced; +use [`getSchema`](#datasetgetschema--schema) for a count or +[`iterate`](#datasetiterate--asyncgeneratorobject) to consume everything. +**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) ### `dataset.iterate({ datasetId, fields?, omit?, clean?, desc?, batchSize? })` → `AsyncGenerator` Async-iterate the **entire** dataset, paging internally so you don't manage -offsets. +offsets. Stops when a page returns fewer than `batchSize` items. | Param | Type | Required | Default | Description | |---|---|---|---|---| @@ -197,6 +246,9 @@ offsets. | `desc` | `boolean` | no | | Reverse order. | | `batchSize` | `number` | no | `1000` | Items fetched per page. | +**Output (custom):** an async generator yielding one item (`object`) at a time. +**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) (paged internally) + ```js let count = 0; for await (const item of apify.dataset.iterate({ datasetId })) count++; @@ -212,18 +264,25 @@ Infer a lightweight schema from a sample of items (Apify has no schema endpoint) | `datasetId` | `string` | yes | | Dataset ID. | | `sample` | `number` | no | `5` | Number of items to inspect. | -Returns: +**Output (custom):** ```js { - itemCount, // number | undefined (dataset metadata; eventually consistent) + itemCount, // number | undefined — from the dataset metadata (eventually consistent) sampleSize, // number of items actually inspected - fields: [ // one entry per field seen in the sample - { name, types, nullable } // types: string[] (e.g. ['string','null']); nullable: boolean + fields: [ // one entry per field seen across the sample + { + name, // field name + types, // string[], e.g. ['string'] or ['number','null'] + nullable // boolean — true if any sampled value was null + } ] } ``` +**Apify API:** [`GET /v2/datasets/{datasetId}`](https://docs.apify.com/api/v2/dataset-get) +(for `itemCount`) + [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) (the sample) + --- ## `apify.kvs` @@ -236,13 +295,16 @@ Create a key-value store and return its record. |---|---|---|---| | `name` | `string` | no | Named (persistent) store; omit for an unnamed (temporary) one. | +**Output:** the Store object (unwrapped `data`). +**Apify API:** [`POST /v2/key-value-stores`](https://docs.apify.com/api/v2/key-value-stores-post) + ### `kvs.set({ storeId, key, value, contentType? })` → `void` Write a record. The content type is inferred from `value`: | `value` type | Stored as | |---|---| -| `object` | `application/json` | +| `object` | `application/json; charset=utf-8` | | `string` | `text/plain; charset=utf-8` | | `Uint8Array` / `ArrayBuffer` | `application/octet-stream` | @@ -253,23 +315,29 @@ Write a record. The content type is inferred from `value`: | `value` | `object \| string \| Uint8Array \| ArrayBuffer` | yes | Value to store. | | `contentType` | `string` | no | Override the inferred content type. | -### `kvs.get({ storeId, key })` → `value` \| `null` - -Read a record. The return type follows the stored content type: +**Output:** none (resolves once the record is stored). +**Apify API:** [`PUT /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-put) -- `application/json` → parsed **object** -- `text/*` → **string** -- anything else → **`Uint8Array`** +### `kvs.get({ storeId, key })` → `value` \| `null` -Returns **`null`** when the key does not exist (404), so you can do -lookup-or-default without a `try/catch`. +Read a record. | Param | Type | Required | Description | |---|---|---|---| | `storeId` | `string` | yes | Store ID. | | `key` | `string` | yes | Record key. | -### `kvs.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, ... }` +**Output (custom):** the value, typed by the stored content type — + +- `application/json` → parsed **object** +- `text/*` → **string** +- anything else → **`Uint8Array`** + +Returns **`null`** when the key does not exist (404) instead of throwing, so you +can do lookup-or-default without a `try/catch`. +**Apify API:** [`GET /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-get) + +### `kvs.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, … }` List keys in a store. @@ -279,7 +347,9 @@ List keys in a store. | `limit` | `number` | no | Max keys to return. | | `exclusiveStartKey` | `string` | no | Continue listing after this key (pagination). | -Returns the store-keys listing, whose `items` is an array of `{ key, size }`. +**Output:** the unwrapped `data`: `{ items: [{ key, size }], count, limit, +isTruncated, exclusiveStartKey, nextExclusiveStartKey }`. +**Apify API:** [`GET /v2/key-value-stores/{storeId}/keys`](https://docs.apify.com/api/v2/key-value-store-keys-get) --- From db006c37f8417f7367f9671f28f163c73c6fb23c Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 7 Jul 2026 13:11:59 +0200 Subject: [PATCH 08/46] feat: add exitCode to run output Write { stdout, stderr, exitCode } instead of { stdout, stderr }. exitCode is the user script's effective status: 0 when it returns normally, 1 when it throws. The Actor run itself still SUCCEEDS on a throw, so callers detect a failed script via exitCode rather than heuristics on stderr (console.error / console.warn are legitimate log channels). Update README and API docs. --- README.md | 11 +++++++---- docs/API.md | 17 ++++++++++++----- worker/runner.js | 9 ++++++++- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 085eb7e..c3343e1 100644 --- a/README.md +++ b/README.md @@ -53,14 +53,17 @@ For full configuration options, use the configurator at ## Output -A single **dataset item** with the captured streams: +A single **dataset item** with the captured streams and the script's exit status: ```json -{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "" } +{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "", "exitCode": 0 } ``` -If the script throws, the error lands in `stderr`; `stdout` keeps whatever was -printed before the failure. +If the script throws, the error lands in `stderr`, `stdout` keeps whatever was +printed before the failure, and `exitCode` is `1`. The Actor run itself still +**succeeds** — `exitCode` is the reliable signal for a failed script (`0` = the +script returned normally, `1` = it threw), since `stderr` is also a legitimate +log channel (`console.error` / `console.warn`). ## Permissions & safety diff --git a/docs/API.md b/docs/API.md index 3347393..c746fb1 100644 --- a/docs/API.md +++ b/docs/API.md @@ -363,15 +363,22 @@ isTruncated, exclusiveStartKey, nextExclusiveStartKey }`. | `console.error`, `console.warn` | **stderr** | Non-string arguments are `JSON.stringify`'d. When the script finishes, both -streams are written to the run's default dataset as a single item: +streams and the script's exit status are written to the run's default dataset as +a single item: ```json -{ "stdout": "...", "stderr": "..." } +{ "stdout": "...", "stderr": "...", "exitCode": 0 } ``` +`exitCode` is the script's effective exit status: `0` when it returns normally, +`1` when it throws. It is distinct from the Actor run's status — see below. + ## Error handling - A non-2xx API response throws `Error: failed: `. -- If your script throws, the error (stack/message) is appended to **stderr** and - the run still **succeeds** with whatever was printed beforehand — so failures - are observable in the output rather than crashing the run. +- If your script throws, the error (stack/message) is appended to **stderr**, + `exitCode` is set to `1`, and the run still **succeeds** with whatever was + printed beforehand — so failures are observable in the output rather than + crashing the run. Check `exitCode` (not `stderr`) to detect a failed script: + `stderr` may be non-empty from ordinary `console.error` / `console.warn` + logging even on a successful run. diff --git a/worker/runner.js b/worker/runner.js index 18b57ac..c7b192e 100644 --- a/worker/runner.js +++ b/worker/runner.js @@ -274,13 +274,20 @@ export default { // A thrown program is a user-level failure: capture it in stderr and still // push the output, so the run SUCCEEDS with diagnostics. Infra failures // (missing env, dataset push) throw and fail the run. + // + // exitCode is the user script's effective status, distinct from the Actor run's + // status: 0 when the script returns normally, 1 when it throws. The run itself + // still SUCCEEDS on a throw, so callers detect a failed script via this field + // rather than heuristics on stderr (console.error is a legitimate log channel). + let exitCode = 0; try { await run(makeApifyBinding(token, apiV2), captureConsole); } catch (err) { stderr.push(err?.stack ?? err?.message ?? String(err)); + exitCode = 1; } - await pushOutput(apiV2, token, env, { stdout: stdout.join('\n'), stderr: stderr.join('\n') }); + await pushOutput(apiV2, token, env, { stdout: stdout.join('\n'), stderr: stderr.join('\n'), exitCode }); return Response.json({ ok: true }); }, }; From 6486cd0e79ed16067cc3b55bc752ac210a4925e4 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 7 Jul 2026 14:27:45 +0200 Subject: [PATCH 09/46] fix: remove nodejs_compat to close sandbox egress and token leaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workerd nodejs_compat flag exposed Node built-ins to user code, which broke the sandbox's egress boundary two ways (ai-team#216): - node:net gave a raw-socket egress path that bypassed guard.js's *.apify.com fetch allowlist — arbitrary public hosts were reachable (finding A). - process.env exposed the run's APIFY_TOKEN to user code (finding B). It also made the docs' 'no imports' claim false (finding C). runner.js and guard.js use only web-standard APIs (fetch/URL/Response/ Uint8Array), so dropping the flag needs no code changes. Without it, node:* imports fail and process/require are undefined, while fetch and the apify binding keep working. Add tests/sandbox-isolation.js (run via test.sh, same deploy+call pattern as binding-smoke.js) asserting node builtins are blocked, process/require are undefined, fetch works, guard.js still blocks non-apify hosts, and the apify binding still works. Verified on the deployed Actor: isolation 9/9, binding smoke 18/18. Update docs to state no imports are available. --- README.md | 11 ++++++-- test.sh | 37 ++++++++++++++----------- tests/sandbox-isolation.js | 55 ++++++++++++++++++++++++++++++++++++++ worker/config.capnp | 6 ++++- 4 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 tests/sandbox-isolation.js diff --git a/README.md b/README.md index c3343e1..9cd014e 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,10 @@ For full configuration options, use the configurator at - **One script per run.** The Actor reads your `code`, runs it once, writes the result, and exits. - The code runs inside a [`workerd`](https://github.com/cloudflare/workerd) V8 - isolate: **no filesystem, no package imports**, and outbound network is - restricted to `*.apify.com`. + isolate: **no imports** — neither npm packages nor Node built-in `node:*` + modules are available (`import`/`require` of any module fails); web-standard + globals such as `fetch` are present. Outbound network is restricted to + `*.apify.com`. - Inside the script a global **`apify`** object exposes a small, typed subset of the Apify API — run Actors, read/write datasets and key-value stores — using the current run's token (see below). @@ -69,6 +71,11 @@ log channel (`console.error` / `console.warn`). - Runs with **limited permissions**: the sandbox has no filesystem and can reach only the Apify API (`*.apify.com`). +- **No imports.** The isolate runs without workerd's `nodejs_compat`, so user + code cannot import Node built-ins (`node:net`, `node:fs`, …) or npm packages. + This removes `node:net` — a raw-socket egress path that would otherwise bypass + the `fetch` allowlist — and keeps the run token out of `process.env` (which is + not defined). - Each run is an isolated, single-use container — nothing persists between runs. ## The `apify` binding diff --git a/test.sh b/test.sh index 6d72c0d..ed09ad5 100755 --- a/test.sh +++ b/test.sh @@ -1,13 +1,15 @@ #!/bin/sh -# Deploy the Actor (apify push) and run the binding smoke test on the freshly -# built version via `apify call`. Exits non-zero if any binding check fails. +# Deploy the Actor (apify push) once, then run each test probe on the freshly +# built version via `apify call`. Each probe is a script submitted as the `code` +# input; it prints ALL_TESTS_PASSED on success. Exits non-zero if any probe fails. # # Usage: ./test.sh set -eu cd "$(dirname "$0")" -TEST_JS="tests/binding-smoke.js" +# Probes run against the built Actor. Add a file here to register a new probe. +PROBES="tests/binding-smoke.js tests/sandbox-isolation.js" command -v apify >/dev/null 2>&1 || { echo "apify CLI not found" >&2; exit 1; } command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 1; } @@ -18,16 +20,19 @@ trap 'rm -f "$input_json"' EXIT echo "==> apify push" apify push -echo "==> building input from ${TEST_JS}" -jq -n --arg code "$(cat "$TEST_JS")" '{ code: $code }' > "$input_json" - -echo "==> apify call (running the test on the built Actor)" -output="$(apify call -f "$input_json" -o 2>&1)" -echo "$output" - -if printf '%s' "$output" | grep -q 'ALL_TESTS_PASSED'; then - echo "==> all binding tests passed" -else - echo "==> binding tests FAILED" >&2 - exit 1 -fi +failed=0 +for probe in $PROBES; do + echo "==> apify call: ${probe}" + jq -n --arg code "$(cat "$probe")" '{ code: $code }' > "$input_json" + output="$(apify call -f "$input_json" -o 2>&1)" + echo "$output" + if printf '%s' "$output" | grep -q 'ALL_TESTS_PASSED'; then + echo "==> ${probe} passed" + else + echo "==> ${probe} FAILED" >&2 + failed=1 + fi +done + +[ "$failed" -eq 0 ] || { echo "==> some probes FAILED" >&2; exit 1; } +echo "==> all probes passed" diff --git a/tests/sandbox-isolation.js b/tests/sandbox-isolation.js new file mode 100644 index 0000000..c091059 --- /dev/null +++ b/tests/sandbox-isolation.js @@ -0,0 +1,55 @@ +// Sandbox isolation test. Submitted as the Actor's `code` input by test.sh and +// executed on the built Actor via `apify call`. Asserts the sandbox boundary +// holds and prints a sentinel line (ALL_TESTS_PASSED) that test.sh greps for. +// +// Guards github.com/apify/ai-team#216 (findings A, B, C): the isolate runs +// WITHOUT workerd's nodejs_compat, so user code cannot reach Node built-ins. +// That removes node:net (a raw-socket egress path that bypassed guard.js's +// *.apify.com fetch allowlist — finding A) and process.env (which held the +// run's APIFY_TOKEN — finding B), and makes the "no imports" docs accurate +// (finding C). fetch() and the apify binding must still work. +const results = []; +function check(name, cond, detail = '') { + if (cond) { + console.log(`PASS ${name}: ${detail}`); + results.push(true); + } else { + console.error(`FAIL ${name}: ${detail}`); + results.push(false); + } +} + +// Node built-ins must NOT be importable (no nodejs_compat). +for (const mod of ['node:net', 'node:fs', 'node:dns', 'node:child_process']) { + let imported = false; + try { await import(mod); imported = true; } catch { /* expected */ } + check(`import ${mod} blocked`, !imported, imported ? 'IMPORTED (leak!)' : 'blocked'); +} + +// The run token lives in process.env under nodejs_compat; without it, process +// and require must be undefined so user code can't read the credential. +check('process undefined', typeof process === 'undefined', `typeof process = ${typeof process}`); +check('require undefined', typeof require === 'undefined', `typeof require = ${typeof require}`); + +// fetch must remain — the apify binding depends on it. +check('fetch available', typeof fetch === 'function', `typeof fetch = ${typeof fetch}`); + +// guard.js must still block a non-apify host over fetch. +let nonApifyBlocked = false; +try { await fetch('https://example.com'); } catch { nonApifyBlocked = true; } +check('non-apify fetch blocked', nonApifyBlocked, nonApifyBlocked ? 'blocked' : 'REACHED example.com (leak!)'); + +// The apify binding must still work (fetch to *.apify.com). +let bindingWorks = false; +try { + const found = await apify.actor.search({ query: 'hello world', limit: 1 }); + bindingWorks = Array.isArray(found); +} catch (e) { + console.error(`apify.actor.search threw: ${e.message}`); +} +check('apify binding works', bindingWorks, bindingWorks ? 'actor.search ok' : 'binding broken'); + +const passed = results.filter(Boolean).length; +console.log(`\n=== SUMMARY: ${passed}/${results.length} passed ===`); +if (passed === results.length) console.log('ALL_TESTS_PASSED'); +else console.error(`SOME_TESTS_FAILED (${results.length - passed} failed)`); diff --git a/worker/config.capnp b/worker/config.capnp index e7288de..cad84c8 100644 --- a/worker/config.capnp +++ b/worker/config.capnp @@ -35,5 +35,9 @@ const codeRuntime :Workerd.Worker = ( ], globalOutbound = "internet", compatibilityDate = "2026-01-15", - compatibilityFlags = ["nodejs_compat"], + # No nodejs_compat: user code runs with web-standard APIs only. This is a + # security boundary, not a convenience toggle — the flag would expose node:net + # (a raw-socket egress path that bypasses guard.js's *.apify.com fetch allowlist) + # and process.env (which holds the run's APIFY_TOKEN). runner.js and guard.js use + # only web-standard APIs (fetch/URL/Response/Uint8Array), so they need nothing here. ); From 7a091e247d74130891946201cb77072b297d4866 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 7 Jul 2026 14:33:33 +0200 Subject: [PATCH 10/46] test: assert fetch allowlist covers apify.com + subdomains only Expand the sandbox isolation probe: positively verify apify.com and an *.apify.com subdomain are allowed, and that guard.js blocks unrelated hosts, subdomain/userinfo look-alikes (evilapify.com, apify.com.evil.com, apify.com@evil.com), and the cloud metadata IP. Classify by the guard's 'Blocked fetch' rejection, not by request success. Verified on the deployed Actor: 15/15. --- tests/sandbox-isolation.js | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/sandbox-isolation.js b/tests/sandbox-isolation.js index c091059..6f8cfe0 100644 --- a/tests/sandbox-isolation.js +++ b/tests/sandbox-isolation.js @@ -34,10 +34,36 @@ check('require undefined', typeof require === 'undefined', `typeof require = ${t // fetch must remain — the apify binding depends on it. check('fetch available', typeof fetch === 'function', `typeof fetch = ${typeof fetch}`); -// guard.js must still block a non-apify host over fetch. -let nonApifyBlocked = false; -try { await fetch('https://example.com'); } catch { nonApifyBlocked = true; } -check('non-apify fetch blocked', nonApifyBlocked, nonApifyBlocked ? 'blocked' : 'REACHED example.com (leak!)'); +// guard.js allowlist: apify.com and *.apify.com only. A guard rejection throws +// synchronously with a "Blocked fetch" message BEFORE any network I/O; anything +// else (a real network/HTTP error) means guard let the request through. So we +// classify by the error message, not by whether the request ultimately succeeds. +async function guardBlocks(url) { + try { + await fetch(url); + return false; // request went out — guard allowed it + } catch (e) { + return /Blocked fetch/.test(e.message); // guard rejection vs. network error + } +} + +// Allowed: the main domain and any subdomain must NOT be guard-blocked. +for (const url of ['https://apify.com/', 'https://api.apify.com/v2/browser-info']) { + check(`allow ${url}`, !(await guardBlocks(url)), 'not blocked by guard'); +} + +// Blocked: other public hosts, subdomain look-alikes, userinfo/host tricks, and +// the cloud metadata IP must all be guard-blocked. +const blockedTargets = [ + 'https://example.com/', // unrelated public host + 'https://evilapify.com/', // suffix without the dot — must not match .apify.com + 'https://apify.com.evil.com/', // real host is evil.com + 'https://apify.com@evil.com/', // userinfo trick — real host is evil.com + 'http://169.254.169.254/', // cloud link-local metadata +]; +for (const url of blockedTargets) { + check(`block ${url}`, await guardBlocks(url), 'blocked by guard'); +} // The apify binding must still work (fetch to *.apify.com). let bindingWorks = false; From 2493f54b0f25fa427f79f42c7e63d2348c7d8985 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 7 Jul 2026 16:55:28 +0200 Subject: [PATCH 11/46] fix: block WebSocket and EventSource egress in guard.js Removing nodejs_compat closed the node:net egress path, but WebSocket and EventSource are web-standard globals present without that flag and connect directly, not through the fetch guard. A script could open a wss:// or SSE channel to any public host and exfiltrate data around the *.apify.com allowlist (apify/ai-team#216 finding A, second primitive). runner.js and the apify binding use only fetch, so neutralize both globals (non-configurable throwing stubs). This closes the JS egress surface: fetch is allowlisted, WebSocket/EventSource are removed, and raw sockets need module imports that are already blocked. Extend the isolation probe with wss:// and SSE egress cases. Verified on the deployed Actor: isolation 17/17, binding 18/18. --- tests/sandbox-isolation.js | 17 +++++++++++++++++ worker/guard.js | 32 ++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/tests/sandbox-isolation.js b/tests/sandbox-isolation.js index 6f8cfe0..fda8b3d 100644 --- a/tests/sandbox-isolation.js +++ b/tests/sandbox-isolation.js @@ -65,6 +65,23 @@ for (const url of blockedTargets) { check(`block ${url}`, await guardBlocks(url), 'blocked by guard'); } +// Non-fetch egress primitives must be neutralized: WebSocket and EventSource +// are web-standard globals that connect directly (not through the fetch guard), +// so a script could otherwise open a wss:// / SSE channel to any public host and +// exfiltrate around the *.apify.com allowlist (apify/ai-team#216 finding A). +function blocksConstruct(name, url) { + const Ctor = globalThis[name]; + if (typeof Ctor !== 'function') return true; // absent → not an egress path + try { + new Ctor(url); + return false; // constructed → egress opened + } catch (e) { + return /Blocked/.test(e.message); // our guard rejection vs. any other error + } +} +check('WebSocket blocked', blocksConstruct('WebSocket', 'wss://echo.websocket.org'), 'no wss egress'); +check('EventSource blocked', blocksConstruct('EventSource', 'https://example.com/sse'), 'no SSE egress'); + // The apify binding must still work (fetch to *.apify.com). let bindingWorks = false; try { diff --git a/worker/guard.js b/worker/guard.js index 426a170..2d2d6e8 100644 --- a/worker/guard.js +++ b/worker/guard.js @@ -1,11 +1,16 @@ -// Restrict the user program's global fetch() to apify.com and its subdomains. -// Imported before usercode.js so the override is in place even for code that +// Restrict the user program's outbound network to apify.com and its subdomains. +// Imported before usercode.js so the overrides are in place even for code that // runs at module-evaluation time. Our own Apify API calls use the exported // realFetch (the internal API is a private IP, not *.apify.com), so they are // unaffected by this guard. // -// NOTE: this guards the fetch() API only. It is not a complete egress boundary — -// the airtight control is workerd's globalOutbound. See SECURITY notes in the repo. +// Egress surface (workerd, no nodejs_compat): the only JS-reachable outbound +// primitives are fetch, WebSocket, and EventSource. Raw sockets (node:net, +// cloudflare:sockets connect()) need module imports, which are already blocked. +// fetch is allowlisted below; WebSocket and EventSource are removed outright +// because runner.js and the apify binding never use them — leaving them would +// be a non-fetch egress path around the allowlist (apify/ai-team#216 finding A, +// via WebSocket). If a future need arises, wrap them like fetch instead. const realFetch = globalThis.fetch.bind(globalThis); @@ -42,4 +47,23 @@ globalThis.fetch = (input, init) => { return realFetch(input, init); }; +// Remove the non-fetch egress primitives. These are web-standard globals present +// even without nodejs_compat, and they connect directly (not through the fetch +// guard), so a script could otherwise open a wss:// or SSE connection to any +// public host and exfiltrate data around the *.apify.com allowlist. +function blockGlobal(name) { + const blocked = function () { + throw new Error(`Blocked ${name}: only fetch() to apify.com and its subdomains is allowed`); + }; + Object.defineProperty(globalThis, name, { + value: blocked, + writable: false, + configurable: false, + enumerable: false, + }); +} +for (const name of ['WebSocket', 'EventSource']) { + if (name in globalThis) blockGlobal(name); +} + export { realFetch }; From 5461aeec650cf3c3af225f7356f37337c1c65953 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 14:12:56 +0200 Subject: [PATCH 12/46] fix: close realFetch allowlist bypass via one-shot claim in guard.js guard.js exported realFetch as a standing module binding. Since ES modules are singleton-cached, the sandboxed user script could recover it at runtime via `(await import('./guard.js')).realFetch(...)`, fully bypassing the *.apify.com fetch allowlist (and reaching Apify's private network, since the outbound service allows public/private/local). Reopened exactly the egress class apify/ai-team#216 was meant to close. guard.js now exposes claimRealFetch(), a one-shot accessor that hands out the real fetch once and nulls its internal reference. runner.js claims it during its own module load, strictly before usercode.js can run. A later import('./guard.js') from inside the script reaches the same cached module instance, but the value is already gone. Verified against real workerd, before/after, same PoC: - before: guard.js exports ["realFetch"] -> BYPASS SUCCEEDED, exfil request logged on a disallowed listener. - after: guard.js exports ["claimRealFetch"] -> BYPASS FAILED (null), zero hits on the disallowed listener. --- worker/guard.js | 24 +++++++++++++++++++----- worker/runner.js | 11 +++++++++-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/worker/guard.js b/worker/guard.js index 2d2d6e8..b14ae12 100644 --- a/worker/guard.js +++ b/worker/guard.js @@ -1,8 +1,9 @@ // Restrict the user program's outbound network to apify.com and its subdomains. // Imported before usercode.js so the overrides are in place even for code that -// runs at module-evaluation time. Our own Apify API calls use the exported -// realFetch (the internal API is a private IP, not *.apify.com), so they are -// unaffected by this guard. +// runs at module-evaluation time. Our own Apify API calls use the real, +// unrestricted fetch (the internal API is a private IP, not *.apify.com) — +// see claimRealFetch() below for how runner.js gets it without leaving it +// reachable from user code. // // Egress surface (workerd, no nodejs_compat): the only JS-reachable outbound // primitives are fetch, WebSocket, and EventSource. Raw sockets (node:net, @@ -14,6 +15,21 @@ const realFetch = globalThis.fetch.bind(globalThis); +// One-shot handoff of the unrestricted fetch to runner.js. ES modules are +// evaluated once and cached, so `guard.js` is the same module instance no +// matter who imports it. runner.js imports this module (and calls +// claimRealFetch()) before usercode.js is ever imported, so it always claims +// first. If the sandboxed script later does `await import('./guard.js')` to +// try to recover the unrestricted fetch, it gets this same cached instance — +// but the value is already gone. A standing `export { realFetch }` would hand +// it to that later import too; don't reintroduce one. +let unclaimedRealFetch = realFetch; +export function claimRealFetch() { + const fetchFn = unclaimedRealFetch; + unclaimedRealFetch = null; + return fetchFn; +} + // Match apify.com exactly or any subdomain. The leading dot in the suffix is // what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` // (ends with `.evil.com`) both fail. @@ -65,5 +81,3 @@ function blockGlobal(name) { for (const name of ['WebSocket', 'EventSource']) { if (name in globalThis) blockGlobal(name); } - -export { realFetch }; diff --git a/worker/runner.js b/worker/runner.js index c7b192e..2e6b1b8 100644 --- a/worker/runner.js +++ b/worker/runner.js @@ -8,10 +8,17 @@ // Loader / per-request isolate is needed — the program runs in this worker, // which is itself the sandbox (no filesystem, restricted outbound network). // guard.js must be imported before usercode.js: it overrides globalThis.fetch -// to allow only apify.com, and exports realFetch for our own (internal) API calls. -import { realFetch } from './guard.js'; +// to allow only apify.com, and hands us the real, unrestricted fetch via a +// one-shot claimRealFetch() for our own (internal) API calls — see guard.js +// for why this is a claim, not a standing export. +import { claimRealFetch } from './guard.js'; import { run } from './usercode.js'; +// Must run before usercode.js's `run()` is ever invoked (it does, here — module +// evaluation order puts this ahead of any dynamic import from inside `run()`). +const realFetch = claimRealFetch(); +if (!realFetch) throw new Error('realFetch already claimed — guard.js imported out of order.'); + const DEFAULT_ITERATE_BATCH = 1000; const DEFAULT_GET_SCHEMA_SAMPLE = 5; From c36f402eb3d276438ece76b963d6e94b0b91b97a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 14:16:10 +0200 Subject: [PATCH 13/46] refactor: rename single-letter locals and de-duplicate the loopback port runner.js: r -> response, d -> page, [k, v] -> [key, value], ct -> contentType / resolvedContentType. Matches code-quality and apify-coding-standards naming rules (no single-letter/abbreviated locals). config.capnp + entrypoint.sh both hardcoded the loopback port 8787 independently. config.capnp now takes a __PORT__ placeholder that entrypoint.sh substitutes from its own $PORT before starting workerd \u2014 one source of truth. --- worker/config.capnp | 4 +++- worker/entrypoint.sh | 3 +++ worker/runner.js | 48 ++++++++++++++++++++++---------------------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/worker/config.capnp b/worker/config.capnp index cad84c8..c5d380b 100644 --- a/worker/config.capnp +++ b/worker/config.capnp @@ -9,10 +9,12 @@ const config :Workerd.Config = ( # the hostname allowlist is enforced by guard.js, not here. (name = "internet", network = (allow = ["public", "private", "local"], tlsOptions = (trustBrowserCas = true))), ], + # __PORT__ is substituted by entrypoint.sh from its own $PORT before workerd + # starts — single source of truth, see entrypoint.sh. sockets = [ ( name = "http", - address = "127.0.0.1:8787", + address = "127.0.0.1:__PORT__", http = (), service = "main", ), diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh index 27892bd..7f9b473 100755 --- a/worker/entrypoint.sh +++ b/worker/entrypoint.sh @@ -35,6 +35,9 @@ fi printf '\n}\n' } > /app/worker/usercode.js +# config.capnp hardcodes __PORT__ as a placeholder so the port has one source ($PORT above). +sed -i "s/__PORT__/${PORT}/" /app/worker/config.capnp + /usr/local/bin/workerd serve --experimental /app/worker/config.capnp & workerd_pid=$! trap 'kill "$workerd_pid" 2>/dev/null || true' EXIT diff --git a/worker/runner.js b/worker/runner.js index 2e6b1b8..7ca82b4 100644 --- a/worker/runner.js +++ b/worker/runner.js @@ -34,8 +34,8 @@ function makeApifyBinding(token, apiV2) { const buildUrl = (path, searchParams) => { const url = new URL(`${apiV2}${path}`); if (searchParams) { - for (const [k, v] of Object.entries(searchParams)) { - if (v !== undefined && v !== null) url.searchParams.set(k, String(v)); + for (const [key, value] of Object.entries(searchParams)) { + if (value !== undefined && value !== null) url.searchParams.set(key, String(value)); } } return url; @@ -50,9 +50,9 @@ function makeApifyBinding(token, apiV2) { init.body = isRaw ? body : JSON.stringify(body); init.headers['content-type'] = contentType ?? (isRaw ? 'application/octet-stream' : 'application/json'); } - const r = await realFetch(buildUrl(path, searchParams), init); - if (!r.ok) throw new Error(`${method} ${path} failed: ${r.status} ${await r.text()}`); - return r; + const response = await realFetch(buildUrl(path, searchParams), init); + if (!response.ok) throw new Error(`${method} ${path} failed: ${response.status} ${await response.text()}`); + return response; }; const apiJson = async (...args) => (await apiCall(...args)).json(); @@ -62,7 +62,7 @@ function makeApifyBinding(token, apiV2) { // GET /v2/store — Apify Store search. Returns the items array directly. search: ({ query, limit, category }) => apiData('GET', '/store', { searchParams: { search: query, limit, category } }) - .then((d) => d.items), + .then((page) => page.items), getDetails: ({ actorId }) => apiData('GET', `/acts/${encodeURIComponent(actorId)}`), @@ -115,8 +115,8 @@ function makeApifyBinding(token, apiV2) { // Returns the full run log as text. `limit` tails the last N characters; the Apify API // does not paginate logs, so this is a client-side slice (the full body is fetched). getLog: async ({ runId, limit }) => { - const r = await apiCall('GET', `/logs/${encodeURIComponent(runId)}`); - const text = await r.text(); + const response = await apiCall('GET', `/logs/${encodeURIComponent(runId)}`); + const text = await response.text(); return limit && text.length > limit ? text.slice(-limit) : text; }, }; @@ -127,7 +127,7 @@ function makeApifyBinding(token, apiV2) { // (eventually consistent), so we don't surface a `total`. Use `getSchema` if you // need an item count, or iterate to consume the whole dataset. listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }) => { - const r = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { + const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { searchParams: { fields: fields?.join(','), omit: omit?.join(','), @@ -137,7 +137,7 @@ function makeApifyBinding(token, apiV2) { desc: desc ? '1' : undefined, }, }); - return r.json(); + return response.json(); }, // Async generator over the entire dataset. Pages internally in `batchSize` chunks @@ -203,34 +203,34 @@ function makeApifyBinding(token, apiV2) { // Returns null when the key does not exist (404), not an error — this matches the common // "lookup or default" pattern in code. get: async ({ storeId, key }) => { - const r = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { + const response = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { headers: baseHeaders, }); - if (r.status === 404) return null; - if (!r.ok) throw new Error(`GET kvs.get failed: ${r.status} ${await r.text()}`); - const ct = r.headers.get('content-type') ?? ''; - if (ct.includes('application/json')) return r.json(); - if (ct.startsWith('text/')) return r.text(); - return new Uint8Array(await r.arrayBuffer()); + if (response.status === 404) return null; + if (!response.ok) throw new Error(`GET kvs.get failed: ${response.status} ${await response.text()}`); + const contentType = response.headers.get('content-type') ?? ''; + if (contentType.includes('application/json')) return response.json(); + if (contentType.startsWith('text/')) return response.text(); + return new Uint8Array(await response.arrayBuffer()); }, // `value`: object → application/json; string → text/plain; Uint8Array/ArrayBuffer → // application/octet-stream (or whatever the caller passed via `contentType`). set: async ({ storeId, key, value, contentType }) => { let body; - let ct = contentType; + let resolvedContentType = contentType; if (value instanceof Uint8Array || value instanceof ArrayBuffer) { body = value; - ct = ct ?? 'application/octet-stream'; + resolvedContentType = resolvedContentType ?? 'application/octet-stream'; } else if (typeof value === 'string') { body = value; - ct = ct ?? 'text/plain; charset=utf-8'; + resolvedContentType = resolvedContentType ?? 'text/plain; charset=utf-8'; } else { body = JSON.stringify(value); - ct = ct ?? 'application/json; charset=utf-8'; + resolvedContentType = resolvedContentType ?? 'application/json; charset=utf-8'; } await apiCall('PUT', `/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`, { - body, contentType: ct, + body, contentType: resolvedContentType, }); }, @@ -250,12 +250,12 @@ function makeApifyBinding(token, apiV2) { async function pushOutput(apiV2, token, env, item) { const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; if (!datasetId) throw new Error('Default dataset ID missing from Actor run environment.'); - const r = await realFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { + const response = await realFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json; charset=utf-8' }, body: JSON.stringify(item), }); - if (!r.ok) throw new Error(`Failed to push dataset item: ${r.status} ${await r.text()}`); + if (!response.ok) throw new Error(`Failed to push dataset item: ${response.status} ${await response.text()}`); } export default { From 4607d26c8aede1891d8c70e39ec5ab94b8535f2d Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 14:52:58 +0200 Subject: [PATCH 14/46] fix: close redirect-following allowlist bypass, lock globalThis.fetch guard.js validated only the initial URL; fetch defaults to redirect:'follow', so an allowlisted *.apify.com host issuing a 3xx to any other host was followed out silently. Verified live against real workerd: an allowed host redirecting to a disallowed one reached it (before) / was blocked (after). Now fetches with redirect:'manual' and re-validates each Location against the allowlist before following, one hop at a time (capped at 5), with WHATWG-spec method/body downgrade rules (303 always -> GET; 301/302 -> GET only if the original method was POST; 307/308 preserve method + body). Also lock globalThis.fetch via Object.defineProperty(writable:false, configurable:false), matching the existing WebSocket/EventSource treatment (a plain assignment could be reassigned/deleted by the sandboxed script). Verified live: reassigning globalThis.fetch from a script now throws TypeError instead of succeeding. --- worker/guard.js | 48 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/worker/guard.js b/worker/guard.js index b14ae12..662fddb 100644 --- a/worker/guard.js +++ b/worker/guard.js @@ -45,7 +45,9 @@ function requestUrl(input) { return String(input); } -globalThis.fetch = (input, init) => { +// Parses and validates one URL against the allowlist. Returns the parsed URL +// (callers use it to resolve a relative redirect Location) or throws. +function validateUrl(input) { let url; try { // Parse to the real host — defeats userinfo (`apify.com@evil.com`), @@ -60,8 +62,48 @@ globalThis.fetch = (input, init) => { if (!isAllowedHost(url.hostname)) { throw new Error(`Blocked fetch to "${url.hostname}": only apify.com and its subdomains are allowed`); } - return realFetch(input, init); -}; + return url; +} + +// fetch() follows redirects internally by default, invisibly to a wrapper that +// only checks the initial URL — an allowlisted host could 302 to anywhere. +// Follow redirects ourselves, one hop at a time, and re-validate each Location +// against the allowlist before following it. Status-to-method mapping matches +// the WHATWG fetch spec: 303 always downgrades to GET; 301/302 downgrade to +// GET only when the original method was POST; 307/308 preserve method + body. +const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); +const MAX_REDIRECT_HOPS = 5; + +function nextRedirectInit(init, status) { + const method = (init?.method ?? 'GET').toUpperCase(); + const downgradeToGet = status === 303 || ((status === 301 || status === 302) && method === 'POST'); + if (!downgradeToGet) return init; + return { ...init, method: 'GET', body: undefined }; +} + +async function guardedFetch(input, init, hop = 0) { + if (hop > MAX_REDIRECT_HOPS) { + throw new Error(`Blocked fetch: exceeded ${MAX_REDIRECT_HOPS} redirects`); + } + const url = validateUrl(input); + const response = await realFetch(input, { ...init, redirect: 'manual' }); + if (!REDIRECT_STATUSES.has(response.status)) return response; + const location = response.headers.get('location'); + if (!location) return response; // redirect status with no Location: nothing to follow + const nextUrl = new URL(location, url); // resolves a relative Location against the current URL + return guardedFetch(nextUrl.href, nextRedirectInit(init, response.status), hop + 1); +} + +// writable:false + configurable:false, matching blockGlobal() below — a plain +// assignment could be overwritten or deleted by the sandboxed script to +// recover the ambient (real, unrestricted) fetch reference some engines +// expose under a different name; locking it closes that off. +Object.defineProperty(globalThis, 'fetch', { + value: (input, init) => guardedFetch(input, init), + writable: false, + configurable: false, + enumerable: true, +}); // Remove the non-fetch egress primitives. These are web-standard globals present // even without nodejs_compat, and they connect directly (not through the fetch From 11e7c968af0e686b9b1d9090bd6e48092d593979 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 14:55:13 +0200 Subject: [PATCH 15/46] fix: catch usercode.js compile failures, scope run.abort, freeze bindings usercode.js wraps the user's code inside 'export async function run(...) { ... }' with nothing else at module scope, so a syntax error in it fails module evaluation before any request reaches runner.js's try/catch \u2014 workerd exits immediately (verified: raw workerd binary, exit code 1, no /health response, ever). The reviewer's proposed fix (dynamic import inside the try/catch) does NOT help: verified live that workerd eagerly evaluates every module declared in config.capnp regardless of static vs dynamic import, so both crash identically. The actual fix has to live in entrypoint.sh, which is what this commit does: - entrypoint.sh now captures workerd's stderr and, if workerd exits before becoming ready, checks whether the crash names usercode.js. If so (this is structurally exact, not a heuristic, given usercode.js's shape above) it pushes a { stdout: '', stderr, exitCode: 1, statusMessage: 'Failed to compile: ...' } item directly and exits 0, so the run SUCCEEDS with diagnostics instead of failing as an opaque infra error. Any other crash (our own runner.js/guard.js, config issue) still hard-fails the run. Verified live for both the positive (usercode.js) and negative (unrelated file) cases. - runner.js adds the same statusMessage field ('Script completed' / 'Script threw: ...') to the normal exitCode 0/1 path, so callers get a prose signal without branching on exitCode. - run.abort({ runId }) is now scoped to run IDs this script itself started. actor.run()/actor.start() share one createRun() helper that records each created run's ID in a Set; abort() throws on an unrecognized runId instead of silently no-op'ing (a script shouldn't think an abort succeeded when it didn't). Previously any account-wide runId could be aborted. Verified live. - console and the apify binding's namespaces are now Object.freeze'd so a script can't reassign e.g. console.log to corrupt its own output capture. Verified live: reassignment now throws TypeError. - entrypoint.sh also validates the default dataset ID up front (needed by the new compile-failure push path), matching the existing token/KV-store check. --- worker/entrypoint.sh | 45 ++++++++++++++++--- worker/runner.js | 105 ++++++++++++++++++++++++++++--------------- 2 files changed, 109 insertions(+), 41 deletions(-) diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh index 7f9b473..bcdb6ce 100755 --- a/worker/entrypoint.sh +++ b/worker/entrypoint.sh @@ -12,10 +12,12 @@ READINESS_ATTEMPTS=100 # 100 * 0.1s = 10s budget for workerd to bind the socke API_BASE="${APIFY_API_BASE_URL:-https://api.apify.com}" API_BASE="${API_BASE%/}" # APIFY_API_BASE_URL ships with a trailing slash STORE_ID="${ACTOR_DEFAULT_KEY_VALUE_STORE_ID:-${APIFY_DEFAULT_KEY_VALUE_STORE_ID:-}}" +DATASET_ID="${ACTOR_DEFAULT_DATASET_ID:-${APIFY_DEFAULT_DATASET_ID:-}}" INPUT_KEY="${APIFY_INPUT_KEY:-INPUT}" +WORKERD_STDERR=/tmp/workerd.err -if [ -z "${APIFY_TOKEN:-}" ] || [ -z "$STORE_ID" ]; then - echo "[code-runtime] missing APIFY_TOKEN or default key-value store ID" >&2 +if [ -z "${APIFY_TOKEN:-}" ] || [ -z "$STORE_ID" ] || [ -z "$DATASET_ID" ]; then + echo "[code-runtime] missing APIFY_TOKEN or default key-value store / dataset ID" >&2 exit 1 fi @@ -38,12 +40,44 @@ fi # config.capnp hardcodes __PORT__ as a placeholder so the port has one source ($PORT above). sed -i "s/__PORT__/${PORT}/" /app/worker/config.capnp -/usr/local/bin/workerd serve --experimental /app/worker/config.capnp & +/usr/local/bin/workerd serve --experimental /app/worker/config.capnp 2>"$WORKERD_STDERR" & workerd_pid=$! trap 'kill "$workerd_pid" 2>/dev/null || true' EXIT +# push_compile_failure reports a usercode.js syntax error as a normal, SUCCEEDED script +# result (same contract as a script that throws at runtime) instead of failing the whole +# Actor run. usercode.js wraps the user's `code` inside `export async function run(...) { +# ... }` with nothing else at module scope, so nothing in it can fail to *parse* except +# that inserted code — a startup crash naming usercode.js is therefore always a syntax +# error in the user's script, never our own code. See detection below. +push_compile_failure() { + crash_log=$(cat "$WORKERD_STDERR" 2>/dev/null || true) + echo "[code-runtime] usercode.js failed to compile: $crash_log" >&2 + item=$(jq -n --arg stderr "$crash_log" \ + '{stdout: "", stderr: $stderr, exitCode: 1, statusMessage: ("Failed to compile: " + $stderr)}') + push_status=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + "${API_BASE}/v2/datasets/${DATASET_ID}/items" \ + -H "Authorization: Bearer ${APIFY_TOKEN}" \ + -H 'content-type: application/json; charset=utf-8' \ + --data-binary "$item") + case "$push_status" in + 2??) exit 0 ;; + *) + echo "[code-runtime] failed to push compile-failure diagnostic (HTTP $push_status)" >&2 + exit 1 + ;; + esac +} + attempt=0 until curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; do + if ! kill -0 "$workerd_pid" 2>/dev/null; then + if grep -q 'usercode\.js' "$WORKERD_STDERR" 2>/dev/null; then + push_compile_failure + fi + echo "[code-runtime] workerd exited before startup: $(cat "$WORKERD_STDERR" 2>/dev/null)" >&2 + exit 1 + fi attempt=$((attempt + 1)) if [ "$attempt" -ge "$READINESS_ATTEMPTS" ]; then echo "[code-runtime] workerd did not become ready in time" >&2 @@ -52,6 +86,7 @@ until curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; do sleep 0.1 done -# Trigger the single run. The worker runs the program and pushes { stdout, stderr } -# to the default dataset. A non-2xx response fails the Actor run (curl -f). +# Trigger the single run. The worker runs the program and pushes { stdout, stderr, +# exitCode, statusMessage } to the default dataset. A non-2xx response fails the Actor +# run (curl -f). curl -fsS -X POST "http://127.0.0.1:${PORT}/run" diff --git a/worker/runner.js b/worker/runner.js index 7ca82b4..9a0dd79 100644 --- a/worker/runner.js +++ b/worker/runner.js @@ -1,8 +1,8 @@ // Single worker for the per-run code-runtime Actor. It runs the user's program // (imported from the generated `usercode.js` module) with the `apify` REST -// binding and a captured `console`, then pushes `{ stdout, stderr }` to the -// run's default dataset. The container entrypoint generates `usercode.js`, -// boots workerd, and triggers `/run` once. +// binding and a captured `console`, then pushes `{ stdout, stderr, exitCode, +// statusMessage }` to the run's default dataset. The container entrypoint +// generates `usercode.js`, boots workerd, and triggers `/run` once. // // Single-tenant: one run = one container = one program = one token. No Worker // Loader / per-request isolate is needed — the program runs in this worker, @@ -58,6 +58,33 @@ function makeApifyBinding(token, apiV2) { const apiJson = async (...args) => (await apiCall(...args)).json(); const apiData = async (...args) => (await apiJson(...args)).data; + // Run IDs this script itself started, via actor.run() / actor.start() (and transitively + // actor.runAndGetItems(), which calls actor.run()). run.abort() below is scoped to this + // set — a script can only abort runs it started, not any account-wide runId it's handed + // or guesses. + const startedRunIds = new Set(); + + // POST /acts/:id/runs, shared by actor.run() (start+wait, waitForFinishSecs defaults to 60, + // capped at 60s per the Apify API — for longer runs use start() + apify.run.wait()) and + // actor.start() (async kickoff, no wait). Returns the run record so the caller can read + // defaultDatasetId / defaultKeyValueStoreId. Intentionally does NOT use /run-sync, which + // returns the OUTPUT KVS record (a pattern only some Actors follow) rather than the + // structured run record. + const createRun = ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }) => + apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { + searchParams: { + waitForFinish: waitForFinishSecs, + memory: memoryMbytes, + timeout: timeoutSecs, + maxTotalChargeUsd, + maxItems, + }, + body: input ?? {}, + }).then((runRecord) => { + startedRunIds.add(runRecord.id); + return runRecord; + }); + const actor = { // GET /v2/store — Apify Store search. Returns the items array directly. search: ({ query, limit, category }) => @@ -67,34 +94,13 @@ function makeApifyBinding(token, apiV2) { getDetails: ({ actorId }) => apiData('GET', `/acts/${encodeURIComponent(actorId)}`), - // POST /runs with waitForFinish blocks until the run completes (max 60s per the - // Apify API; for longer runs the caller should use start() + apify.run.wait()). - // Returns the run record so the caller can read defaultDatasetId / defaultKeyValueStoreId. - // Intentionally does NOT use /run-sync, which returns the OUTPUT KVS record (a pattern - // only some Actors follow) rather than the structured run record. - run: ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs = 60, maxTotalChargeUsd, maxItems }) => - apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { - searchParams: { - waitForFinish: waitForFinishSecs, - memory: memoryMbytes, - timeout: timeoutSecs, - maxTotalChargeUsd, - maxItems, - }, - body: input ?? {}, - }), + // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether + // waitForFinish is set. Records the created run's ID in startedRunIds so run.abort() + // can be scoped to runs this script itself started (see the run.abort definition below). + run: (opts) => createRun({ waitForFinishSecs: 60, ...opts }), // Async kickoff. Returns immediately with a run record in READY/RUNNING state. - start: ({ actorId, input, memoryMbytes, timeoutSecs, maxTotalChargeUsd, maxItems }) => - apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { - searchParams: { - memory: memoryMbytes, - timeout: timeoutSecs, - maxTotalChargeUsd, - maxItems, - }, - body: input ?? {}, - }), + start: (opts) => createRun(opts), // runAndGetItems is added below once `dataset.listItems` is defined. }; @@ -109,8 +115,15 @@ function makeApifyBinding(token, apiV2) { searchParams: { waitForFinish: waitForFinishSecs }, }), - abort: ({ runId }) => - apiData('POST', `/actor-runs/${encodeURIComponent(runId)}/abort`), + // Scoped to runs this script itself started (see startedRunIds above) — without this, + // any runId a script is handed (e.g. read from a dataset item, or guessed) could abort + // an unrelated, account-wide run. + abort: ({ runId }) => { + if (!startedRunIds.has(runId)) { + throw new Error(`Blocked run.abort: "${runId}" was not started by this script`); + } + return apiData('POST', `/actor-runs/${encodeURIComponent(runId)}/abort`); + }, // Returns the full run log as text. `limit` tails the last N characters; the Apify API // does not paginate logs, so this is a client-side slice (the full body is fetched). @@ -243,7 +256,14 @@ function makeApifyBinding(token, apiV2) { apiData('POST', '/key-value-stores', { searchParams: { name } }), }; - return { actor, run, dataset, kvs }; + // Freeze every namespace (and the wrapper) so the script can't reassign a method to + // corrupt its own behavior or, for `console` below, its own output capture. + return Object.freeze({ + actor: Object.freeze(actor), + run: Object.freeze(run), + dataset: Object.freeze(dataset), + kvs: Object.freeze(kvs), + }); } // Push the captured streams as a single item to the run's default dataset. @@ -271,12 +291,13 @@ export default { const stdout = []; const stderr = []; - const captureConsole = { + // Frozen so the script can't reassign e.g. console.log to corrupt its own capture. + const captureConsole = Object.freeze({ log: (...args) => stdout.push(args.map(stringify).join(' ')), error: (...args) => stderr.push(args.map(stringify).join(' ')), warn: (...args) => stderr.push(args.map(stringify).join(' ')), info: (...args) => stdout.push(args.map(stringify).join(' ')), - }; + }); // A thrown program is a user-level failure: capture it in stderr and still // push the output, so the run SUCCEEDS with diagnostics. Infra failures @@ -286,15 +307,27 @@ export default { // status: 0 when the script returns normally, 1 when it throws. The run itself // still SUCCEEDS on a throw, so callers detect a failed script via this field // rather than heuristics on stderr (console.error is a legitimate log channel). + // statusMessage carries the same signal in prose, for callers that don't want to + // branch on exitCode. A script that fails to *compile* never reaches this handler at + // all (workerd fails the whole run before any request arrives) — entrypoint.sh + // handles that case directly, see its statusMessage "Failed to compile: ...". let exitCode = 0; + let statusMessage = 'Script completed'; try { await run(makeApifyBinding(token, apiV2), captureConsole); } catch (err) { - stderr.push(err?.stack ?? err?.message ?? String(err)); + const message = err?.message ?? String(err); + stderr.push(err?.stack ?? message); exitCode = 1; + statusMessage = `Script threw: ${message}`; } - await pushOutput(apiV2, token, env, { stdout: stdout.join('\n'), stderr: stderr.join('\n'), exitCode }); + await pushOutput(apiV2, token, env, { + stdout: stdout.join('\n'), + stderr: stderr.join('\n'), + exitCode, + statusMessage, + }); return Response.json({ ok: true }); }, }; From 5bc33749af0e93fffa626189a612123a2d034637 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 14:57:52 +0200 Subject: [PATCH 16/46] refactor!: rename actor.getDetails() to actor.get() Council-reviewed rename (both a data-structure and an interface-design lens independently converged on this one): actor.get matches run.get's existing naming convention in this same binding, which getDetails did not. Breaking change, acceptable pre-release. docs/API.md and README.md's binding summary updated to match; the fuller docs/description self-containment pass (a separate, later PR) will carry this through the rest of the copy. --- README.md | 2 +- docs/API.md | 2 +- worker/runner.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9cd014e..8dbbf67 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Every method takes one options object and returns parsed JSON ```js // Actors apify.actor.search({ query, limit?, category? }) // → actors[] -apify.actor.getDetails({ actorId }) // → actor +apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run apify.actor.run({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) apify.actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } diff --git a/docs/API.md b/docs/API.md index c746fb1..b57da64 100644 --- a/docs/API.md +++ b/docs/API.md @@ -49,7 +49,7 @@ const actors = await apify.actor.search({ query: 'web scraper', limit: 5 }); console.log(actors.map((a) => `${a.username}/${a.name}`).join('\n')); ``` -### `actor.getDetails({ actorId })` → `Actor` +### `actor.get({ actorId })` → `Actor` Fetch the full record for one Actor. diff --git a/worker/runner.js b/worker/runner.js index 9a0dd79..6005ce0 100644 --- a/worker/runner.js +++ b/worker/runner.js @@ -91,7 +91,7 @@ function makeApifyBinding(token, apiV2) { apiData('GET', '/store', { searchParams: { search: query, limit, category } }) .then((page) => page.items), - getDetails: ({ actorId }) => + get: ({ actorId }) => apiData('GET', `/acts/${encodeURIComponent(actorId)}`), // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether From d59cec3e1c9334a3c38908a8e5e6dcd4fcc5dcdb Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 16:56:35 +0200 Subject: [PATCH 17/46] refactor: migrate worker/*.js to TypeScript, add CI type-checking worker/guard.js and worker/runner.js -> guard.ts / runner.ts, with real types for the apify binding's full public surface (every method's options object named and typed, not `any`), the Env/Run/ApifyRecord shapes this code reads, and the redirect/fetch-guard helpers. Compiles clean under strict:true. API response shapes stay honestly `any` at the api{Json,Data} boundary -- the Apify API's JSON envelope isn't something this repo has a verified schema for, and asserting a precise shape we haven't checked would be worse than not typing it. Everything this code actually authors (every destructured options object, every local helper's params/return) is fully typed. Dockerfile's builder stage already runs Node (to resolve the workerd binary path via require('workerd')) -- compiling TypeScript there is one more RUN line, not a new toolchain. The runtime stage still copies only the compiled JS + entrypoint.sh + config.capnp; no Node, no npm packages, no .ts sources ship in the final image. worker/runner.js and worker/guard.js are now build artifacts (pnpm build / pnpm typecheck), gitignored, generated from the .ts source at Docker build time -- matching how the workerd binary itself is already obtained via the same Node stage. Added .github/workflows/typecheck.yml (pnpm typecheck on push/PR) -- this is the actual CI gap the reviewer flagged; it needs no Apify credentials, unlike test.sh's live apify push + call, so it's safe to run automatically. Verified zero behavioral regression: reran the full live-workerd suite (valid script, usercode.js compile-failure diagnostic, runtime throw, run.abort scoping, console/apify-binding freeze, redirect-following allowlist bypass) against the freshly compiled output -- all six identical to the pre-migration JS. --- .github/workflows/typecheck.yml | 19 +++ .gitignore | 4 + Dockerfile | 21 ++- package.json | 7 + pnpm-lock.yaml | 11 ++ tsconfig.json | 10 ++ worker/{guard.js => guard.ts} | 20 +-- worker/{runner.js => runner.ts} | 287 +++++++++++++++++++++++++------- worker/usercode.d.ts | 7 + 9 files changed, 308 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/typecheck.yml create mode 100644 tsconfig.json rename worker/{guard.js => guard.ts} (89%) rename worker/{runner.js => runner.ts} (60%) create mode 100644 worker/usercode.d.ts diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml new file mode 100644 index 0000000..ccac2fd --- /dev/null +++ b/.github/workflows/typecheck.yml @@ -0,0 +1,19 @@ +name: Typecheck + +on: + push: + branches: [master] + pull_request: + +jobs: + typecheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm run typecheck diff --git a/.gitignore b/.gitignore index 1e13533..e347e03 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ node_modules/ +# Generated at container startup by entrypoint.sh (never checked in): worker/usercode.js +# Compiled from worker/*.ts by `pnpm build` (tsconfig.json emits next to the source): +worker/runner.js +worker/guard.js *.log .DS_Store diff --git a/Dockerfile b/Dockerfile index b675bf7..9fe4169 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,27 @@ # Two-stage build: drop the Node runtime entirely. workerd is a standalone # glibc binary; only libc + libm are needed at runtime (verified via `ldd`). # -# Stage 1: pull the workerd binary via a Node base. pnpm keeps it in the virtual -# store (not hoisted), so resolve the path through `require('workerd')`. +# Stage 1: pull the workerd binary + compile worker/*.ts -> worker/*.js, via a Node +# base. Node already runs here to resolve workerd's binary path, so compiling +# TypeScript is one more RUN line, not a new toolchain. pnpm keeps workerd in the +# virtual store (not hoisted), so resolve the path through `require('workerd')`. FROM node:24-bookworm-slim AS builder WORKDIR /build -COPY package.json pnpm-lock.yaml ./ +COPY package.json pnpm-lock.yaml tsconfig.json ./ +COPY worker/ ./worker/ # --ignore-scripts skips workerd's postinstall (a binary-download fallback we # don't need — the binary ships in the @cloudflare/workerd-linux-64 optional dep) -# and avoids pnpm's hard error on unapproved dependency build scripts. +# and avoids pnpm's hard error on unapproved dependency build scripts. Full +# (non --prod) install: typescript is a devDependency, needed by `pnpm build` below. RUN corepack enable \ - && pnpm install --prod --frozen-lockfile --ignore-scripts \ + && pnpm install --frozen-lockfile --ignore-scripts \ + && pnpm run build \ && BIN="$(node -e "process.stdout.write(require('workerd').default)")" \ && cp "$BIN" /workerd \ && chmod +x /workerd -# Stage 2: minimal runtime — debian + ca-certificates + the workerd binary. +# Stage 2: minimal runtime — debian + ca-certificates + the workerd binary + the +# compiled JS. No Node, no TypeScript, no npm packages in this image. FROM debian:bookworm-slim # curl: loopback HTTP client + Actor-input fetch; jq: extract `code` from the input JSON. @@ -26,6 +32,7 @@ RUN apt-get update \ COPY --from=builder /workerd /usr/local/bin/workerd WORKDIR /app -COPY worker/ ./worker/ +COPY worker/entrypoint.sh worker/config.capnp ./worker/ +COPY --from=builder /build/worker/runner.js /build/worker/guard.js ./worker/ ENTRYPOINT ["sh", "/app/worker/entrypoint.sh"] diff --git a/package.json b/package.json index 33190c6..8de1c9f 100644 --- a/package.json +++ b/package.json @@ -9,5 +9,12 @@ }, "engines": { "node": ">=24" + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "devDependencies": { + "typescript": "6.0.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95ed99b..8d7fee7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,10 @@ importers: workerd: specifier: 1.20260402.1 version: 1.20260402.1 + devDependencies: + typescript: + specifier: 6.0.3 + version: 6.0.3 packages: @@ -44,6 +48,11 @@ packages: cpu: [x64] os: [win32] + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} + engines: {node: '>=14.17'} + hasBin: true + workerd@1.20260402.1: resolution: {integrity: sha512-Cg+OUlukdcCHrTTg0MBCIMFRE6XO3yGVGiWCnJPvfffy2Ga2girrEq3qF/YlHSTmbIyEE5ebCFxBYYYZueQ/Mg==} engines: {node: '>=16'} @@ -66,6 +75,8 @@ snapshots: '@cloudflare/workerd-windows-64@1.20260402.1': optional: true + typescript@6.0.3: {} + workerd@1.20260402.1: optionalDependencies: '@cloudflare/workerd-darwin-64': 1.20260402.1 diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..ac781ef --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "moduleResolution": "bundler", + "lib": ["es2022", "dom"], + "strict": true + }, + "include": ["worker/*.ts"] +} diff --git a/worker/guard.js b/worker/guard.ts similarity index 89% rename from worker/guard.js rename to worker/guard.ts index 662fddb..17840ef 100644 --- a/worker/guard.js +++ b/worker/guard.ts @@ -23,8 +23,8 @@ const realFetch = globalThis.fetch.bind(globalThis); // try to recover the unrestricted fetch, it gets this same cached instance — // but the value is already gone. A standing `export { realFetch }` would hand // it to that later import too; don't reintroduce one. -let unclaimedRealFetch = realFetch; -export function claimRealFetch() { +let unclaimedRealFetch: typeof realFetch | null = realFetch; +export function claimRealFetch(): typeof realFetch | null { const fetchFn = unclaimedRealFetch; unclaimedRealFetch = null; return fetchFn; @@ -33,12 +33,12 @@ export function claimRealFetch() { // Match apify.com exactly or any subdomain. The leading dot in the suffix is // what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` // (ends with `.evil.com`) both fail. -function isAllowedHost(hostname) { +function isAllowedHost(hostname: string): boolean { const host = hostname.toLowerCase().replace(/\.$/, ''); // strip FQDN trailing dot return host === 'apify.com' || host.endsWith('.apify.com'); } -function requestUrl(input) { +function requestUrl(input: RequestInfo | URL): string { if (typeof input === 'string') return input; if (input instanceof URL) return input.href; if (input && typeof input.url === 'string') return input.url; // Request @@ -47,8 +47,8 @@ function requestUrl(input) { // Parses and validates one URL against the allowlist. Returns the parsed URL // (callers use it to resolve a relative redirect Location) or throws. -function validateUrl(input) { - let url; +function validateUrl(input: RequestInfo | URL): URL { + let url: URL; try { // Parse to the real host — defeats userinfo (`apify.com@evil.com`), // path/query/fragment (`evil.com/apify.com`) and similar tricks. @@ -74,14 +74,14 @@ function validateUrl(input) { const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); const MAX_REDIRECT_HOPS = 5; -function nextRedirectInit(init, status) { +function nextRedirectInit(init: RequestInit | undefined, status: number): RequestInit | undefined { const method = (init?.method ?? 'GET').toUpperCase(); const downgradeToGet = status === 303 || ((status === 301 || status === 302) && method === 'POST'); if (!downgradeToGet) return init; return { ...init, method: 'GET', body: undefined }; } -async function guardedFetch(input, init, hop = 0) { +async function guardedFetch(input: RequestInfo | URL, init: RequestInit | undefined, hop = 0): Promise { if (hop > MAX_REDIRECT_HOPS) { throw new Error(`Blocked fetch: exceeded ${MAX_REDIRECT_HOPS} redirects`); } @@ -99,7 +99,7 @@ async function guardedFetch(input, init, hop = 0) { // recover the ambient (real, unrestricted) fetch reference some engines // expose under a different name; locking it closes that off. Object.defineProperty(globalThis, 'fetch', { - value: (input, init) => guardedFetch(input, init), + value: (input: RequestInfo | URL, init?: RequestInit) => guardedFetch(input, init), writable: false, configurable: false, enumerable: true, @@ -109,7 +109,7 @@ Object.defineProperty(globalThis, 'fetch', { // even without nodejs_compat, and they connect directly (not through the fetch // guard), so a script could otherwise open a wss:// or SSE connection to any // public host and exfiltrate data around the *.apify.com allowlist. -function blockGlobal(name) { +function blockGlobal(name: string): void { const blocked = function () { throw new Error(`Blocked ${name}: only fetch() to apify.com and its subdomains is allowed`); }; diff --git a/worker/runner.js b/worker/runner.ts similarity index 60% rename from worker/runner.js rename to worker/runner.ts index 6005ce0..f8d71bb 100644 --- a/worker/runner.js +++ b/worker/runner.ts @@ -16,22 +16,172 @@ import { run } from './usercode.js'; // Must run before usercode.js's `run()` is ever invoked (it does, here — module // evaluation order puts this ahead of any dynamic import from inside `run()`). -const realFetch = claimRealFetch(); -if (!realFetch) throw new Error('realFetch already claimed — guard.js imported out of order.'); +// Factored into a function (rather than a bare `const` + `if (!x) throw`) so the +// non-null guarantee is encoded in the return type once, here — TS doesn't carry +// a narrowed-from-null check across the later function declarations that close +// over `realFetch`, but a return type with the `null` branch already thrown away +// needs no further narrowing anywhere downstream. +function requireRealFetch(): typeof globalThis.fetch { + const fetchFn = claimRealFetch(); + if (!fetchFn) throw new Error('realFetch already claimed — guard.js imported out of order.'); + return fetchFn; +} +const realFetch = requireRealFetch(); const DEFAULT_ITERATE_BATCH = 1000; const DEFAULT_GET_SCHEMA_SAMPLE = 5; -function stringify(x) { +// --- Types --------------------------------------------------------------- +// The Apify API returns many more fields per record than this code reads. Rather +// than inventing a full schema we don't have, ApifyRecord asserts nothing beyond +// "a JSON object" and each specific shape below only names the fields this code +// actually consumes. + +interface ApifyRecord { + [key: string]: unknown; +} + +interface RunRecord extends ApifyRecord { + id: string; +} + +type SearchParamValue = string | number | boolean | undefined | null; +type SearchParams = Record; + +interface ApiCallOptions { + searchParams?: SearchParams; + body?: unknown; + contentType?: string; +} + +interface SearchOptions { + query: string; + limit?: number; + category?: string; +} + +interface ActorIdOptions { + actorId: string; +} + +interface StartOptions { + actorId: string; + input?: unknown; + memoryMbytes?: number; + timeoutSecs?: number; + waitForFinishSecs?: number; + maxTotalChargeUsd?: number; + maxItems?: number; +} + +interface RunAndGetItemsOptions extends StartOptions { + fields?: string[]; + limit?: number; +} + +interface RunIdOptions { + runId: string; +} + +interface WaitOptions extends RunIdOptions { + waitForFinishSecs?: number; +} + +interface GetLogOptions extends RunIdOptions { + limit?: number; +} + +interface DatasetListOptions { + datasetId: string; + fields?: string[]; + omit?: string[]; + limit?: number; + offset?: number; + clean?: boolean; + desc?: boolean; +} + +interface DatasetIterateOptions extends Omit { + batchSize?: number; +} + +interface DatasetSchemaOptions { + datasetId: string; + sample?: number; +} + +interface DatasetSchema { + itemCount: unknown; + sampleSize: number; + fields: { name: string; types: string[]; nullable: boolean }[]; +} + +interface CreateOptions { + name?: string; +} + +interface PushItemsOptions { + datasetId: string; + items: unknown[]; +} + +interface KvsGetOptions { + storeId: string; + key: string; +} + +interface KvsSetOptions extends KvsGetOptions { + value: unknown; + contentType?: string; +} + +interface KvsListOptions { + storeId: string; + limit?: number; + exclusiveStartKey?: string; +} + +interface ConsoleLike { + log: (...args: unknown[]) => void; + error: (...args: unknown[]) => void; + warn: (...args: unknown[]) => void; + info: (...args: unknown[]) => void; +} + +interface Env { + APIFY_TOKEN?: string; + DEFAULT_DATASET_ID?: string; + DEFAULT_DATASET_ID_LEGACY?: string; + API_BASE_URL?: string; +} + +interface OutputItem { + stdout: string; + stderr: string; + exitCode: number; + statusMessage: string; +} + +// --------------------------------------------------------------------------- + +function stringify(x: unknown): string { if (typeof x === 'string') return x; try { return JSON.stringify(x); } catch { return String(x); } } -function makeApifyBinding(token, apiV2) { - const baseHeaders = { Authorization: `Bearer ${token}` }; +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +function errorDetail(err: unknown): string { + return err instanceof Error && err.stack ? err.stack : errorMessage(err); +} + +function makeApifyBinding(token: string, apiV2: string) { + const baseHeaders: Record = { Authorization: `Bearer ${token}` }; // Build a URL with optional query params; null/undefined values are dropped. - const buildUrl = (path, searchParams) => { + const buildUrl = (path: string, searchParams?: SearchParams): URL => { const url = new URL(`${apiV2}${path}`); if (searchParams) { for (const [key, value] of Object.entries(searchParams)) { @@ -43,26 +193,40 @@ function makeApifyBinding(token, apiV2) { // Single-source HTTP wrapper. Throws on non-2xx with the response body in the message. // `body`: string / Uint8Array passed through; objects are JSON.stringify'd. - const apiCall = async (method, path, { searchParams, body, contentType } = {}) => { - const init = { method, headers: { ...baseHeaders } }; + const apiCall = async (method: string, path: string, options: ApiCallOptions = {}): Promise => { + const { searchParams, body, contentType } = options; + const headers: Record = { ...baseHeaders }; + let requestBody: BodyInit | undefined; if (body !== undefined) { - const isRaw = typeof body === 'string' || body instanceof Uint8Array || body instanceof ArrayBuffer; - init.body = isRaw ? body : JSON.stringify(body); - init.headers['content-type'] = contentType ?? (isRaw ? 'application/octet-stream' : 'application/json'); + if (typeof body === 'string' || body instanceof Uint8Array || body instanceof ArrayBuffer) { + // Cast: this TS/DOM-lib pairing types Uint8Array generically over its buffer, + // which doesn't structurally match BodyInit here even though it's a valid + // fetch body at runtime (an ArrayBufferView). + requestBody = body as BodyInit; + headers['content-type'] = contentType ?? 'application/octet-stream'; + } else { + requestBody = JSON.stringify(body); + headers['content-type'] = contentType ?? 'application/json'; + } } - const response = await realFetch(buildUrl(path, searchParams), init); + const response = await realFetch(buildUrl(path, searchParams), { method, headers, body: requestBody }); if (!response.ok) throw new Error(`${method} ${path} failed: ${response.status} ${await response.text()}`); return response; }; - const apiJson = async (...args) => (await apiCall(...args)).json(); - const apiData = async (...args) => (await apiJson(...args)).data; + // The Apify API's JSON envelope (`{ data: ... }`) carries whatever shape the endpoint + // returns; there's no schema to check it against here, so this stays honestly `any` + // rather than asserting a shape we haven't verified. + const apiJson = async (method: string, path: string, options?: ApiCallOptions): Promise => + (await apiCall(method, path, options)).json(); + const apiData = async (method: string, path: string, options?: ApiCallOptions): Promise => + (await apiJson(method, path, options)).data; // Run IDs this script itself started, via actor.run() / actor.start() (and transitively // actor.runAndGetItems(), which calls actor.run()). run.abort() below is scoped to this // set — a script can only abort runs it started, not any account-wide runId it's handed // or guesses. - const startedRunIds = new Set(); + const startedRunIds = new Set(); // POST /acts/:id/runs, shared by actor.run() (start+wait, waitForFinishSecs defaults to 60, // capped at 60s per the Apify API — for longer runs use start() + apify.run.wait()) and @@ -70,7 +234,7 @@ function makeApifyBinding(token, apiV2) { // defaultDatasetId / defaultKeyValueStoreId. Intentionally does NOT use /run-sync, which // returns the OUTPUT KVS record (a pattern only some Actors follow) rather than the // structured run record. - const createRun = ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }) => + const createRun = ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }: StartOptions): Promise => apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { searchParams: { waitForFinish: waitForFinishSecs, @@ -80,37 +244,47 @@ function makeApifyBinding(token, apiV2) { maxItems, }, body: input ?? {}, - }).then((runRecord) => { + }).then((runRecord: RunRecord) => { startedRunIds.add(runRecord.id); return runRecord; }); const actor = { // GET /v2/store — Apify Store search. Returns the items array directly. - search: ({ query, limit, category }) => + search: ({ query, limit, category }: SearchOptions): Promise => apiData('GET', '/store', { searchParams: { search: query, limit, category } }) - .then((page) => page.items), + .then((page: { items: ApifyRecord[] }) => page.items), - get: ({ actorId }) => + get: ({ actorId }: ActorIdOptions): Promise => apiData('GET', `/acts/${encodeURIComponent(actorId)}`), // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether // waitForFinish is set. Records the created run's ID in startedRunIds so run.abort() // can be scoped to runs this script itself started (see the run.abort definition below). - run: (opts) => createRun({ waitForFinishSecs: 60, ...opts }), + run: (opts: StartOptions): Promise => createRun({ waitForFinishSecs: 60, ...opts }), // Async kickoff. Returns immediately with a run record in READY/RUNNING state. - start: (opts) => createRun(opts), - // runAndGetItems is added below once `dataset.listItems` is defined. + start: (opts: StartOptions): Promise => createRun(opts), + + // Runs an Actor (same as run(), waitForFinishSecs defaults to 60) and returns its + // dataset items in one call. Calls createRun() directly rather than through + // `actor.run()` — same underlying request, no self-reference to `actor` needed. + runAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { + const runRecord = await createRun({ actorId, input, waitForFinishSecs: 60, ...runOpts }); + const items = await dataset.listItems({ + datasetId: runRecord.defaultDatasetId as string, fields, limit, + }); + return { run: runRecord, items }; + }, }; const run = { - get: ({ runId }) => + get: ({ runId }: RunIdOptions): Promise => apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`), // Block until the run terminates or `waitForFinishSecs` elapses (whichever comes first). // The Apify API caps this at 60s per request; longer waits require a polling loop. - wait: ({ runId, waitForFinishSecs = 60 }) => + wait: ({ runId, waitForFinishSecs = 60 }: WaitOptions): Promise => apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { searchParams: { waitForFinish: waitForFinishSecs }, }), @@ -118,7 +292,7 @@ function makeApifyBinding(token, apiV2) { // Scoped to runs this script itself started (see startedRunIds above) — without this, // any runId a script is handed (e.g. read from a dataset item, or guessed) could abort // an unrelated, account-wide run. - abort: ({ runId }) => { + abort: ({ runId }: RunIdOptions): Promise => { if (!startedRunIds.has(runId)) { throw new Error(`Blocked run.abort: "${runId}" was not started by this script`); } @@ -127,7 +301,7 @@ function makeApifyBinding(token, apiV2) { // Returns the full run log as text. `limit` tails the last N characters; the Apify API // does not paginate logs, so this is a client-side slice (the full body is fetched). - getLog: async ({ runId, limit }) => { + getLog: async ({ runId, limit }: GetLogOptions): Promise => { const response = await apiCall('GET', `/logs/${encodeURIComponent(runId)}`); const text = await response.text(); return limit && text.length > limit ? text.slice(-limit) : text; @@ -139,7 +313,7 @@ function makeApifyBinding(token, apiV2) { // `x-apify-pagination-total` header is unreliable for freshly-created datasets // (eventually consistent), so we don't surface a `total`. Use `getSchema` if you // need an item count, or iterate to consume the whole dataset. - listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }) => { + listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }: DatasetListOptions): Promise => { const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { searchParams: { fields: fields?.join(','), @@ -157,7 +331,7 @@ function makeApifyBinding(token, apiV2) { // so the user can `for await (const item of apify.dataset.iterate({...}))` without // worrying about offsets. Stops when a page returns fewer items than `batchSize` // (the natural end-of-data signal — pagination total is not used, see listItems). - iterate: async function* ({ datasetId, fields, omit, clean, desc, batchSize = DEFAULT_ITERATE_BATCH }) { + iterate: async function* ({ datasetId, fields, omit, clean, desc, batchSize = DEFAULT_ITERATE_BATCH }: DatasetIterateOptions): AsyncGenerator { let offset = 0; while (true) { const items = await dataset.listItems({ @@ -172,16 +346,15 @@ function makeApifyBinding(token, apiV2) { }, // Apify has no dedicated schema endpoint; we infer one from a small sample of items. - // Returns { itemCount, sampleSize, fields: [{ name, types, nullable }] }. - getSchema: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }) => { + getSchema: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { const meta = await apiData('GET', `/datasets/${encodeURIComponent(datasetId)}`); const items = await dataset.listItems({ datasetId, limit: sample }); - const fields = new Map(); + const fields = new Map>(); for (const item of items) { for (const [name, value] of Object.entries(item ?? {})) { const type = value === null ? 'null' : Array.isArray(value) ? 'array' : typeof value; if (!fields.has(name)) fields.set(name, new Set()); - fields.get(name).add(type); + fields.get(name)!.add(type); } } return { @@ -195,27 +368,19 @@ function makeApifyBinding(token, apiV2) { }; }, - create: ({ name } = {}) => + create: ({ name }: CreateOptions = {}): Promise => apiData('POST', '/datasets', { searchParams: { name } }), - pushItems: async ({ datasetId, items }) => { + pushItems: async ({ datasetId, items }: PushItemsOptions): Promise => { await apiCall('POST', `/datasets/${encodeURIComponent(datasetId)}/items`, { body: items }); }, }; - actor.runAndGetItems = async ({ actorId, input, fields, limit, ...runOpts }) => { - const runRecord = await actor.run({ actorId, input, ...runOpts }); - const items = await dataset.listItems({ - datasetId: runRecord.defaultDatasetId, fields, limit, - }); - return { run: runRecord, items }; - }; - const kvs = { // Returns the value directly (parsed when JSON, string when text/*, Uint8Array otherwise). // Returns null when the key does not exist (404), not an error — this matches the common // "lookup or default" pattern in code. - get: async ({ storeId, key }) => { + get: async ({ storeId, key }: KvsGetOptions): Promise => { const response = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { headers: baseHeaders, }); @@ -229,11 +394,12 @@ function makeApifyBinding(token, apiV2) { // `value`: object → application/json; string → text/plain; Uint8Array/ArrayBuffer → // application/octet-stream (or whatever the caller passed via `contentType`). - set: async ({ storeId, key, value, contentType }) => { - let body; + set: async ({ storeId, key, value, contentType }: KvsSetOptions): Promise => { + let body: BodyInit; let resolvedContentType = contentType; if (value instanceof Uint8Array || value instanceof ArrayBuffer) { - body = value; + // Cast: see the equivalent Uint8Array-vs-BodyInit comment in apiCall() above. + body = value as BodyInit; resolvedContentType = resolvedContentType ?? 'application/octet-stream'; } else if (typeof value === 'string') { body = value; @@ -247,12 +413,12 @@ function makeApifyBinding(token, apiV2) { }); }, - list: ({ storeId, limit, exclusiveStartKey }) => + list: ({ storeId, limit, exclusiveStartKey }: KvsListOptions): Promise => apiData('GET', `/key-value-stores/${encodeURIComponent(storeId)}/keys`, { searchParams: { limit, exclusiveStartKey }, }), - create: ({ name } = {}) => + create: ({ name }: CreateOptions = {}): Promise => apiData('POST', '/key-value-stores', { searchParams: { name } }), }; @@ -267,7 +433,7 @@ function makeApifyBinding(token, apiV2) { } // Push the captured streams as a single item to the run's default dataset. -async function pushOutput(apiV2, token, env, item) { +async function pushOutput(apiV2: string, token: string, env: Env, item: OutputItem): Promise { const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; if (!datasetId) throw new Error('Default dataset ID missing from Actor run environment.'); const response = await realFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { @@ -279,7 +445,7 @@ async function pushOutput(apiV2, token, env, item) { } export default { - async fetch(request, env) { + async fetch(request: Request, env: Env): Promise { const url = new URL(request.url); if (url.pathname === '/health') return new Response('ok'); if (url.pathname !== '/run') return new Response('Not found', { status: 404 }); @@ -289,14 +455,14 @@ export default { // APIFY_API_BASE_URL is the platform-internal API (may have a trailing slash). const apiV2 = `${(env.API_BASE_URL || 'https://api.apify.com').replace(/\/+$/, '')}/v2`; - const stdout = []; - const stderr = []; + const stdout: string[] = []; + const stderr: string[] = []; // Frozen so the script can't reassign e.g. console.log to corrupt its own capture. - const captureConsole = Object.freeze({ - log: (...args) => stdout.push(args.map(stringify).join(' ')), - error: (...args) => stderr.push(args.map(stringify).join(' ')), - warn: (...args) => stderr.push(args.map(stringify).join(' ')), - info: (...args) => stdout.push(args.map(stringify).join(' ')), + const captureConsole: ConsoleLike = Object.freeze({ + log: (...args: unknown[]) => stdout.push(args.map(stringify).join(' ')), + error: (...args: unknown[]) => stderr.push(args.map(stringify).join(' ')), + warn: (...args: unknown[]) => stderr.push(args.map(stringify).join(' ')), + info: (...args: unknown[]) => stdout.push(args.map(stringify).join(' ')), }); // A thrown program is a user-level failure: capture it in stderr and still @@ -316,10 +482,9 @@ export default { try { await run(makeApifyBinding(token, apiV2), captureConsole); } catch (err) { - const message = err?.message ?? String(err); - stderr.push(err?.stack ?? message); + stderr.push(errorDetail(err)); exitCode = 1; - statusMessage = `Script threw: ${message}`; + statusMessage = `Script threw: ${errorMessage(err)}`; } await pushOutput(apiV2, token, env, { diff --git a/worker/usercode.d.ts b/worker/usercode.d.ts new file mode 100644 index 0000000..f36a870 --- /dev/null +++ b/worker/usercode.d.ts @@ -0,0 +1,7 @@ +// usercode.js is generated at container startup by entrypoint.sh (gitignored, never +// checked in) — it wraps the run's `code` input in `export async function run(apify, +// console) { ...code... }`. This declaration lets `tsc` resolve runner.ts's import +// without the generated file present. The user's code is untyped JS text spliced +// into the function body, so `apify`/`console` stay `unknown` here on purpose — +// runner.ts's own ApifyBinding/ConsoleLike types describe what's actually passed in. +export function run(apify: unknown, consoleLike: unknown): Promise; From 90201416bbfcc6ec0272e430e9b5f68095e45b8a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 16:56:57 +0200 Subject: [PATCH 18/46] docs: drop TypeScript claim for the user's code input The code input is fetched at container runtime and wrapped verbatim into usercode.js (entrypoint.sh); nothing transpiles it and workerd doesn't strip types, so a TypeScript type annotation in user code is a SyntaxError at load today, not a supported input. actor.json's description/input schema and README's intro + input table claimed TypeScript/JavaScript; corrected to JavaScript only, with the SyntaxError-at-load reason stated so it's not mistaken for an oversight. Scoped to the user-facing code input contract only -- the separate question of what language the Actor's own source is written in was resolved in the previous commit (TypeScript, compiled at build time). --- .actor/actor.json | 4 ++-- README.md | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 3b0434e..4da5e2f 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -2,7 +2,7 @@ "actorSpecification": 1, "name": "code-runtime", "title": "Code Runtime", - "description": "Runs an LLM-submitted TypeScript/JavaScript program in a sandboxed workerd V8 isolate with Apify bindings. One program per run; the captured { stdout, stderr } is pushed as a single item to the default dataset.", + "description": "Runs an LLM-submitted JavaScript program in a sandboxed workerd V8 isolate with Apify bindings. One program per run; the captured { stdout, stderr } is pushed as a single item to the default dataset.", "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, @@ -15,7 +15,7 @@ "code": { "title": "Code", "type": "string", - "description": "TypeScript/JavaScript program executed inside the sandbox. It receives an `apify` binding and `console`; stdout and stderr are captured separately and pushed to the default dataset as { stdout, stderr }.", + "description": "JavaScript program executed inside the sandbox (JS only — nothing transpiles it, so a TypeScript type annotation is a SyntaxError at load). It receives an `apify` binding and `console`; stdout and stderr are captured separately and pushed to the default dataset as { stdout, stderr }.", "editor": "javascript" } }, diff --git a/README.md b/README.md index 8dbbf67..d58d97b 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,9 @@ ## What it does -This Actor executes TypeScript/JavaScript that an AI agent submits through the -Apify MCP Server's **Code Mode**, then returns whatever the script printed. +This Actor executes JavaScript that an AI agent submits through the Apify MCP +Server's **Code Mode**, then returns whatever the script printed. JS only — +nothing transpiles it, so a TypeScript type annotation is a SyntaxError at load. Code Mode exists so an agent can do many Apify operations in **one go** — search the Store, run an Actor, read its dataset, filter and aggregate the @@ -51,7 +52,7 @@ For full configuration options, use the configurator at | Field | Type | Description | |---|---|---| -| `code` | string | The TypeScript/JavaScript script to run. It receives the `apify` binding and `console`. | +| `code` | string | The JavaScript script to run (JS only, not transpiled). It receives the `apify` binding and `console`. | ## Output From 976d15157ce76d12c24ce56b6061be125ca2e8ff Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 17:03:02 +0200 Subject: [PATCH 19/46] refactor!: rename actor.run/runAndGetItems, run.wait, dataset.getSchema Council-reviewed renames (both a data-structure and an interface-design lens independently kept these three, out of a larger proposed table): - actor.run() -> actor.call(): 'run' was doing double duty as a verb (start+wait) and as the top-level run-management namespace. call avoids the collision and matches the CLI (apify call) / MCP (call-actor) vocabulary. - actor.runAndGetItems() -> actor.callAndGetItems(): follows from the above. - run.wait() -> run.waitForFinish(): passes straight through to the REST waitForFinish query param; matches the platform vocabulary. The waitForFinishSecs *parameter* name is intentionally NOT touched -- pairing this method rename with dropping the unit suffix would produce waitForFinish({ waitForFinish: 30 }), read as a boolean, not a duration. - dataset.getSchema() -> dataset.inferFields(): 'schema' already means two other things in this Actor's own ecosystem (input schema, and this Actor's own *declared* dataset schema in actor.json) -- inferFields names what the method actually does (infer field types from a sample) without colliding. Breaking change, acceptable pre-release. Verified live: apify.actor.call, apify.actor.callAndGetItems, apify.run.waitForFinish, apify.dataset.inferFields all exercised end-to-end against a mocked Apify API via real workerd. docs/API.md and README.md still describe the old names -- updated in the next commit alongside the rest of the self-containment docs pass. --- worker/runner.ts | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/worker/runner.ts b/worker/runner.ts index f8d71bb..d762d4a 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -222,15 +222,15 @@ function makeApifyBinding(token: string, apiV2: string) { const apiData = async (method: string, path: string, options?: ApiCallOptions): Promise => (await apiJson(method, path, options)).data; - // Run IDs this script itself started, via actor.run() / actor.start() (and transitively - // actor.runAndGetItems(), which calls actor.run()). run.abort() below is scoped to this - // set — a script can only abort runs it started, not any account-wide runId it's handed - // or guesses. + // Run IDs this script itself started, via actor.call() / actor.start() (and transitively + // actor.callAndGetItems(), which shares createRun() below). run.abort() below is scoped to + // this set — a script can only abort runs it started, not any account-wide runId it's + // handed or guesses. const startedRunIds = new Set(); - // POST /acts/:id/runs, shared by actor.run() (start+wait, waitForFinishSecs defaults to 60, - // capped at 60s per the Apify API — for longer runs use start() + apify.run.wait()) and - // actor.start() (async kickoff, no wait). Returns the run record so the caller can read + // POST /acts/:id/runs, shared by actor.call() (start+wait, waitForFinishSecs defaults to 60, + // capped at 60s per the Apify API — for longer runs use start() + apify.run.waitForFinish()) + // and actor.start() (async kickoff, no wait). Returns the run record so the caller can read // defaultDatasetId / defaultKeyValueStoreId. Intentionally does NOT use /run-sync, which // returns the OUTPUT KVS record (a pattern only some Actors follow) rather than the // structured run record. @@ -261,15 +261,15 @@ function makeApifyBinding(token: string, apiV2: string) { // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether // waitForFinish is set. Records the created run's ID in startedRunIds so run.abort() // can be scoped to runs this script itself started (see the run.abort definition below). - run: (opts: StartOptions): Promise => createRun({ waitForFinishSecs: 60, ...opts }), + call: (opts: StartOptions): Promise => createRun({ waitForFinishSecs: 60, ...opts }), // Async kickoff. Returns immediately with a run record in READY/RUNNING state. start: (opts: StartOptions): Promise => createRun(opts), - // Runs an Actor (same as run(), waitForFinishSecs defaults to 60) and returns its + // Runs an Actor (same as call(), waitForFinishSecs defaults to 60) and returns its // dataset items in one call. Calls createRun() directly rather than through - // `actor.run()` — same underlying request, no self-reference to `actor` needed. - runAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { + // `actor.call()` — same underlying request, no self-reference to `actor` needed. + callAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { const runRecord = await createRun({ actorId, input, waitForFinishSecs: 60, ...runOpts }); const items = await dataset.listItems({ datasetId: runRecord.defaultDatasetId as string, fields, limit, @@ -284,7 +284,7 @@ function makeApifyBinding(token: string, apiV2: string) { // Block until the run terminates or `waitForFinishSecs` elapses (whichever comes first). // The Apify API caps this at 60s per request; longer waits require a polling loop. - wait: ({ runId, waitForFinishSecs = 60 }: WaitOptions): Promise => + waitForFinish: ({ runId, waitForFinishSecs = 60 }: WaitOptions): Promise => apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { searchParams: { waitForFinish: waitForFinishSecs }, }), @@ -311,7 +311,7 @@ function makeApifyBinding(token: string, apiV2: string) { const dataset = { // Returns the items array directly (no wrapper). The Apify API's // `x-apify-pagination-total` header is unreliable for freshly-created datasets - // (eventually consistent), so we don't surface a `total`. Use `getSchema` if you + // (eventually consistent), so we don't surface a `total`. Use `inferFields` if you // need an item count, or iterate to consume the whole dataset. listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }: DatasetListOptions): Promise => { const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { @@ -346,7 +346,9 @@ function makeApifyBinding(token: string, apiV2: string) { }, // Apify has no dedicated schema endpoint; we infer one from a small sample of items. - getSchema: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { + // Named inferFields (not getSchema) to avoid colliding with the Actor's own *declared* + // dataset schema (a different concept, described in this Actor's own actor.json). + inferFields: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { const meta = await apiData('GET', `/datasets/${encodeURIComponent(datasetId)}`); const items = await dataset.listItems({ datasetId, limit: sample }); const fields = new Map>(); From 8e18d6b5a62f6e8d23b9dd6d2065dc169b31e0c9 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 17:12:31 +0200 Subject: [PATCH 20/46] docs: self-contained actor.json description, dataset schema, README Final self-containment pass (previously planned as a separate stacked PR, landing directly on this branch per instruction). All content was council- reviewed earlier in the design phase; this lands it against the actual current code (post rename + statusMessage + TS migration). .actor/actor.json: - description (249/300 chars): what it does, the data-heavy/fan-out framing, sandbox limits moved out to README (kept here: run mechanics + billing). - input.code.description (375/500 chars): the correctness-critical facts -- only console output round-trips (a top-level return is NOT captured), call apify.actor.get() before running an Actor, print a small summary. - output.description: now includes exitCode + statusMessage. - defaultRunOptions: timeoutSecs 900, memoryMbytes 1024 -- this Actor had no default before; overridable per call. - storages.dataset: declared fields (stdout/stderr/exitCode/statusMessage, exitCode kept as enum:[0,1] -- not widened to nullable, see prior design notes) plus a dataset-level description stating the run-level-kill cardinality caveat (zero items is a distinct case from any field's value). No `views` block -- a single always-one-item dataset doesn't need one. README.md: - "Calling this Actor" replaces the stale `?tools=run-code,get-code-docs` section -- that mechanism doesn't exist; call-actor/search-actors/ fetch-actor-details are already default MCP tools, no opt-in needed. - Data-heavy/fan-out framing + the free-text-extraction anti-pattern (kept here, cut from the character-capped actor.json description). - Workflow tips (get-before-running, log storage IDs before processing, print-small-summary, return values aren't captured) folded into "How it works" rather than a new section duplicating the code-input description. - "Limits & failure modes": defaultRunOptions + override, and the exitCode/statusMessage-vs-run-status distinction for resource kills. - "Recipes": bounded parallel fan-out (this Actor's actual measured strength) and the >60s start+poll pattern. - "Limitations": sub-Actor runs aren't MCP-attributed -- known, unresolved, tracked separately, not fixed by this PR. - Egress-safety wording softened to "no direct fetch-based exfil", not "contained" -- actor.start()/dataset writes are still exfil paths outside this guard's scope. - Binding summary + Input example updated to the renamed methods (call/callAndGetItems/waitForFinish/inferFields/get). docs/API.md: renamed section headers/cross-references to match (actor.call, actor.callAndGetItems, run.waitForFinish, dataset.inferFields). package.json: description no longer claims "Worker Loader API" (this Actor is one static worker, not per-request isolate loading) -- stale since before this branch, corrected here alongside the rest of the accuracy pass. --- .actor/actor.json | 44 ++++++++++++++-- README.md | 130 +++++++++++++++++++++++++++++++++++++++------- docs/API.md | 20 +++---- package.json | 2 +- 4 files changed, 163 insertions(+), 33 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 4da5e2f..3458a96 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -2,10 +2,14 @@ "actorSpecification": 1, "name": "code-runtime", "title": "Code Runtime", - "description": "Runs an LLM-submitted JavaScript program in a sandboxed workerd V8 isolate with Apify bindings. One program per run; the captured { stdout, stderr } is pushed as a single item to the default dataset.", + "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Best for data-heavy jobs: scrape hundreds+ places/items, chain Actor outputs, and filter/sort/aggregate in one billed run. Results land in the dataset.", "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, + "defaultRunOptions": { + "timeoutSecs": 900, + "memoryMbytes": 1024 + }, "input": { "title": "Code Runtime Input", "description": "The program to run inside the sandbox.", @@ -15,7 +19,7 @@ "code": { "title": "Code", "type": "string", - "description": "JavaScript program executed inside the sandbox (JS only — nothing transpiles it, so a TypeScript type annotation is a SyntaxError at load). It receives an `apify` binding and `console`; stdout and stderr are captured separately and pushed to the default dataset as { stdout, stderr }.", + "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Call apify.actor.get({ actorId }) before running an Actor, to read its schema first. Print a small JSON summary of the result — never dump full datasets.", "editor": "javascript" } }, @@ -24,7 +28,7 @@ "output": { "actorOutputSchemaVersion": 1, "title": "Code Runtime Output", - "description": "One dataset item { stdout, stderr } with the program's captured output.", + "description": "One dataset item { stdout, stderr, exitCode, statusMessage } (exitCode: 0=returned, 1=threw); see the exitCode field description for the timeout/OOM case.", "type": "object", "properties": { "output": { @@ -34,5 +38,39 @@ } } }, + "storages": { + "dataset": { + "actorSpecification": 1, + "description": "Present only if the script ran to completion (returned or threw). A run-level timeout or OOM kill produces zero items for that run — the calling Actor-run's own status (SUCCEEDED vs FAILED/TIMED-OUT) is the signal for that case, not item presence.", + "fields": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "stdout": { + "type": "string", + "title": "stdout", + "description": "Captured console.log/console.info output." + }, + "stderr": { + "type": "string", + "title": "stderr", + "description": "Captured console.error/console.warn output, plus the thrown error (or compile failure) message when exitCode is 1." + }, + "exitCode": { + "type": "integer", + "enum": [0, 1], + "title": "Exit code", + "description": "0 = script returned normally, 1 = script threw (or failed to compile)." + }, + "statusMessage": { + "type": "string", + "title": "Status message", + "description": "Prose form of the same signal: 'Script completed' / 'Script threw: ...' / 'Failed to compile: ...'." + } + }, + "required": ["stdout", "stderr", "exitCode", "statusMessage"] + } + } + }, "dockerfile": "../Dockerfile" } diff --git a/README.md b/README.md index d58d97b..c028a08 100644 --- a/README.md +++ b/README.md @@ -15,17 +15,27 @@ search the Store, run an Actor, read its dataset, filter and aggregate the results — instead of sending every intermediate result back through the model and wasting tokens. This Actor is the sandbox that runs that script. -## Enabling Code Mode on the MCP Server +**Best suited for data-heavy jobs** — scraping hundreds or thousands of +places/items via an Actor, then filtering, sorting, or aggregating them +locally before returning a small summary. **Weaker fit** for steps that +require reading or judging free text (picking a fact out of an article, +choosing a search term) — keep the model in the loop there instead; a wrong +guess inside the sandbox fails silently until the whole script finishes. -Code Mode is opt-in. Add the Code Mode tools to the `tools` query parameter of -your mcp.apify.com connection URL: +## Calling this Actor + +Self-contained — no special MCP-server opt-in required. Any MCP client +already has `search-actors`, `fetch-actor-details`, and `call-actor` as +default tools: ``` -https://mcp.apify.com/?tools=run-code,get-code-docs +call-actor({ actor: "apify/code-runtime", input: { code: "..." } }) ``` -For full configuration options, use the configurator at -[mcp.apify.com](https://mcp.apify.com). +Or via the raw API: `POST /v2/acts/apify~code-runtime/runs` with `{ code }` +as the body. Results land in the run's default dataset, same as any Actor +call — follow the response's `nextStep` (or call `get-dataset-items`/ +`GET /v2/datasets/{datasetId}/items`) to read it. ## How it works @@ -41,12 +51,22 @@ For full configuration options, use the configurator at the current run's token (see below). - `console.log` / `console.info` go to **stdout**; `console.error` / `console.warn` go to **stderr**. The two streams are captured separately. +- Before running an Actor from your script, call `apify.actor.get({ actorId })` + once to read its input/output schema. +- As each nested run finishes, log its `run.id` / `defaultDatasetId` / + `defaultKeyValueStoreId` **before** processing its output — if the script + then throws, a re-run can read those existing storages instead of paying to + re-run the Actor (nothing persists between this Actor's own runs, but the + Actors it started keep their results). +- Print a small, JSON-stringified summary of the result — never dump full + datasets. Only what you `console.log`/`console.info` comes back; a + top-level `return` value is **not** captured. ## Input ```json { - "code": "const { items } = await apify.actor.runAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" + "code": "const { items } = await apify.actor.callAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" } ``` @@ -56,28 +76,100 @@ For full configuration options, use the configurator at ## Output -A single **dataset item** with the captured streams and the script's exit status: +A single **dataset item** with the captured streams, the script's exit +status, and a prose status message: ```json -{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "", "exitCode": 0 } +{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "", "exitCode": 0, "statusMessage": "Script completed" } ``` If the script throws, the error lands in `stderr`, `stdout` keeps whatever was -printed before the failure, and `exitCode` is `1`. The Actor run itself still -**succeeds** — `exitCode` is the reliable signal for a failed script (`0` = the -script returned normally, `1` = it threw), since `stderr` is also a legitimate -log channel (`console.error` / `console.warn`). +printed before the failure, `exitCode` is `1`, and `statusMessage` is +`"Script threw: ..."`. The Actor run itself still **succeeds** — check +`exitCode`/`statusMessage`, not `stderr` content, to detect a failed script, +since `stderr` is also a legitimate log channel (`console.error` / +`console.warn`). + +If the script fails to **compile** (a syntax error), the same contract +applies — `exitCode: 1`, `statusMessage: "Failed to compile: ..."` — pushed +by the container entrypoint directly, since a malformed script never reaches +the sandboxed worker at all. + +A run-level **timeout or out-of-memory kill** is a different, third outcome: +the container is killed before it can push anything, so this dataset item may +not exist for that run at all. That case is signaled by the Actor run's own +status (`SUCCEEDED` vs `FAILED`/`TIMED-OUT`), not by this item's absence — +see [Limits & failure modes](#limits--failure-modes). + +## Limits & failure modes + +- Default `defaultRunOptions`: `timeoutSecs: 900`, `memoryMbytes: 1024` + (`.actor/actor.json`). Override per call — e.g. the MCP `call-actor` tool's + `callOptions.timeout`/`callOptions.memory`, or the API's `timeout`/`memory` + run options — for scripts that chain several long-running Actor calls. +- `exitCode`/`statusMessage` signal the **script's** outcome only (returned / + threw / failed to compile). A resource-limit kill is a **run-level** + outcome instead — check the Actor run's own `status`, not this dataset + item, for that case (see [Output](#output) above). ## Permissions & safety -- Runs with **limited permissions**: the sandbox has no filesystem and can reach - only the Apify API (`*.apify.com`). +- Runs with **limited permissions**: the sandbox has no filesystem and + outbound `fetch` (including through redirects, which are re-validated per + hop) is limited to the Apify API (`*.apify.com`). - **No imports.** The isolate runs without workerd's `nodejs_compat`, so user code cannot import Node built-ins (`node:net`, `node:fs`, …) or npm packages. This removes `node:net` — a raw-socket egress path that would otherwise bypass the `fetch` allowlist — and keeps the run token out of `process.env` (which is not defined). - Each run is an isolated, single-use container — nothing persists between runs. +- This closes off **direct fetch-based exfil** from the container — it does not + stop every path to move data out (e.g. `actor.start({ input })` on an Actor + with its own open internet access, or writing to a dataset/key-value store). + +## Recipes + +### Bounded parallel fan-out + +This Actor's clearest win: run several independent Actors (or the same Actor +over several inputs) concurrently, then reduce before returning. Chunk the +fan-out (e.g. 5–10 at a time) — an unbounded `Promise.all` over many inputs +can hit your account's concurrent-run or memory limits. + +```js +const inputs = [{ query: 'a' }, { query: 'b' }, { query: 'c' } /* ... */]; +const CHUNK = 5; +const results = []; +for (let i = 0; i < inputs.length; i += CHUNK) { + const batch = inputs.slice(i, i + CHUNK); + const batchResults = await Promise.all( + batch.map((input) => apify.actor.callAndGetItems({ actorId: 'apify/rag-web-browser', input, limit: 5 })), + ); + results.push(...batchResults.flatMap((r) => r.items)); +} +console.log(JSON.stringify(results.slice(0, 5))); // small summary, not the full dump +``` + +### Runs longer than 60s: start, then poll + +`actor.call`'s wait is capped at 60s per request (a REST API limit, not this +Actor's). For a longer-running Actor, start it and poll: + +```js +let run = await apify.actor.start({ actorId, input }); +const TERMINAL = ['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT']; +while (!TERMINAL.includes(run.status)) { + run = await apify.run.waitForFinish({ runId: run.id, waitForFinishSecs: 60 }); +} +``` + +## Limitations + +Actor runs launched from inside the sandbox (via the `apify` binding) are +recorded as ordinary Actor runs — they aren't attributed back to the MCP +session that ultimately triggered them. If you're measuring "Actor runs +driven by MCP," Code Mode's sub-runs won't show up as such. Known, +unresolved, tracked separately from this Actor. ## The `apify` binding @@ -90,12 +182,12 @@ Every method takes one options object and returns parsed JSON apify.actor.search({ query, limit?, category? }) // → actors[] apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run -apify.actor.run({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) -apify.actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } +apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) +apify.actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } // Runs apify.run.get({ runId }) // → run -apify.run.wait({ runId, waitForFinishSecs = 60 }) // → run +apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run apify.run.abort({ runId }) // → run apify.run.getLog({ runId, limit? }) // → string @@ -104,7 +196,7 @@ apify.dataset.create({ name? }) // → dataset apify.dataset.pushItems({ datasetId, items }) // → void apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) // → items[] apify.dataset.iterate({ datasetId, batchSize = 1000, ...filters }) // → async iterable -apify.dataset.getSchema({ datasetId, sample = 5 }) // → { itemCount, fields[] } +apify.dataset.inferFields({ datasetId, sample = 5 }) // → { itemCount, fields[] } // Key-value stores apify.kvs.create({ name? }) // → store diff --git a/docs/API.md b/docs/API.md index b57da64..ad3e5b3 100644 --- a/docs/API.md +++ b/docs/API.md @@ -77,7 +77,7 @@ Start an Actor **asynchronously** and return immediately with a run record in **Output:** the Run object (unwrapped `data`). **Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) -### `actor.run({ actorId, input?, waitForFinishSecs?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` +### `actor.call({ actorId, input?, waitForFinishSecs?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` Start an Actor and **wait** for it to finish (or until `waitForFinishSecs` elapses), then return the run record. @@ -86,7 +86,7 @@ elapses), then return the run record. |---|---|---|---|---| | `actorId` | `string` | yes | | `username/name` or Actor ID. | | `input` | `object` | no | `{}` | Actor input. | -| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.wait()` loop. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.waitForFinish()` loop. | | `memoryMbytes` | `number` | no | | Memory limit. | | `timeoutSecs` | `number` | no | | Run timeout. | | `maxTotalChargeUsd` | `number` | no | | Cost cap. | @@ -97,9 +97,9 @@ elapses), then return the run record. (not `/run-sync`, which returns the output record instead of the run object). **Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) -### `actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` +### `actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` -Convenience wrapper: `actor.run(...)` followed by reading the run's default +Convenience wrapper: `actor.call(...)` followed by reading the run's default dataset via `dataset.listItems`. | Param | Type | Required | Description | @@ -108,13 +108,13 @@ dataset via `dataset.listItems`. | `input` | `object` | no | Actor input. | | `fields` | `string[]` | no | Restrict returned item fields. | | `limit` | `number` | no | Max items to fetch. | -| `...runOpts` | | no | Any `actor.run` option (`waitForFinishSecs`, `memoryMbytes`, `timeoutSecs`, `maxTotalChargeUsd`, `maxItems`). | +| `...runOpts` | | no | Any `actor.call` option (`waitForFinishSecs`, `memoryMbytes`, `timeoutSecs`, `maxTotalChargeUsd`, `maxItems`). | **Output (custom):** ```js { - run: Run, // the run object, as actor.run returns + run: Run, // the run object, as actor.call returns items: object[] // items from run.defaultDatasetId } ``` @@ -123,7 +123,7 @@ dataset via `dataset.listItems`. then [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) ```js -const { run, items } = await apify.actor.runAndGetItems({ +const { run, items } = await apify.actor.callAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3, @@ -146,7 +146,7 @@ Fetch the current run record (status, stats, default storage IDs). **Output:** the Run object (unwrapped `data`). **Apify API:** [`GET /v2/actor-runs/{runId}`](https://docs.apify.com/api/v2/actor-run-get) -### `run.wait({ runId, waitForFinishSecs? })` → `Run` +### `run.waitForFinish({ runId, waitForFinishSecs? })` → `Run` Block until the run terminates or `waitForFinishSecs` elapses, whichever comes first, then return the run record. @@ -228,7 +228,7 @@ Read a page of items. **Output (custom):** the **items array directly** — this endpoint already returns a bare array (no `data`/pagination wrapper). A dataset's pagination total is eventually consistent right after creation, so no `total` is surfaced; -use [`getSchema`](#datasetgetschema--schema) for a count or +use [`inferFields`](#datasetinferfields--schema) for a count or [`iterate`](#datasetiterate--asyncgeneratorobject) to consume everything. **Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) @@ -255,7 +255,7 @@ for await (const item of apify.dataset.iterate({ datasetId })) count++; console.log('total items:', count); ``` -### `dataset.getSchema({ datasetId, sample? })` → `Schema` +### `dataset.inferFields({ datasetId, sample? })` → `Schema` Infer a lightweight schema from a sample of items (Apify has no schema endpoint). diff --git a/package.json b/package.json index 8de1c9f..b8152c7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "code-runtime", "version": "0.1.0", - "description": "workerd as a normal (per-run) Apify Actor; one V8 isolate per run via the Worker Loader API.", + "description": "workerd as a normal (per-run) Apify Actor: one static worker, booted once per run, runs the submitted script and exits.", "private": true, "packageManager": "pnpm@11.1.3", "dependencies": { From 9d251d4001ff6d97bf0894ca4487fca998c989fb Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 14 Jul 2026 10:22:21 +0200 Subject: [PATCH 21/46] refactor: migrate tests/*.js probes to TypeScript, fix stale run.wait doc link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tests/binding-smoke.ts, tests/sandbox-isolation.ts: typed against the same ApifyBinding surface runner.ts exposes at runtime (exported type-only from runner.ts), and updated to the renamed methods (get, call, callAndGetItems, waitForFinish, inferFields) that a prior commit renamed but these probes still called under their old names. - tests/globals.d.ts: ambient apify/process/require declarations via `declare global` for these standalone-compiled probe files (apify/console are real function parameters at runtime once entrypoint.sh splices the compiled JS into the code input, not globals). - tsconfig.json: tests/*.ts now type-checked alongside worker/*.ts. - package.json build: strips the `export {};` marker tsc appends to each probe (needed so tsc treats each file as its own module — otherwise their top-level consts collide across files, and top-level await needs a module). Left in, that line would be a syntax error once spliced into the wrapping `async function run(apify, console) { ... }`. Verified with node --check against the actual wrapped shape. - test.sh: runs `pnpm build` before pushing/calling so probes compile first. - docs/API.md: fix a stale `run.wait` anchor missed by an earlier rename commit (actual method is `run.waitForFinish`). --- .gitignore | 3 +- docs/API.md | 2 +- package.json | 2 +- test.sh | 6 ++- tests/{binding-smoke.js => binding-smoke.ts} | 52 +++++++++++-------- tests/globals.d.ts | 19 +++++++ ...dbox-isolation.js => sandbox-isolation.ts} | 26 ++++++---- tsconfig.json | 2 +- worker/runner.ts | 5 ++ 9 files changed, 81 insertions(+), 36 deletions(-) rename tests/{binding-smoke.js => binding-smoke.ts} (64%) create mode 100644 tests/globals.d.ts rename tests/{sandbox-isolation.js => sandbox-isolation.ts} (81%) diff --git a/.gitignore b/.gitignore index e347e03..39046c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ node_modules/ # Generated at container startup by entrypoint.sh (never checked in): worker/usercode.js -# Compiled from worker/*.ts by `pnpm build` (tsconfig.json emits next to the source): +# Compiled from worker/*.ts and tests/*.ts by `pnpm build` (tsconfig.json emits next to source): worker/runner.js worker/guard.js +tests/*.js *.log .DS_Store diff --git a/docs/API.md b/docs/API.md index ad3e5b3..66331e6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -63,7 +63,7 @@ Fetch the full record for one Actor. ### `actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` Start an Actor **asynchronously** and return immediately with a run record in -`READY`/`RUNNING` state. Use [`run.wait`](#runwait--run) to block for the result. +`READY`/`RUNNING` state. Use [`run.waitForFinish`](#runwaitforfinish--run) to block for the result. | Param | Type | Required | Description | |---|---|---|---| diff --git a/package.json b/package.json index b8152c7..4bd7ecd 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "node": ">=24" }, "scripts": { - "build": "tsc -p tsconfig.json", + "build": "tsc -p tsconfig.json && sed -i '/^export {};$/d' tests/*.js", "typecheck": "tsc --noEmit -p tsconfig.json" }, "devDependencies": { diff --git a/test.sh b/test.sh index ed09ad5..2184a65 100755 --- a/test.sh +++ b/test.sh @@ -8,12 +8,16 @@ set -eu cd "$(dirname "$0")" -# Probes run against the built Actor. Add a file here to register a new probe. +# Probes are written in tests/*.ts and compiled by `pnpm build`; add a .ts file +# there to register a new probe. Run against the built Actor. PROBES="tests/binding-smoke.js tests/sandbox-isolation.js" command -v apify >/dev/null 2>&1 || { echo "apify CLI not found" >&2; exit 1; } command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 1; } +echo "==> pnpm build" +pnpm build + input_json="$(mktemp)" trap 'rm -f "$input_json"' EXIT diff --git a/tests/binding-smoke.js b/tests/binding-smoke.ts similarity index 64% rename from tests/binding-smoke.js rename to tests/binding-smoke.ts index 98b5b55..c7b2369 100644 --- a/tests/binding-smoke.js +++ b/tests/binding-smoke.ts @@ -2,14 +2,22 @@ // the Actor's `code` input by test.sh and executed on the built Actor via // `apify call`. Exercises every binding method and prints a sentinel line // (ALL_TESTS_PASSED) that test.sh greps for. -const results = []; -async function check(name, fn) { +// +// `export {}` marks this file as its own ES module, so its top-level consts +// don't collide with the other probe's (both are type-checked in one tsc +// program) and top-level await below is legal. `pnpm build` strips this line +// post-compile (see package.json) -- left in, it would be a syntax error once +// spliced into the wrapping `async function run(apify, console) { ... }`. +export {}; + +const results: boolean[] = []; +async function check(name: string, fn: () => Promise): Promise { try { const out = await fn(); console.log(`PASS ${name}: ${out ?? ''}`); results.push(true); } catch (e) { - console.error(`FAIL ${name}: ${e.message}`); + console.error(`FAIL ${name}: ${(e as Error).message}`); results.push(false); } } @@ -22,15 +30,15 @@ await check('actor.search', async () => { if (!Array.isArray(items)) throw new Error('expected array'); return `${items.length} actors`; }); -await check('actor.getDetails', async () => { - const d = await apify.actor.getDetails({ actorId: ACTOR }); +await check('actor.get', async () => { + const d = await apify.actor.get({ actorId: ACTOR }); return `${d.username}/${d.name}`; }); // ---- dataset ---- -let datasetId; +let datasetId = ''; await check('dataset.create', async () => { - datasetId = (await apify.dataset.create()).id; + datasetId = (await apify.dataset.create()).id as string; return datasetId; }); await check('dataset.pushItems', async () => { @@ -41,8 +49,8 @@ await check('dataset.listItems', async () => { const items = await apify.dataset.listItems({ datasetId }); return `${items.length} items`; }); -await check('dataset.getSchema', async () => { - const s = await apify.dataset.getSchema({ datasetId }); +await check('dataset.inferFields', async () => { + const s = await apify.dataset.inferFields({ datasetId }); return `itemCount=${s.itemCount} fields=${s.fields.map((f) => f.name).join(',')}`; }); await check('dataset.iterate', async () => { @@ -52,9 +60,9 @@ await check('dataset.iterate', async () => { }); // ---- key-value store ---- -let storeId; +let storeId = ''; await check('kvs.create', async () => { - storeId = (await apify.kvs.create()).id; + storeId = (await apify.kvs.create()).id as string; return storeId; }); await check('kvs.set', async () => { @@ -63,46 +71,46 @@ await check('kvs.set', async () => { return 'set obj + txt'; }); await check('kvs.get', async () => { - const obj = await apify.kvs.get({ storeId, key: 'obj' }); + const obj = await apify.kvs.get({ storeId, key: 'obj' }) as { hello: string }; const txt = await apify.kvs.get({ storeId, key: 'txt' }); const missing = await apify.kvs.get({ storeId, key: 'nope' }); return `obj.hello=${obj.hello} txt=${txt} missing=${missing}`; }); await check('kvs.list', async () => { - const l = await apify.kvs.list({ storeId }); + const l = await apify.kvs.list({ storeId }) as { items: unknown[] }; return `${l.items.length} keys`; }); // ---- run lifecycle ---- -let runId; +let runId = ''; await check('actor.start', async () => { const run = await apify.actor.start({ actorId: ACTOR }); - runId = run.id; + runId = run.id as string; return `runId=${runId} status=${run.status}`; }); await check('run.get', async () => { return `status=${(await apify.run.get({ runId })).status}`; }); -await check('run.wait', async () => { - return `status=${(await apify.run.wait({ runId, waitForFinishSecs: 60 })).status}`; +await check('run.waitForFinish', async () => { + return `status=${(await apify.run.waitForFinish({ runId, waitForFinishSecs: 60 })).status}`; }); await check('run.getLog', async () => { return `${(await apify.run.getLog({ runId, limit: 200 })).length} chars`; }); // ---- run + get items (sync) ---- -await check('actor.run', async () => { - return `status=${(await apify.actor.run({ actorId: ACTOR, waitForFinishSecs: 60 })).status}`; +await check('actor.call', async () => { + return `status=${(await apify.actor.call({ actorId: ACTOR, waitForFinishSecs: 60 })).status}`; }); -await check('actor.runAndGetItems', async () => { - const { run, items } = await apify.actor.runAndGetItems({ actorId: ACTOR, limit: 5, waitForFinishSecs: 60 }); +await check('actor.callAndGetItems', async () => { + const { run, items } = await apify.actor.callAndGetItems({ actorId: ACTOR, limit: 5, waitForFinishSecs: 60 }); return `status=${run.status} items=${items.length}`; }); // ---- abort ---- await check('run.abort', async () => { const run = await apify.actor.start({ actorId: ACTOR }); - return `status=${(await apify.run.abort({ runId: run.id })).status}`; + return `status=${(await apify.run.abort({ runId: run.id as string })).status}`; }); const passed = results.filter(Boolean).length; diff --git a/tests/globals.d.ts b/tests/globals.d.ts new file mode 100644 index 0000000..2114679 --- /dev/null +++ b/tests/globals.d.ts @@ -0,0 +1,19 @@ +// Ambient declarations for probes in this directory. Each probe is compiled +// standalone by tsc, then submitted as the Actor's `code` input — entrypoint.sh +// splices that text into `export async function run(apify, console) { ... }` +// (see worker/usercode.d.ts), so `apify` and `console` are real function +// parameters at runtime, not globals. `declare global` here only satisfies the +// compiler for these standalone probe files; nothing in this file is emitted to JS. +import type { ApifyBinding } from '../worker/runner.js'; + +declare global { + const apify: ApifyBinding; + + // Sandbox-isolation probe checks `typeof process`/`typeof require` — both are + // genuinely absent at runtime (no nodejs_compat). `typeof` never throws on an + // undeclared identifier, so declaring these here doesn't change that: a + // `declare` emits no JS, so the runtime binding stays exactly as absent as + // the probe expects. + const process: unknown; + const require: unknown; +} diff --git a/tests/sandbox-isolation.js b/tests/sandbox-isolation.ts similarity index 81% rename from tests/sandbox-isolation.js rename to tests/sandbox-isolation.ts index fda8b3d..3019095 100644 --- a/tests/sandbox-isolation.js +++ b/tests/sandbox-isolation.ts @@ -8,8 +8,16 @@ // *.apify.com fetch allowlist — finding A) and process.env (which held the // run's APIFY_TOKEN — finding B), and makes the "no imports" docs accurate // (finding C). fetch() and the apify binding must still work. -const results = []; -function check(name, cond, detail = '') { +// +// `export {}` marks this file as its own ES module, so its top-level consts +// don't collide with the other probe's (both are type-checked in one tsc +// program) and top-level await below is legal. `pnpm build` strips this line +// post-compile (see package.json) -- left in, it would be a syntax error once +// spliced into the wrapping `async function run(apify, console) { ... }`. +export {}; + +const results: boolean[] = []; +function check(name: string, cond: boolean, detail = ''): void { if (cond) { console.log(`PASS ${name}: ${detail}`); results.push(true); @@ -38,12 +46,12 @@ check('fetch available', typeof fetch === 'function', `typeof fetch = ${typeof f // synchronously with a "Blocked fetch" message BEFORE any network I/O; anything // else (a real network/HTTP error) means guard let the request through. So we // classify by the error message, not by whether the request ultimately succeeds. -async function guardBlocks(url) { +async function guardBlocks(url: string): Promise { try { await fetch(url); return false; // request went out — guard allowed it } catch (e) { - return /Blocked fetch/.test(e.message); // guard rejection vs. network error + return /Blocked fetch/.test((e as Error).message); // guard rejection vs. network error } } @@ -69,14 +77,14 @@ for (const url of blockedTargets) { // are web-standard globals that connect directly (not through the fetch guard), // so a script could otherwise open a wss:// / SSE channel to any public host and // exfiltrate around the *.apify.com allowlist (apify/ai-team#216 finding A). -function blocksConstruct(name, url) { - const Ctor = globalThis[name]; +function blocksConstruct(name: string, url: string): boolean { + const Ctor = (globalThis as Record)[name]; if (typeof Ctor !== 'function') return true; // absent → not an egress path try { - new Ctor(url); + new (Ctor as new (u: string) => unknown)(url); return false; // constructed → egress opened } catch (e) { - return /Blocked/.test(e.message); // our guard rejection vs. any other error + return /Blocked/.test((e as Error).message); // our guard rejection vs. any other error } } check('WebSocket blocked', blocksConstruct('WebSocket', 'wss://echo.websocket.org'), 'no wss egress'); @@ -88,7 +96,7 @@ try { const found = await apify.actor.search({ query: 'hello world', limit: 1 }); bindingWorks = Array.isArray(found); } catch (e) { - console.error(`apify.actor.search threw: ${e.message}`); + console.error(`apify.actor.search threw: ${(e as Error).message}`); } check('apify binding works', bindingWorks, bindingWorks ? 'actor.search ok' : 'binding broken'); diff --git a/tsconfig.json b/tsconfig.json index ac781ef..704b2ad 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,5 +6,5 @@ "lib": ["es2022", "dom"], "strict": true }, - "include": ["worker/*.ts"] + "include": ["worker/*.ts", "tests/*.ts"] } diff --git a/worker/runner.ts b/worker/runner.ts index d762d4a..da55a5d 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -434,6 +434,11 @@ function makeApifyBinding(token: string, apiV2: string) { }); } +// The shape handed to user code as the `apify` binding. Exported (type-only — +// erased at compile time) so tests/*.ts can type-check probes against the same +// surface real usercode.js runs against, without importing runner.ts at runtime. +export type ApifyBinding = ReturnType; + // Push the captured streams as a single item to the run's default dataset. async function pushOutput(apiV2: string, token: string, env: Env, item: OutputItem): Promise { const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; From 2169a10d524f5d77eaae42839ab6c894a3e18d5e Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 14 Jul 2026 11:06:03 +0200 Subject: [PATCH 22/46] refactor!: apify.store top-level binding, kvs -> keyValueStore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revisits the two renames the council flagged as bikeshedding and deferred to v2 (now resolved per your call: breaking changes are fine, this is a POC). - api.apify.com/v2/openapi.json tags GET /v2/store as its own top-level 'Store' resource (operationId store_get), sibling to 'Actors', not a sub-resource of it. Moved actor.search() out of the actor.* namespace to a bare apify.store({ search, limit?, category? }) binding to match — the Store resource has exactly one operation, so a namespace object would be one method wrapping nothing. - Renamed the search param query -> search to match the API's own field name (GET /v2/store?search=...), so the options object no longer invents a name the wire format doesn't use. - Renamed kvs -> keyValueStore (namespace + all Kvs*Options interfaces) to match the API's own key-value-stores resource name instead of an abbreviation found nowhere in the Apify API itself. - README, docs/API.md, tests/*.ts updated to match. docs/API.md gets its own ## apify.store section ahead of ## apify.actor. --- README.md | 12 +++++++----- docs/API.md | 30 +++++++++++++++++++----------- tests/binding-smoke.ts | 26 +++++++++++++------------- tests/sandbox-isolation.ts | 6 +++--- worker/runner.ts | 35 +++++++++++++++++++---------------- 5 files changed, 61 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index c028a08..fe5cb9c 100644 --- a/README.md +++ b/README.md @@ -178,8 +178,10 @@ Every method takes one options object and returns parsed JSON [here](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). ```js +// Store — GET /v2/store, a top-level Apify API resource (not an Actor method) +apify.store({ search, limit?, category? }) // → actors[] + // Actors -apify.actor.search({ query, limit?, category? }) // → actors[] apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) @@ -199,10 +201,10 @@ apify.dataset.iterate({ datasetId, batchSize = 1000, ...filters }) // → async apify.dataset.inferFields({ datasetId, sample = 5 }) // → { itemCount, fields[] } // Key-value stores -apify.kvs.create({ name? }) // → store -apify.kvs.set({ storeId, key, value, contentType? }) // → void -apify.kvs.get({ storeId, key }) // → value | null -apify.kvs.list({ storeId, limit?, exclusiveStartKey? }) // → { items } +apify.keyValueStore.create({ name? }) // → store +apify.keyValueStore.set({ storeId, key, value, contentType? }) // → void +apify.keyValueStore.get({ storeId, key }) // → value | null +apify.keyValueStore.list({ storeId, limit?, exclusiveStartKey? }) // → { items } ``` ## Learn more diff --git a/docs/API.md b/docs/API.md index 66331e6..f5bd5b8 100644 --- a/docs/API.md +++ b/docs/API.md @@ -22,21 +22,25 @@ document describes every method in detail. live request/response schema. - **Errors.** A non-2xx API response throws an `Error` whose message is ` failed: `. The one exception is - [`kvs.get`](#kvsget--value--null), which returns `null` for a missing key. + [`keyValueStore.get`](#keyvaluestoreget--value--null), which returns `null` for a missing key. - **Network.** Outbound `fetch` from your script is restricted to `apify.com` and its subdomains. --- -## `apify.actor` +## `apify.store` + +Apify's own API tags this endpoint `Store` — a top-level resource, not an +Actor method — so the binding mirrors that: `apify.store(...)`, not +`apify.actor.store(...)`. -### `actor.search({ query, limit?, category? })` → `Actor[]` +### `apify.store({ search, limit?, category? })` → `Actor[]` Search the Apify Store. | Param | Type | Required | Description | |---|---|---|---| -| `query` | `string` | yes | Full-text search query. | +| `search` | `string` | yes | Full-text search query. | | `limit` | `number` | no | Maximum number of results. | | `category` | `string` | no | Restrict to a Store category. | @@ -45,10 +49,14 @@ is dropped) — i.e. an `Actor[]`. **Apify API:** [`GET /v2/store`](https://docs.apify.com/api/v2/store-get) ```js -const actors = await apify.actor.search({ query: 'web scraper', limit: 5 }); +const actors = await apify.store({ search: 'web scraper', limit: 5 }); console.log(actors.map((a) => `${a.username}/${a.name}`).join('\n')); ``` +--- + +## `apify.actor` + ### `actor.get({ actorId })` → `Actor` Fetch the full record for one Actor. @@ -285,9 +293,9 @@ Infer a lightweight schema from a sample of items (Apify has no schema endpoint) --- -## `apify.kvs` +## `apify.keyValueStore` -### `kvs.create({ name? })` → `Store` +### `keyValueStore.create({ name? })` → `KeyValueStore` Create a key-value store and return its record. @@ -295,10 +303,10 @@ Create a key-value store and return its record. |---|---|---|---| | `name` | `string` | no | Named (persistent) store; omit for an unnamed (temporary) one. | -**Output:** the Store object (unwrapped `data`). +**Output:** the key-value store's record object (unwrapped `data`). **Apify API:** [`POST /v2/key-value-stores`](https://docs.apify.com/api/v2/key-value-stores-post) -### `kvs.set({ storeId, key, value, contentType? })` → `void` +### `keyValueStore.set({ storeId, key, value, contentType? })` → `void` Write a record. The content type is inferred from `value`: @@ -318,7 +326,7 @@ Write a record. The content type is inferred from `value`: **Output:** none (resolves once the record is stored). **Apify API:** [`PUT /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-put) -### `kvs.get({ storeId, key })` → `value` \| `null` +### `keyValueStore.get({ storeId, key })` → `value` \| `null` Read a record. @@ -337,7 +345,7 @@ Returns **`null`** when the key does not exist (404) instead of throwing, so you can do lookup-or-default without a `try/catch`. **Apify API:** [`GET /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-get) -### `kvs.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, … }` +### `keyValueStore.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, … }` List keys in a store. diff --git a/tests/binding-smoke.ts b/tests/binding-smoke.ts index c7b2369..76fa9f5 100644 --- a/tests/binding-smoke.ts +++ b/tests/binding-smoke.ts @@ -25,8 +25,8 @@ async function check(name: string, fn: () => Promise): Promise { const ACTOR = 'apify/hello-world'; // ---- actor (read) ---- -await check('actor.search', async () => { - const items = await apify.actor.search({ query: 'hello world', limit: 3 }); +await check('store', async () => { + const items = await apify.store({ search: 'hello world', limit: 3 }); if (!Array.isArray(items)) throw new Error('expected array'); return `${items.length} actors`; }); @@ -61,23 +61,23 @@ await check('dataset.iterate', async () => { // ---- key-value store ---- let storeId = ''; -await check('kvs.create', async () => { - storeId = (await apify.kvs.create()).id as string; +await check('keyValueStore.create', async () => { + storeId = (await apify.keyValueStore.create()).id as string; return storeId; }); -await check('kvs.set', async () => { - await apify.kvs.set({ storeId, key: 'obj', value: { hello: 'world' } }); - await apify.kvs.set({ storeId, key: 'txt', value: 'plain' }); +await check('keyValueStore.set', async () => { + await apify.keyValueStore.set({ storeId, key: 'obj', value: { hello: 'world' } }); + await apify.keyValueStore.set({ storeId, key: 'txt', value: 'plain' }); return 'set obj + txt'; }); -await check('kvs.get', async () => { - const obj = await apify.kvs.get({ storeId, key: 'obj' }) as { hello: string }; - const txt = await apify.kvs.get({ storeId, key: 'txt' }); - const missing = await apify.kvs.get({ storeId, key: 'nope' }); +await check('keyValueStore.get', async () => { + const obj = await apify.keyValueStore.get({ storeId, key: 'obj' }) as { hello: string }; + const txt = await apify.keyValueStore.get({ storeId, key: 'txt' }); + const missing = await apify.keyValueStore.get({ storeId, key: 'nope' }); return `obj.hello=${obj.hello} txt=${txt} missing=${missing}`; }); -await check('kvs.list', async () => { - const l = await apify.kvs.list({ storeId }) as { items: unknown[] }; +await check('keyValueStore.list', async () => { + const l = await apify.keyValueStore.list({ storeId }) as { items: unknown[] }; return `${l.items.length} keys`; }); diff --git a/tests/sandbox-isolation.ts b/tests/sandbox-isolation.ts index 3019095..335032d 100644 --- a/tests/sandbox-isolation.ts +++ b/tests/sandbox-isolation.ts @@ -93,12 +93,12 @@ check('EventSource blocked', blocksConstruct('EventSource', 'https://example.com // The apify binding must still work (fetch to *.apify.com). let bindingWorks = false; try { - const found = await apify.actor.search({ query: 'hello world', limit: 1 }); + const found = await apify.store({ search: 'hello world', limit: 1 }); bindingWorks = Array.isArray(found); } catch (e) { - console.error(`apify.actor.search threw: ${(e as Error).message}`); + console.error(`apify.store threw: ${(e as Error).message}`); } -check('apify binding works', bindingWorks, bindingWorks ? 'actor.search ok' : 'binding broken'); +check('apify binding works', bindingWorks, bindingWorks ? 'store ok' : 'binding broken'); const passed = results.filter(Boolean).length; console.log(`\n=== SUMMARY: ${passed}/${results.length} passed ===`); diff --git a/worker/runner.ts b/worker/runner.ts index da55a5d..5b6fe41 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -54,8 +54,8 @@ interface ApiCallOptions { contentType?: string; } -interface SearchOptions { - query: string; +interface StoreSearchOptions { + search: string; limit?: number; category?: string; } @@ -125,17 +125,17 @@ interface PushItemsOptions { items: unknown[]; } -interface KvsGetOptions { +interface KeyValueStoreGetOptions { storeId: string; key: string; } -interface KvsSetOptions extends KvsGetOptions { +interface KeyValueStoreSetOptions extends KeyValueStoreGetOptions { value: unknown; contentType?: string; } -interface KvsListOptions { +interface KeyValueStoreListOptions { storeId: string; limit?: number; exclusiveStartKey?: string; @@ -250,11 +250,6 @@ function makeApifyBinding(token: string, apiV2: string) { }); const actor = { - // GET /v2/store — Apify Store search. Returns the items array directly. - search: ({ query, limit, category }: SearchOptions): Promise => - apiData('GET', '/store', { searchParams: { search: query, limit, category } }) - .then((page: { items: ApifyRecord[] }) => page.items), - get: ({ actorId }: ActorIdOptions): Promise => apiData('GET', `/acts/${encodeURIComponent(actorId)}`), @@ -378,16 +373,16 @@ function makeApifyBinding(token: string, apiV2: string) { }, }; - const kvs = { + const keyValueStore = { // Returns the value directly (parsed when JSON, string when text/*, Uint8Array otherwise). // Returns null when the key does not exist (404), not an error — this matches the common // "lookup or default" pattern in code. - get: async ({ storeId, key }: KvsGetOptions): Promise => { + get: async ({ storeId, key }: KeyValueStoreGetOptions): Promise => { const response = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { headers: baseHeaders, }); if (response.status === 404) return null; - if (!response.ok) throw new Error(`GET kvs.get failed: ${response.status} ${await response.text()}`); + if (!response.ok) throw new Error(`GET keyValueStore.get failed: ${response.status} ${await response.text()}`); const contentType = response.headers.get('content-type') ?? ''; if (contentType.includes('application/json')) return response.json(); if (contentType.startsWith('text/')) return response.text(); @@ -396,7 +391,7 @@ function makeApifyBinding(token: string, apiV2: string) { // `value`: object → application/json; string → text/plain; Uint8Array/ArrayBuffer → // application/octet-stream (or whatever the caller passed via `contentType`). - set: async ({ storeId, key, value, contentType }: KvsSetOptions): Promise => { + set: async ({ storeId, key, value, contentType }: KeyValueStoreSetOptions): Promise => { let body: BodyInit; let resolvedContentType = contentType; if (value instanceof Uint8Array || value instanceof ArrayBuffer) { @@ -415,7 +410,7 @@ function makeApifyBinding(token: string, apiV2: string) { }); }, - list: ({ storeId, limit, exclusiveStartKey }: KvsListOptions): Promise => + list: ({ storeId, limit, exclusiveStartKey }: KeyValueStoreListOptions): Promise => apiData('GET', `/key-value-stores/${encodeURIComponent(storeId)}/keys`, { searchParams: { limit, exclusiveStartKey }, }), @@ -424,13 +419,21 @@ function makeApifyBinding(token: string, apiV2: string) { apiData('POST', '/key-value-stores', { searchParams: { name } }), }; + // GET /v2/store — Apify Store search (a top-level resource in the Apify API, + // tagged `Store`, distinct from `Actors` — hence a top-level binding rather + // than an `actor.*` method). Returns the items array directly. + const store = ({ search, limit, category }: StoreSearchOptions): Promise => + apiData('GET', '/store', { searchParams: { search, limit, category } }) + .then((page: { items: ApifyRecord[] }) => page.items); + // Freeze every namespace (and the wrapper) so the script can't reassign a method to // corrupt its own behavior or, for `console` below, its own output capture. return Object.freeze({ actor: Object.freeze(actor), + store, run: Object.freeze(run), dataset: Object.freeze(dataset), - kvs: Object.freeze(kvs), + keyValueStore: Object.freeze(keyValueStore), }); } From f23e4e3f738b754e966f83f94995c7b17afeda3f Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 14 Jul 2026 11:35:22 +0200 Subject: [PATCH 23/46] feat: forward X-Apify-Request-Origin: MCP to sub-runs when this run is MCP-started Closes (partially) the MCP-attribution gap flagged in README's Limitations. Verified end-to-end against apify-core / apify-worker source (not just docs): - apify-mcp-server sends X-Apify-Request-Origin: MCP on every API call (src/apify_client.ts). - apify-core's requestOriginParserMiddleware validates it against META_ORIGINS (includes MCP) and sets req.origin (src/api/src/middleware/request_origin_parser.ts). - actor_jobs.ts's createMeta(req.origin || defaultOrigin, ...) -- defaultOrigin is META_ORIGINS.ACTOR for actor-run-token callers -- stores it as the new run's meta.origin (src/api/src/lib/actor_jobs.ts:374-378). - apify-worker injects it back into the container as APIFY_META_ORIGIN (act2_run_job.ts:2212, APIFY_ENV_VARS.META_ORIGIN = "APIFY_META_ORIGIN", matches apify-docs' environment_variables.md). So this Actor's own container already receives APIFY_META_ORIGIN=MCP when apify-mcp-server started it -- no mcp-server change, no new Actor input field. Rejected a hidden `isMCPRun` input field: call-actor's tool schema is built directly from actor.json, so a hidden field is either LLM-visible-and-spoofable or never set by anyone. APIFY_META_ORIGIN is platform-injected and unspoofable from inside the sandbox. - worker/config.capnp: bind PARENT_ORIGIN from APIFY_META_ORIGIN (same pattern as the existing APIFY_TOKEN binding). - worker/runner.ts: makeApifyBinding() takes parentOrigin; sends X-Apify-Request-Origin: MCP on all its own API calls only when parentOrigin === 'MCP'. Also sets a plain User-Agent (previously unset). - README: rewrote Limitations to document the (already-existing, no-code-needed) meta.actorRunId parent-run link apify-core sets on every sub-run automatically, and what this change adds on top of it. Verified live with the real workerd binary + a mock Apify API asserting the header: absent when APIFY_META_ORIGIN is unset, absent when it's 'ACTOR', present as 'MCP' only when it's 'MCP'. --- README.md | 19 ++++++++++++++----- worker/config.capnp | 3 +++ worker/runner.ts | 30 +++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fe5cb9c..b6ac87f 100644 --- a/README.md +++ b/README.md @@ -165,11 +165,20 @@ while (!TERMINAL.includes(run.status)) { ## Limitations -Actor runs launched from inside the sandbox (via the `apify` binding) are -recorded as ordinary Actor runs — they aren't attributed back to the MCP -session that ultimately triggered them. If you're measuring "Actor runs -driven by MCP," Code Mode's sub-runs won't show up as such. Known, -unresolved, tracked separately from this Actor. +Sub-runs started from inside the sandbox (via `apify.actor.start/call/callAndGetItems`) +get `meta.origin: 'MCP'` on the Apify platform, same as this Actor's own run, when +this Actor was itself started via the Apify MCP Server — this Actor reads its own +`APIFY_META_ORIGIN` env var (platform-set, not spoofable from inside the sandbox) +and forwards `X-Apify-Request-Origin: MCP` on its own API calls only when that's +`MCP`. Every sub-run also gets a platform-native `meta.actorRunId` link back to +this run regardless of origin (set automatically from the run-scoped token, no +code needed here) — so even a fully generic query can already walk sub-run → +`meta.actorRunId` → parent `meta.origin` to reconstruct the chain; the origin +forwarding above just makes single-field origin queries work without that join. + +What's still not attributed: the specific MCP *session* (which client, which +conversation) that triggered this Actor's own run in the first place — that +context isn't part of the platform's Run schema at all, on or off Code Mode. ## The `apify` binding diff --git a/worker/config.capnp b/worker/config.capnp index c5d380b..c78ebbb 100644 --- a/worker/config.capnp +++ b/worker/config.capnp @@ -34,6 +34,9 @@ const codeRuntime :Workerd.Worker = ( (name = "DEFAULT_DATASET_ID", fromEnvironment = "ACTOR_DEFAULT_DATASET_ID"), (name = "DEFAULT_DATASET_ID_LEGACY", fromEnvironment = "APIFY_DEFAULT_DATASET_ID"), (name = "API_BASE_URL", fromEnvironment = "APIFY_API_BASE_URL"), + # This run's own meta.origin (e.g. "MCP" when apify-mcp-server started it), + # forwarded to sub-runs this script starts — see PARENT_ORIGIN in runner.ts. + (name = "PARENT_ORIGIN", fromEnvironment = "APIFY_META_ORIGIN"), ], globalOutbound = "internet", compatibilityDate = "2026-01-15", diff --git a/worker/runner.ts b/worker/runner.ts index 5b6fe41..2a323c9 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -153,6 +153,13 @@ interface Env { DEFAULT_DATASET_ID?: string; DEFAULT_DATASET_ID_LEGACY?: string; API_BASE_URL?: string; + // APIFY_META_ORIGIN, forwarded from the platform's own env var of the same + // name (bound as PARENT_ORIGIN in config.capnp). Reflects this run's own + // meta.origin, set by apify-core from the X-Apify-Request-Origin request + // header the caller sent when creating THIS run — 'MCP' when apify-mcp-server + // started it. Platform-injected, not user-settable: unlike an Actor input + // field, a script running inside this Actor cannot spoof it. + PARENT_ORIGIN?: string; } interface OutputItem { @@ -177,8 +184,25 @@ function errorDetail(err: unknown): string { return err instanceof Error && err.stack ? err.stack : errorMessage(err); } -function makeApifyBinding(token: string, apiV2: string) { - const baseHeaders: Record = { Authorization: `Bearer ${token}` }; +// 'MCP' matches apify-core's META_ORIGINS.MCP / apify-mcp-server's own +// X-Apify-Request-Origin header value — reusing the platform's existing +// convention rather than inventing a new one. +const MCP_ORIGIN = 'MCP'; +const REQUEST_ORIGIN_HEADER = 'X-Apify-Request-Origin'; + +function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | undefined) { + // Every request this Actor makes identifies itself; requests made while THIS + // run's own origin is MCP additionally forward that origin so runs started by + // apify.actor.start/call/callAndGetItems() below get meta.origin: 'MCP' too, + // instead of the platform's default meta.origin: 'ACTOR' for actor-to-actor + // calls. Gated on parentOrigin (verified server-side, see the Env.PARENT_ORIGIN + // comment) rather than any Actor input, so a script can't forge an origin this + // run wasn't actually started with. + const baseHeaders: Record = { + Authorization: `Bearer ${token}`, + 'User-Agent': 'apify-code-runtime', + ...(parentOrigin === MCP_ORIGIN ? { [REQUEST_ORIGIN_HEADER]: MCP_ORIGIN } : {}), + }; // Build a URL with optional query params; null/undefined values are dropped. const buildUrl = (path: string, searchParams?: SearchParams): URL => { @@ -490,7 +514,7 @@ export default { let exitCode = 0; let statusMessage = 'Script completed'; try { - await run(makeApifyBinding(token, apiV2), captureConsole); + await run(makeApifyBinding(token, apiV2, env.PARENT_ORIGIN), captureConsole); } catch (err) { stderr.push(errorDetail(err)); exitCode = 1; From 2b842803d34850402704857737f8b7d73bcb2276 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 14 Jul 2026 13:06:07 +0200 Subject: [PATCH 24/46] fix(ci): allow workerd postinstall build script under pnpm 11 pnpm 11 blocks dependency install scripts by default (ERR_PNPM_IGNORED_BUILDS) unless explicitly allowed. Adds pnpm-workspace.yaml with allowBuilds: { workerd: true } -- workerd's postinstall fetches its platform-specific binary, which is what CI's pnpm typecheck (and every other pnpm command) needs present. --- pnpm-workspace.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 pnpm-workspace.yaml diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..797a5c3 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + workerd: true From 5fbbc5328ec31fd3d399ba923e652ce618ba2bb6 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 14 Jul 2026 13:21:28 +0200 Subject: [PATCH 25/46] docs: add Chain Actors recipe, dataset/keyValueStore usage examples Closes 2 of the 4 doc-parity gaps found vs. apify-mcp-server#1044's original guide content (the other two -- discover-and-inspect snippet, Promise.allSettled fix for the fan-out recipe -- postponed): - README: Chain Actors recipe (one run's output feeds the next), using the renamed callAndGetItems. - docs/API.md: chained create -> pushItems -> listItems example for apify.dataset (previously signature tables only); chained create -> set -> get example for apify.keyValueStore (previously zero code examples in that section). --- README.md | 13 +++++++++++++ docs/API.md | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/README.md b/README.md index b6ac87f..3c5d7c2 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,19 @@ see [Limits & failure modes](#limits--failure-modes). ## Recipes +### Chain Actors (one run's output feeds the next) + +```js +const { items: results } = await apify.actor.callAndGetItems({ + actorId: 'apify/google-search-scraper', input: { queries: 'apify' }, limit: 10, +}); +const startUrls = results.flatMap((r) => r.organicResults ?? []).map((r) => ({ url: r.url })); +const { items: pages } = await apify.actor.callAndGetItems({ + actorId: 'apify/website-content-crawler', input: { startUrls }, +}); +console.log(JSON.stringify(pages.slice(0, 3).map((p) => p.url))); +``` + ### Bounded parallel fan-out This Actor's clearest win: run several independent Actors (or the same Actor diff --git a/docs/API.md b/docs/API.md index f5bd5b8..10ad7ae 100644 --- a/docs/API.md +++ b/docs/API.md @@ -219,6 +219,12 @@ Append one or more items to a dataset. **Output:** none (resolves once the items are stored). **Apify API:** [`POST /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-post) +```js +const ds = await apify.dataset.create(); +await apify.dataset.pushItems({ datasetId: ds.id, items: [{ a: 1 }, { a: 2 }] }); +const items = await apify.dataset.listItems({ datasetId: ds.id }); +``` + ### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `object[]` Read a page of items. @@ -345,6 +351,13 @@ Returns **`null`** when the key does not exist (404) instead of throwing, so you can do lookup-or-default without a `try/catch`. **Apify API:** [`GET /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-get) +```js +const kv = await apify.keyValueStore.create(); +await apify.keyValueStore.set({ storeId: kv.id, key: 'state', value: { seen: [] } }); +const state = await apify.keyValueStore.get({ storeId: kv.id, key: 'state' }); // → { seen: [] } +const missing = await apify.keyValueStore.get({ storeId: kv.id, key: 'nope' }); // → null +``` + ### `keyValueStore.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, … }` List keys in a store. From 0f610792e8f34d01f2970bc61f5f881c88d1e03d Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 16 Jul 2026 13:03:46 +0200 Subject: [PATCH 26/46] fix: use draft-07 dataset schema, not draft 2020-12 Actor build failed with: Dataset schema compilation failed with: no schema with key or ref "https://json-schema.org/draft/2020-12/schema" apify-core compiles storages.dataset.fields with a plain `new Ajv(...)` (src/packages/storages/src/dataset_validation.ts) - no Ajv2020 variant, no addMetaSchema for 2020-12, so that $schema value has no meta-schema to resolve against. Switch to draft-07, which this Ajv instance supports natively; the schema itself only uses draft-07-compatible keywords (type/properties/required/enum), so no semantic change. Also nudge the agent to read the README before using the Actor, added to the top-level description (277/300 chars). --- .actor/actor.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 3458a96..466ea41 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -2,7 +2,7 @@ "actorSpecification": 1, "name": "code-runtime", "title": "Code Runtime", - "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Best for data-heavy jobs: scrape hundreds+ places/items, chain Actor outputs, and filter/sort/aggregate in one billed run. Results land in the dataset.", + "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Best for data-heavy jobs: scrape hundreds+ places/items, chain Actor outputs, and filter/sort/aggregate in one billed run. Results land in the dataset. Read the README before use.", "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, @@ -43,7 +43,7 @@ "actorSpecification": 1, "description": "Present only if the script ran to completion (returned or threw). A run-level timeout or OOM kill produces zero items for that run — the calling Actor-run's own status (SUCCEEDED vs FAILED/TIMED-OUT) is the signal for that case, not item presence.", "fields": { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "stdout": { From 854d5364e92d877fe3ce6b156cf87ed49bb8954a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 16 Jul 2026 13:12:54 +0200 Subject: [PATCH 27/46] fix: Docker builder stage doesn't need tests/ compiled Build failed: sed: can't read tests/*.js: No such file or directory The builder stage only COPYs worker/ (tests/ is dev-only probe fixtures for test.sh, submitted as Actor input at run time - never part of the image, correctly never copied in). `pnpm run build` compiles both worker/*.ts and tests/*.ts per tsconfig, then sed's the tsc-appended `export {};` marker out of tests/*.js - that sed fails outright when the directory doesn't exist. Call tsc directly instead of the shared pnpm script: tsconfig's tests/*.ts include glob simply matches nothing when the directory is absent (no error), producing exactly worker/runner.js + worker/guard.js - the only two files stage 2 copies. Local dev / test.sh, where tests/ does exist, are unaffected. Verified: reproduced the exact original failure in an isolated copy of the builder context (no tests/), confirmed tsc alone succeeds and produces both worker/*.js files, then ran the full multi-stage Dockerfile build to completion with podman. --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9fe4169..fe47c06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,15 @@ COPY worker/ ./worker/ # --ignore-scripts skips workerd's postinstall (a binary-download fallback we # don't need — the binary ships in the @cloudflare/workerd-linux-64 optional dep) # and avoids pnpm's hard error on unapproved dependency build scripts. Full -# (non --prod) install: typescript is a devDependency, needed by `pnpm build` below. +# (non --prod) install: typescript is a devDependency, needed to compile below. +# Compile with tsc directly, not `pnpm run build`: that script also sed's +# tests/*.js (dev-only probe fixtures for test.sh, submitted as Actor input at +# run time — never part of the image), which isn't copied into this build +# context and doesn't need to be; tsconfig's tests/*.ts include glob simply +# matches nothing here. RUN corepack enable \ && pnpm install --frozen-lockfile --ignore-scripts \ - && pnpm run build \ + && pnpm exec tsc -p tsconfig.json \ && BIN="$(node -e "process.stdout.write(require('workerd').default)")" \ && cp "$BIN" /workerd \ && chmod +x /workerd From c9d05b5a5573d43df78bb2b82d1726eeafe30fdb Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 16 Jul 2026 13:40:35 +0200 Subject: [PATCH 28/46] docs: warn against dangling-promise pattern and wrong API shape in code field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent transcripts show scripts wrapping logic in an unawaited 'async function main(){...}; main().catch(...)' pattern truncate silently after the top-level body returns — no error, no partial result, exitCode 0. Also confirmed agents defaulting to the public apify-client SDK's curried .actor(id).call() shape instead of this binding's flat { actorId, input } options-object shape. Both warnings added to the code field's own description since it's always in context (unlike the README, which agents often skip fetching before first use). --- .actor/actor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actor/actor.json b/.actor/actor.json index 466ea41..a78f9ee 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -19,7 +19,7 @@ "code": { "title": "Code", "type": "string", - "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Call apify.actor.get({ actorId }) before running an Actor, to read its schema first. Print a small JSON summary of the result — never dump full datasets.", + "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Call apify.actor.get({ actorId }) before running an Actor, to read its schema first. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape.", "editor": "javascript" } }, From 6c5225627bd59cce28599f6e003c9eb894a62c40 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 16 Jul 2026 16:48:20 +0200 Subject: [PATCH 29/46] feat: opt into fullReadmeOnly, bypass the auto-generated summary This Actor's README documents an exact API contract (apify.* method names/shapes) -- the platform's auto-generated readmeSummary omits that section entirely, confirmed via a live eval trace where the agent fetched the README, got the summary, and guessed the wrong dataset method as a result. Requires the matching apify-mcp-server change (resolveReadmeContent honoring this flag). Unverified: whether this unrecognized actor.json key survives the platform's build validation into the API response -- needs a live check after this Actor is rebuilt. --- .actor/actor.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.actor/actor.json b/.actor/actor.json index a78f9ee..615d94b 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -6,6 +6,7 @@ "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, + "fullReadmeOnly": true, "defaultRunOptions": { "timeoutSecs": 900, "memoryMbytes": 1024 From 77a3d8de97b56d8cc6b2a53709da59c1491047fc Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 16 Jul 2026 17:05:59 +0200 Subject: [PATCH 30/46] docs: lead schema-check guidance with fetch-actor-details, not just in-sandbox check Live eval trace: agent wrote apify.actor.callAndGetItems({ actorId: 'apify/rag-web-browser', input: { url, maxPages } }) from memory -- wrong field names (real one is 'query') -- despite this field's existing 'call apify.actor.get() first' guidance. All 20 calls failed fast with a clear 400, so no time was wasted discovering it, but it still cost a full wasted round trip before the agent self-corrected. The in-sandbox-only phrasing was easy to skip since it costs an extra nested Actor call inside the script. Reworded to lead with the outer fetch-actor-details tool -- the same tool the agent already uses correctly for every other Actor it discovers via search-actors, just not yet for ones it only decides to call while writing code -- and keep apify.actor.get() as the fallback for Actors picked at runtime. --- .actor/actor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actor/actor.json b/.actor/actor.json index 615d94b..6727e94 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -20,7 +20,7 @@ "code": { "title": "Code", "type": "string", - "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Call apify.actor.get({ actorId }) before running an Actor, to read its schema first. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape.", + "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Before writing code that calls a specific Actor, check its real input field names first — via fetch-actor-details (outside this script, before you write it) or apify.actor.get({ actorId }) (inside it, for an Actor picked at runtime). Do not guess field names from memory; a wrong one throws a fast 400, but costs a wasted round trip. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape.", "editor": "javascript" } }, From dae5b1dea17b69e5185dc12dca26393a0d6ae6f2 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 12:43:43 +0200 Subject: [PATCH 31/46] docs: make dataset-ID reuse imperative, clarify non-terminal wait status Council review (Torvalds/Hotz/Pike/Hoare) + live eval traces: 1 of 3 recovery attempts wasted a full re-run of an already-succeeded nested Actor call instead of reusing its logged defaultDatasetId, because the guidance was advisory ("can read those existing storages") not a rule. Reworded to an imperative 'reuse it, do not re-run' in the code field description, README, and docs/API.md. Separately: actor.call/run.waitForFinish's 60s wait cap was already documented, but a trace showed an agent still treating a returned READY/RUNNING status as a hard failure right after a single wait -- the cap and its corollary (non-terminal isn't an error) lived in two different places. Added an inline note directly on the binding-table/method-doc lines themselves, not just the separate polling recipe. --- .actor/actor.json | 2 +- README.md | 14 ++++++++------ docs/API.md | 14 +++++++++++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 6727e94..95202c8 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -20,7 +20,7 @@ "code": { "title": "Code", "type": "string", - "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Before writing code that calls a specific Actor, check its real input field names first — via fetch-actor-details (outside this script, before you write it) or apify.actor.get({ actorId }) (inside it, for an Actor picked at runtime). Do not guess field names from memory; a wrong one throws a fast 400, but costs a wasted round trip. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape.", + "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Before writing code that calls a specific Actor, check its real input field names first — via fetch-actor-details (outside this script, before you write it) or apify.actor.get({ actorId }) (inside it, for an Actor picked at runtime). Do not guess field names from memory; a wrong one throws a fast 400, but costs a wasted round trip. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape. If a prior attempt already logged a nested run's defaultDatasetId/defaultKeyValueStoreId (visible in your own earlier turns), reuse it — do NOT re-run the same Actor call with identical input, that wastes compute on a call that already succeeded. apify.actor.call/run.waitForFinish may return non-terminal (READY/RUNNING) once the 60s wait cap elapses — that is NOT a failure, poll again instead of throwing.", "editor": "javascript" } }, diff --git a/README.md b/README.md index 3c5d7c2..112575c 100644 --- a/README.md +++ b/README.md @@ -54,10 +54,12 @@ call — follow the response's `nextStep` (or call `get-dataset-items`/ - Before running an Actor from your script, call `apify.actor.get({ actorId })` once to read its input/output schema. - As each nested run finishes, log its `run.id` / `defaultDatasetId` / - `defaultKeyValueStoreId` **before** processing its output — if the script - then throws, a re-run can read those existing storages instead of paying to - re-run the Actor (nothing persists between this Actor's own runs, but the - Actors it started keep their results). + `defaultKeyValueStoreId` **before** processing its output (nothing persists + between this Actor's own runs, but the Actors it started keep their + results). **If a prior attempt's `defaultDatasetId`/`defaultKeyValueStoreId` + is visible in your own earlier turns, reuse it — do not re-run the same + Actor call with identical input.** Re-running wastes the compute/cost of a + call that already succeeded. - Print a small, JSON-stringified summary of the result — never dump full datasets. Only what you `console.log`/`console.info` comes back; a top-level `return` value is **not** captured. @@ -206,12 +208,12 @@ apify.store({ search, limit?, category? }) // → actors[] // Actors apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run -apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) +apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits; may return non-terminal READY/RUNNING if the 60s cap elapses first — not an error, poll run.waitForFinish) apify.actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } // Runs apify.run.get({ runId }) // → run -apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run +apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run (same non-terminal caveat as actor.call above) apify.run.abort({ runId }) // → run apify.run.getLog({ runId, limit? }) // → string diff --git a/docs/API.md b/docs/API.md index 10ad7ae..58bec56 100644 --- a/docs/API.md +++ b/docs/API.md @@ -25,6 +25,10 @@ document describes every method in detail. [`keyValueStore.get`](#keyvaluestoreget--value--null), which returns `null` for a missing key. - **Network.** Outbound `fetch` from your script is restricted to `apify.com` and its subdomains. +- **Reuse, don't re-run.** If a prior attempt already logged a nested run's + `defaultDatasetId`/`defaultKeyValueStoreId` (visible in your own earlier + turns), reuse it — do not re-run the same Actor call with identical input. + Re-running wastes the compute/cost of a call that already succeeded. --- @@ -103,6 +107,10 @@ elapses), then return the run record. **Output:** the Run object (unwrapped `data`), exposing `defaultDatasetId` and `defaultKeyValueStoreId` for reading results. Uses the standard run endpoint (not `/run-sync`, which returns the output record instead of the run object). +**May return non-terminal** (`status: 'READY'`/`'RUNNING'`) if `waitForFinishSecs` +(capped at 60s by the API) elapses before the run finishes — this is **not** +an error; poll [`run.waitForFinish`](#runwaitforfinish--run) until `status` is +terminal (`SUCCEEDED`/`FAILED`/`ABORTED`/`TIMED-OUT`). **Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) ### `actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` @@ -164,7 +172,11 @@ first, then return the run record. | `runId` | `string` | yes | | The run ID. | | `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **Capped at 60s by the API**; poll in a loop for longer runs. | -**Output:** the Run object (unwrapped `data`). +**Output:** the Run object (unwrapped `data`). **May return non-terminal** +(`status: 'READY'`/`'RUNNING'`) if the cap elapses first — this is **not** an +error, it means keep polling; check `status` against the terminal set +(`SUCCEEDED`/`FAILED`/`ABORTED`/`TIMED-OUT`) before treating any other value +as a failure. **Apify API:** [`GET /v2/actor-runs/{runId}`](https://docs.apify.com/api/v2/actor-run-get) ### `run.abort({ runId })` → `Run` From 776c58141f29bef998a970102ff018db43859787 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 12:57:26 +0200 Subject: [PATCH 32/46] feat!: dual-mode listItems/store, remove dataset.iterate() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grounded in a fresh clone of apify-client-js (the real Apify TS SDK) and a council review (Torvalds/Hotz/Pike/Hoare) — all four independently converged on the same root cause behind the highest-frequency bug in this project: listItems()/callAndGetItems()/store() each returned a different envelope for 'here are your records' (bare array / bare array / {run,items}), confirmed still recurring 3x even with the full (non-summarized) README available, so more prose wasn't going to fix it. dataset.listItems() and store() now return a value that's both a Promise (await -> one page, { items, count, offset, limit, desc }) and an AsyncIterable (for await -> every item, auto-paginated) -- the same dual nature as apify-client's own PaginatedIterator, implemented via the same mechanism (Object.defineProperty(promise, Symbol.asyncIterator, ...)), but as ONE shared implementation (makePaginatedList) instead of apify-client's own three independent, subtly-different copies of this trick. This makes dataset.iterate() redundant -- removed, along with DatasetIterateOptions and DEFAULT_ITERATE_BATCH. actor.callAndGetItems() and dataset.inferFields() updated for the new page shape (no behavior change -- both only ever consumed a single page). Verified beyond typecheck: a standalone runtime smoke test confirms the dual nature actually works (await resolves to one page; for-await auto-paginates across multiple pages; a call with no limit makes exactly one HTTP request instead of over-fetching). tests/binding-smoke.ts and tests/sandbox-isolation.ts updated for the new shapes. Explicitly NOT adopted from apify-client: its curried client.actor(id).call() shape (this binding's flat options-object convention already fixed a confirmed confusion bug this session and stays), and its own internal inconsistencies (mixed positional/options args, waitForFinish vs waitSecs naming split for the same concept). Breaking change for any script written against the old bare-array listItems()/store() shape -- every script is freshly generated per run (one script per Actor run, no shipped callers), so no migration path is needed. --- README.md | 34 +++++++-- docs/API.md | 94 ++++++++++++++----------- tests/binding-smoke.ts | 19 +++-- tests/sandbox-isolation.ts | 2 +- worker/runner.ts | 138 ++++++++++++++++++++++++++----------- 5 files changed, 193 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 112575c..b878660 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,24 @@ for (let i = 0; i < inputs.length; i += CHUNK) { console.log(JSON.stringify(results.slice(0, 5))); // small summary, not the full dump ``` +### Read an entire dataset without managing offsets + +`dataset.listItems` (and `store`) return a value that's both a `Promise` (one +page) and an `AsyncIterable` (every item, auto-paginated) — pick whichever +you need: + +```js +// One page — e.g. a quick peek +const { items, count } = await apify.dataset.listItems({ datasetId, limit: 10 }); + +// Every item, however many pages that takes +let matches = 0; +for await (const item of apify.dataset.listItems({ datasetId })) { + if (item.rating >= 4.5) matches++; +} +console.log(`${matches} matching items`); +``` + ### Runs longer than 60s: start, then poll `actor.call`'s wait is capped at 60s per request (a REST API limit, not this @@ -197,13 +215,18 @@ context isn't part of the platform's Run schema at all, on or off Code Mode. ## The `apify` binding -Every method takes one options object and returns parsed JSON -(`?` = optional, `= x` = default). Full API documentation is available +Every method takes one options object and returns parsed JSON — except +`store` and `dataset.listItems`, which return a value that's both a `Promise` +(one page) and an `AsyncIterable` (every match/item, auto-paginated) — see +their own lines below (`?` = optional, `= x` = default). Full API +documentation is available [here](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). ```js // Store — GET /v2/store, a top-level Apify API resource (not an Actor method) -apify.store({ search, limit?, category? }) // → actors[] +// `await` for one page, `for await` to walk every match (same dual nature as +// dataset.listItems below). +apify.store({ search, limit?, offset?, category? }) // → { items, count, offset, limit } // Actors apify.actor.get({ actorId }) // → actor @@ -220,8 +243,9 @@ apify.run.getLog({ runId, limit? }) // → string // Datasets apify.dataset.create({ name? }) // → dataset apify.dataset.pushItems({ datasetId, items }) // → void -apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) // → items[] -apify.dataset.iterate({ datasetId, batchSize = 1000, ...filters }) // → async iterable +// `await` for one page: { items, count, offset, limit, desc }. `for await` auto-paginates +// through the whole dataset, one item at a time — no separate iterate() method needed. +apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) apify.dataset.inferFields({ datasetId, sample = 5 }) // → { itemCount, fields[] } // Key-value stores diff --git a/docs/API.md b/docs/API.md index 58bec56..691ea88 100644 --- a/docs/API.md +++ b/docs/API.md @@ -6,10 +6,19 @@ document describes every method in detail. ## Conventions -- **Every method is `async`** — `await` the result (or `for await` for - `dataset.iterate`). +- **Every method is `async`** — except `dataset.listItems`/`store` (see + below), which are plain functions that return an already-awaitable value; + `await`ing one behaves identically to awaiting a true `async` call. - **One options object.** Each method takes a single object argument; there are no positional parameters. +- **Paginated methods are dual-mode.** [`dataset.listItems`](#datasetlistitems--object-count-offset-limit-desc-) + and [`store`](#apifystore--object-count-offset-limit-) return a value that's + both a `Promise` and an `AsyncIterable`: `await` it for one page (`{ items, + count, offset, limit, ... }`); `for await (const item of ...)` it to + auto-paginate through everything, one item at a time. One call, one name, + two ways to consume it — matching the official + [`apify-client`](https://github.com/apify/apify-client-js) SDK's + `PaginatedIterator` convention. There's no separate "iterate" method. - **`actorId`** accepts either `username/name` (e.g. `apify/rag-web-browser`) or the Actor's ID. - **Return values.** The Apify API wraps most responses in a `{ "data": … }` @@ -38,23 +47,31 @@ Apify's own API tags this endpoint `Store` — a top-level resource, not an Actor method — so the binding mirrors that: `apify.store(...)`, not `apify.actor.store(...)`. -### `apify.store({ search, limit?, category? })` → `Actor[]` +### `apify.store({ search, limit?, offset?, category? })` → `{ items, count, offset, limit }` -Search the Apify Store. +Search the Apify Store. Dual-mode — see [Conventions](#conventions). | Param | Type | Required | Description | |---|---|---|---| | `search` | `string` | yes | Full-text search query. | -| `limit` | `number` | no | Maximum number of results. | +| `limit` | `number` | no | Page size (`await` mode) / items per page (`for await` mode). | +| `offset` | `number` | no | Starting offset. | | `category` | `string` | no | Restrict to a Store category. | -**Output:** the `data.items` array of the Store listing (the pagination wrapper -is dropped) — i.e. an `Actor[]`. +**Output (custom):** one page — `{ items: Actor[], count, offset, limit }` +(`count` is this page's actual item count; `offset`/`limit` echo the request). **Apify API:** [`GET /v2/store`](https://docs.apify.com/api/v2/store-get) ```js -const actors = await apify.store({ search: 'web scraper', limit: 5 }); -console.log(actors.map((a) => `${a.username}/${a.name}`).join('\n')); +// One page +const { items } = await apify.store({ search: 'web scraper', limit: 5 }); +console.log(items.map((a) => `${a.username}/${a.name}`).join('\n')); + +// Every match +const names = []; +for await (const actor of apify.store({ search: 'web scraper', limit: 20 })) { + names.push(`${actor.username}/${actor.name}`); +} ``` --- @@ -234,51 +251,48 @@ Append one or more items to a dataset. ```js const ds = await apify.dataset.create(); await apify.dataset.pushItems({ datasetId: ds.id, items: [{ a: 1 }, { a: 2 }] }); -const items = await apify.dataset.listItems({ datasetId: ds.id }); +const { items } = await apify.dataset.listItems({ datasetId: ds.id }); ``` -### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `object[]` +### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `{ items, count, offset, limit, desc }` -Read a page of items. +Read the dataset. Dual-mode — see [Conventions](#conventions): `await` for one +page, `for await` to auto-paginate through everything (replaces what used to +be a separate `iterate()` method — one name, one method, both jobs). | Param | Type | Required | Description | |---|---|---|---| | `datasetId` | `string` | yes | Dataset ID. | | `fields` | `string[]` | no | Only include these fields (joined into `fields`). | | `omit` | `string[]` | no | Exclude these fields. | -| `limit` | `number` | no | Page size. | -| `offset` | `number` | no | Starting offset. | +| `limit` | `number` | no | Page size (`await` mode) / items fetched per page (`for await` mode). Omit for the API's own default (effectively unbounded — a single `await` then returns everything in one page). | +| `offset` | `number` | no | Starting offset. Default `0`. | | `clean` | `boolean` | no | Skip empty items / hidden fields (`clean=1`). | | `desc` | `boolean` | no | Reverse (newest first, `desc=1`). | -**Output (custom):** the **items array directly** — this endpoint already -returns a bare array (no `data`/pagination wrapper). A dataset's pagination -total is eventually consistent right after creation, so no `total` is surfaced; -use [`inferFields`](#datasetinferfields--schema) for a count or -[`iterate`](#datasetiterate--asyncgeneratorobject) to consume everything. -**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) - -### `dataset.iterate({ datasetId, fields?, omit?, clean?, desc?, batchSize? })` → `AsyncGenerator` - -Async-iterate the **entire** dataset, paging internally so you don't manage -offsets. Stops when a page returns fewer than `batchSize` items. - -| Param | Type | Required | Default | Description | -|---|---|---|---|---| -| `datasetId` | `string` | yes | | Dataset ID. | -| `fields` | `string[]` | no | | Only include these fields. | -| `omit` | `string[]` | no | | Exclude these fields. | -| `clean` | `boolean` | no | | Skip empty items / hidden fields. | -| `desc` | `boolean` | no | | Reverse order. | -| `batchSize` | `number` | no | `1000` | Items fetched per page. | - -**Output (custom):** an async generator yielding one item (`object`) at a time. -**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) (paged internally) +**Output (custom):** +- `await apify.dataset.listItems({...})` → one page: `{ items: object[], count, + offset, limit, desc }`. `offset`/`limit`/`desc` echo what the API actually + applied (read from its `x-apify-pagination-*` response headers, not just + the request); `count` is this page's actual item count. **No `total`** — + the API's `x-apify-pagination-total` header is unreliable for + freshly-created datasets (eventually consistent); use + [`inferFields`](#datasetinferfields--schema) for an approximate count. +- `for await (const item of apify.dataset.listItems({...}))` → every item in + the dataset, one at a time, paging internally. Stops when a page comes back + shorter than requested (the natural end-of-data signal). + +**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) (paged internally for `for await`) ```js -let count = 0; -for await (const item of apify.dataset.iterate({ datasetId })) count++; -console.log('total items:', count); +// One page +const { items, count } = await apify.dataset.listItems({ datasetId, limit: 100 }); +console.log(`${count} items on this page`); + +// Every item +let total = 0; +for await (const item of apify.dataset.listItems({ datasetId })) total++; +console.log('total items:', total); ``` ### `dataset.inferFields({ datasetId, sample? })` → `Schema` diff --git a/tests/binding-smoke.ts b/tests/binding-smoke.ts index 76fa9f5..c43ab83 100644 --- a/tests/binding-smoke.ts +++ b/tests/binding-smoke.ts @@ -26,9 +26,14 @@ const ACTOR = 'apify/hello-world'; // ---- actor (read) ---- await check('store', async () => { - const items = await apify.store({ search: 'hello world', limit: 3 }); - if (!Array.isArray(items)) throw new Error('expected array'); - return `${items.length} actors`; + const page = await apify.store({ search: 'hello world', limit: 3 }); + if (!Array.isArray(page.items)) throw new Error('expected items array'); + return `${page.count} actors`; +}); +await check('store (for await)', async () => { + let n = 0; + for await (const _ of apify.store({ search: 'hello world', limit: 3 })) n++; + return `${n} actors iterated`; }); await check('actor.get', async () => { const d = await apify.actor.get({ actorId: ACTOR }); @@ -46,16 +51,16 @@ await check('dataset.pushItems', async () => { return '2 pushed'; }); await check('dataset.listItems', async () => { - const items = await apify.dataset.listItems({ datasetId }); - return `${items.length} items`; + const page = await apify.dataset.listItems({ datasetId }); + return `${page.count} items, offset=${page.offset}, limit=${page.limit}`; }); await check('dataset.inferFields', async () => { const s = await apify.dataset.inferFields({ datasetId }); return `itemCount=${s.itemCount} fields=${s.fields.map((f) => f.name).join(',')}`; }); -await check('dataset.iterate', async () => { +await check('dataset.listItems (for await)', async () => { let n = 0; - for await (const _ of apify.dataset.iterate({ datasetId, batchSize: 1 })) n++; + for await (const _ of apify.dataset.listItems({ datasetId, limit: 1 })) n++; return `${n} iterated`; }); diff --git a/tests/sandbox-isolation.ts b/tests/sandbox-isolation.ts index 335032d..bf877df 100644 --- a/tests/sandbox-isolation.ts +++ b/tests/sandbox-isolation.ts @@ -94,7 +94,7 @@ check('EventSource blocked', blocksConstruct('EventSource', 'https://example.com let bindingWorks = false; try { const found = await apify.store({ search: 'hello world', limit: 1 }); - bindingWorks = Array.isArray(found); + bindingWorks = Array.isArray(found.items); } catch (e) { console.error(`apify.store threw: ${(e as Error).message}`); } diff --git a/worker/runner.ts b/worker/runner.ts index 2a323c9..f90acbf 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -28,7 +28,6 @@ function requireRealFetch(): typeof globalThis.fetch { } const realFetch = requireRealFetch(); -const DEFAULT_ITERATE_BATCH = 1000; const DEFAULT_GET_SCHEMA_SAMPLE = 5; // --- Types --------------------------------------------------------------- @@ -57,6 +56,7 @@ interface ApiCallOptions { interface StoreSearchOptions { search: string; limit?: number; + offset?: number; category?: string; } @@ -101,10 +101,24 @@ interface DatasetListOptions { desc?: boolean; } -interface DatasetIterateOptions extends Omit { - batchSize?: number; +// One page, from a single request. Mirrors apify-client's PaginatedList shape (items, count, +// offset, limit, desc) except `total`, which stays deliberately unsurfaced — see makePaginatedList. +interface ItemsPage { + items: T[]; + count: number; + offset: number; + limit: number; +} + +interface DatasetItemsPage extends ItemsPage { + desc: boolean; } +// Awaiting this value resolves to one page (ItemsPage); `for await`-ing it auto-paginates +// through every item, one at a time. Same dual nature as apify-client's own PaginatedIterator +// (one call, one name, two ways to consume it) — see makePaginatedList for the mechanism. +type PaginatedItems> = Promise & AsyncIterable; + interface DatasetSchemaOptions { datasetId: string; sample?: number; @@ -184,6 +198,44 @@ function errorDetail(err: unknown): string { return err instanceof Error && err.stack ? err.stack : errorMessage(err); } +// Wraps a page-fetcher into a value that's both a Promise (awaits to the first page) and an +// AsyncIterable (walks every page, yielding one item at a time) — the same dual nature as +// apify-client's own PaginatedIterator, implemented via the same trick it uses: attach +// Symbol.asyncIterator to a live Promise object (a Promise is a plain object at runtime, so +// this is legal, no class or wrapper needed). +// +// One shared implementation, unlike apify-client itself, which independently re-implements +// this exact trick three times (dataset items, key-value-store keys, request-queue requests) +// with three subtly different cursor/offset conventions — see docs/API.md's Conventions +// section for that comparison. Every paginated method in this binding goes through this one +// function instead. +// +// Continuation stops the same way the old dataset.iterate() did: a page shorter than the +// limit it was asked for is the natural end-of-data signal (no dataset `total` is trusted — +// see the listItems comment below for why). +function makePaginatedList>( + fetchPage: (offset: number, limit: number | undefined) => Promise, + offset: number, + limit: number | undefined, +): PaginatedItems { + const firstPagePromise = fetchPage(offset, limit); + + async function* iterateAll(): AsyncGenerator { + let page = await firstPagePromise; + yield* page.items; + let nextOffset = page.offset + page.items.length; + while (page.items.length > 0 && page.items.length >= page.limit) { + page = await fetchPage(nextOffset, page.limit); + yield* page.items; + nextOffset += page.items.length; + } + } + + return Object.defineProperty(firstPagePromise, Symbol.asyncIterator, { + value: iterateAll, + }) as PaginatedItems; +} + // 'MCP' matches apify-core's META_ORIGINS.MCP / apify-mcp-server's own // X-Apify-Request-Origin header value — reusing the platform's existing // convention rather than inventing a new one. @@ -290,7 +342,7 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // `actor.call()` — same underlying request, no self-reference to `actor` needed. callAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { const runRecord = await createRun({ actorId, input, waitForFinishSecs: 60, ...runOpts }); - const items = await dataset.listItems({ + const { items } = await dataset.listItems({ datasetId: runRecord.defaultDatasetId as string, fields, limit, }); return { run: runRecord, items }; @@ -328,40 +380,36 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u }; const dataset = { - // Returns the items array directly (no wrapper). The Apify API's + // `await` resolves to one page: { items, count, offset, limit, desc }. `for await` + // auto-paginates through the entire dataset, one item at a time (replaces the old, + // separate dataset.iterate() method — see makePaginatedList). offset/limit/desc echo + // back what the API actually applied (read from its x-apify-pagination-* response + // headers, not just the request), except `total`: the Apify API's // `x-apify-pagination-total` header is unreliable for freshly-created datasets - // (eventually consistent), so we don't surface a `total`. Use `inferFields` if you - // need an item count, or iterate to consume the whole dataset. - listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }: DatasetListOptions): Promise => { - const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { - searchParams: { - fields: fields?.join(','), - omit: omit?.join(','), - limit, - offset, - clean: clean ? '1' : undefined, - desc: desc ? '1' : undefined, - }, - }); - return response.json(); - }, - - // Async generator over the entire dataset. Pages internally in `batchSize` chunks - // so the user can `for await (const item of apify.dataset.iterate({...}))` without - // worrying about offsets. Stops when a page returns fewer items than `batchSize` - // (the natural end-of-data signal — pagination total is not used, see listItems). - iterate: async function* ({ datasetId, fields, omit, clean, desc, batchSize = DEFAULT_ITERATE_BATCH }: DatasetIterateOptions): AsyncGenerator { - let offset = 0; - while (true) { - const items = await dataset.listItems({ - datasetId, fields, omit, clean, desc, - limit: batchSize, offset, + // (eventually consistent), so it's never surfaced — `count` (this page's actual item + // count) is what you want instead. Use `inferFields` if you need an approximate total. + listItems: ({ datasetId, fields, omit, limit, offset = 0, clean, desc }: DatasetListOptions): PaginatedItems => { + const fetchPage = async (pageOffset: number, pageLimit?: number): Promise => { + const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { + searchParams: { + fields: fields?.join(','), + omit: omit?.join(','), + limit: pageLimit, + offset: pageOffset, + clean: clean ? '1' : undefined, + desc: desc ? '1' : undefined, + }, }); - if (items.length === 0) break; - for (const item of items) yield item; - if (items.length < batchSize) break; - offset += items.length; - } + const items: ApifyRecord[] = await response.json(); + return { + items, + count: items.length, + offset: Number(response.headers.get('x-apify-pagination-offset') ?? pageOffset), + limit: Number(response.headers.get('x-apify-pagination-limit') ?? pageLimit ?? items.length), + desc: response.headers.get('x-apify-pagination-desc') === 'true', + }; + }; + return makePaginatedList(fetchPage, offset, limit); }, // Apify has no dedicated schema endpoint; we infer one from a small sample of items. @@ -369,7 +417,7 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // dataset schema (a different concept, described in this Actor's own actor.json). inferFields: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { const meta = await apiData('GET', `/datasets/${encodeURIComponent(datasetId)}`); - const items = await dataset.listItems({ datasetId, limit: sample }); + const { items } = await dataset.listItems({ datasetId, limit: sample }); const fields = new Map>(); for (const item of items) { for (const [name, value] of Object.entries(item ?? {})) { @@ -445,10 +493,18 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // GET /v2/store — Apify Store search (a top-level resource in the Apify API, // tagged `Store`, distinct from `Actors` — hence a top-level binding rather - // than an `actor.*` method). Returns the items array directly. - const store = ({ search, limit, category }: StoreSearchOptions): Promise => - apiData('GET', '/store', { searchParams: { search, limit, category } }) - .then((page: { items: ApifyRecord[] }) => page.items); + // than an `actor.*` method). Same dual nature as dataset.listItems: `await` for one + // page, `for await` to walk every match. offset/limit echo back the request (the + // endpoint's own JSON body doesn't carry pagination metadata beyond `items`, unlike + // dataset's header-based pagination — see makePaginatedList). + const store = ({ search, limit, offset = 0, category }: StoreSearchOptions): PaginatedItems> => { + const fetchPage = async (pageOffset: number, pageLimit: number | undefined): Promise> => { + const page = await apiData('GET', '/store', { searchParams: { search, limit: pageLimit, offset: pageOffset, category } }); + const items: ApifyRecord[] = page.items; + return { items, count: items.length, offset: pageOffset, limit: pageLimit ?? items.length }; + }; + return makePaginatedList(fetchPage, offset, limit); + }; // Freeze every namespace (and the wrapper) so the script can't reassign a method to // corrupt its own behavior or, for `console` below, its own output capture. From 83eb3ae625063ddc30a24d017d482c11ae305505 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 14:55:46 +0200 Subject: [PATCH 33/46] docs: nudge Actor description/README with data-backed usage thresholds A/B eval (run 2, apify-mcp-server evals/workflows) shows code mode's worth-it point isn't item count alone but item count x per-item payload size, given the sandbox's fixed ~20K token round-trip overhead: - 5-item single lookup: +78% slower, +46% more tokens (loses) - 100-record filter/sort/aggregate: -35% faster, -19% fewer tokens (wins) - 20-item fan-out over per-item web pages: -59% faster, -75% fewer tokens (wins decisively even at low item count) Encode this as concrete thresholds in both the actor.json description (agent-facing at discovery time) and README (agent-facing after fetch-actor-details). --- .actor/actor.json | 2 +- README.md | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 95202c8..e715d0d 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -2,7 +2,7 @@ "actorSpecification": 1, "name": "code-runtime", "title": "Code Runtime", - "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Best for data-heavy jobs: scrape hundreds+ places/items, chain Actor outputs, and filter/sort/aggregate in one billed run. Results land in the dataset. Read the README before use.", + "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Worth it only for bulk work: filtering/sorting/aggregating 50+ dataset records, or fanning out over 10+ sub-resources (e.g. visiting many pages) per run. Skip it for a single small (<10 item) lookup \u2014 a direct Actor call is faster and cheaper there. Results land in the dataset. Read the README before use.", "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, diff --git a/README.md b/README.md index b878660..d6499b1 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,24 @@ search the Store, run an Actor, read its dataset, filter and aggregate the results — instead of sending every intermediate result back through the model and wasting tokens. This Actor is the sandbox that runs that script. -**Best suited for data-heavy jobs** — scraping hundreds or thousands of -places/items via an Actor, then filtering, sorting, or aggregating them -locally before returning a small summary. **Weaker fit** for steps that -require reading or judging free text (picking a fact out of an article, -choosing a search term) — keep the model in the loop there instead; a wrong -guess inside the sandbox fails silently until the whole script finishes. +**Worth it only for bulk work**, measured empirically (A/B eval, tokens/duration +vs. calling Actor tools directly): + +- **Filtering/sorting/aggregating ~50+ dataset records** in one dataset — + modest win (~20-35% less time, ~20% fewer tokens at 100 records). +- **Fanning out over ~10+ sub-resources with a sizeable payload each** + (e.g. visiting many pages, chaining Actor outputs) — decisive win even at + small counts, since every skipped round trip avoids funneling a whole raw + page/document through the model (~60% less time, ~75% fewer tokens at 20 + places × 20 page fetches). +- **Below ~10 items with no fan-out** (a single small lookup) — **don't use + this Actor**. The sandbox's own round-trip overhead (~20K tokens) isn't + paid back; a direct Actor call is both faster and cheaper. + +**Weaker fit** for steps that require reading or judging free text (picking a +fact out of an article, choosing a search term) — keep the model in the loop +there instead; a wrong guess inside the sandbox fails silently until the +whole script finishes. ## Calling this Actor From 73468849ec91dbcf3d65b951330d93f085ef660a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 14:57:03 +0200 Subject: [PATCH 34/46] fix: trim actor.json description to fit under 300 chars --- .actor/actor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actor/actor.json b/.actor/actor.json index e715d0d..cf3d551 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -2,7 +2,7 @@ "actorSpecification": 1, "name": "code-runtime", "title": "Code Runtime", - "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Worth it only for bulk work: filtering/sorting/aggregating 50+ dataset records, or fanning out over 10+ sub-resources (e.g. visiting many pages) per run. Skip it for a single small (<10 item) lookup \u2014 a direct Actor call is faster and cheaper there. Results land in the dataset. Read the README before use.", + "description": "Runs one JS script in a sandboxed Actor with an apify binding (Actors, datasets, KV stores). Worth it for bulk work: 50+ record filter/sort/aggregate, or 10+ item fan-out (e.g. many page visits). Skip for a single <10-item lookup \u2014 call the Actor directly instead. Read the README first.", "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, From a767a0eae1520c0dbcf5e88e9c2f6b98a749199e Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 15:05:52 +0200 Subject: [PATCH 35/46] docs: cut internal-telemetry section, dedupe repeated facts, tighten prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Council review (Pike, Davis, Hotz) flagged the README as over-explained: same facts stated 2-4x across sections with circular cross-references. - Remove 'Limitations' section (meta.origin/meta.actorRunId telemetry attribution) entirely — internal plumbing irrelevant to a caller deciding how to use this Actor. - Merge 'Limits & failure modes' into Output — both described the same exitCode/statusMessage vs run-status distinction, cross-referencing each other in a circle. - 'No imports' facts now live once in Permissions & safety; How it works just points there instead of restating. - Split the nested-run bullet (was one sentence doing four jobs) into two. - 'Worth it only for bulk work' bullets -> table. - Dual Promise/AsyncIterable explanation for listItems/store stated once (apify binding section); Recipes and inline comments now just reference it. - Fixed a real inconsistency: Output's terminal-status list was missing ABORTED (present in the Recipes poll-loop code right below it). 273 -> 221 lines, same information. --- README.md | 204 ++++++++++++++++++++---------------------------------- 1 file changed, 76 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index d6499b1..a764b23 100644 --- a/README.md +++ b/README.md @@ -6,33 +6,24 @@ ## What it does -This Actor executes JavaScript that an AI agent submits through the Apify MCP +Executes one JS script that an AI agent submits through the Apify MCP Server's **Code Mode**, then returns whatever the script printed. JS only — -nothing transpiles it, so a TypeScript type annotation is a SyntaxError at load. - -Code Mode exists so an agent can do many Apify operations in **one go** — -search the Store, run an Actor, read its dataset, filter and aggregate the -results — instead of sending every intermediate result back through the model -and wasting tokens. This Actor is the sandbox that runs that script. - -**Worth it only for bulk work**, measured empirically (A/B eval, tokens/duration -vs. calling Actor tools directly): - -- **Filtering/sorting/aggregating ~50+ dataset records** in one dataset — - modest win (~20-35% less time, ~20% fewer tokens at 100 records). -- **Fanning out over ~10+ sub-resources with a sizeable payload each** - (e.g. visiting many pages, chaining Actor outputs) — decisive win even at - small counts, since every skipped round trip avoids funneling a whole raw - page/document through the model (~60% less time, ~75% fewer tokens at 20 - places × 20 page fetches). -- **Below ~10 items with no fan-out** (a single small lookup) — **don't use - this Actor**. The sandbox's own round-trip overhead (~20K tokens) isn't - paid back; a direct Actor call is both faster and cheaper. - -**Weaker fit** for steps that require reading or judging free text (picking a -fact out of an article, choosing a search term) — keep the model in the loop -there instead; a wrong guess inside the sandbox fails silently until the -whole script finishes. +a TypeScript annotation is a SyntaxError at load, nothing transpiles it. + +Code Mode lets an agent do many Apify operations in **one call** — search +the Store, run an Actor, read its dataset, filter and aggregate — instead of +sending every intermediate result back through the model. This Actor is the +sandbox that runs that script. + +**Worth it only for bulk work** (measured, A/B eval vs. calling Actor tools +directly): + +| Workload | Verdict | +|---|---| +| Filter/sort/aggregate 50+ dataset records | Modest win — ~20-35% less time, ~20% fewer tokens | +| Fan out over 10+ sub-resources with a sizeable payload each (visit many pages, chain Actors) | Decisive win — ~60% less time, ~75% fewer tokens | +| Under 10 items, no fan-out | **Don't use this Actor** — ~20K-token sandbox overhead isn't paid back | +| Reading/judging free text (pick a fact, choose a search term) | Keep the model in the loop — a wrong guess fails silently until the script ends | ## Calling this Actor @@ -49,32 +40,30 @@ as the body. Results land in the run's default dataset, same as any Actor call — follow the response's `nextStep` (or call `get-dataset-items`/ `GET /v2/datasets/{datasetId}/items`) to read it. +Default `timeoutSecs: 900`, `memoryMbytes: 1024` (`.actor/actor.json`) — +override per call for scripts chaining several long Actor runs (MCP +`call-actor`'s `callOptions.timeout`/`callOptions.memory`, or the API's +`timeout`/`memory`). + ## How it works -- **One script per run.** The Actor reads your `code`, runs it once, writes the - result, and exits. -- The code runs inside a [`workerd`](https://github.com/cloudflare/workerd) V8 - isolate: **no imports** — neither npm packages nor Node built-in `node:*` - modules are available (`import`/`require` of any module fails); web-standard - globals such as `fetch` are present. Outbound network is restricted to - `*.apify.com`. -- Inside the script a global **`apify`** object exposes a small, typed subset of - the Apify API — run Actors, read/write datasets and key-value stores — using - the current run's token (see below). -- `console.log` / `console.info` go to **stdout**; `console.error` / - `console.warn` go to **stderr**. The two streams are captured separately. -- Before running an Actor from your script, call `apify.actor.get({ actorId })` - once to read its input/output schema. -- As each nested run finishes, log its `run.id` / `defaultDatasetId` / - `defaultKeyValueStoreId` **before** processing its output (nothing persists - between this Actor's own runs, but the Actors it started keep their - results). **If a prior attempt's `defaultDatasetId`/`defaultKeyValueStoreId` - is visible in your own earlier turns, reuse it — do not re-run the same - Actor call with identical input.** Re-running wastes the compute/cost of a - call that already succeeded. -- Print a small, JSON-stringified summary of the result — never dump full - datasets. Only what you `console.log`/`console.info` comes back; a - top-level `return` value is **not** captured. +- **One script per run.** Reads `code`, runs it once, writes the result, exits. +- Runs inside a sandboxed [`workerd`](https://github.com/cloudflare/workerd) + V8 isolate — see [Permissions & safety](#permissions--safety) for what's allowed. +- A global **`apify`** object exposes a small, typed subset of the Apify API + — run Actors, read/write datasets and key-value stores — using the + current run's token. +- `console.log`/`console.info` → **stdout**; `console.error`/`console.warn` + → **stderr**, captured separately. +- Call `apify.actor.get({ actorId })` before running an Actor you haven't + checked — don't guess its input schema. +- Log a nested run's `run.id`/`defaultDatasetId`/`defaultKeyValueStoreId` + before processing its output — nothing persists between this Actor's own + runs, but the Actors it started keep theirs. +- Already have a dataset/store ID from an earlier turn? Reuse it — don't + re-run an identical call, it wastes cost. +- Print a small JSON summary, never a full dataset — only `console.log`/ + `console.info` output comes back; a top-level `return` is **not** captured. ## Input @@ -90,56 +79,37 @@ call — follow the response's `nextStep` (or call `get-dataset-items`/ ## Output -A single **dataset item** with the captured streams, the script's exit -status, and a prose status message: +A single **dataset item**: ```json { "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "", "exitCode": 0, "statusMessage": "Script completed" } ``` -If the script throws, the error lands in `stderr`, `stdout` keeps whatever was -printed before the failure, `exitCode` is `1`, and `statusMessage` is -`"Script threw: ..."`. The Actor run itself still **succeeds** — check -`exitCode`/`statusMessage`, not `stderr` content, to detect a failed script, -since `stderr` is also a legitimate log channel (`console.error` / -`console.warn`). - -If the script fails to **compile** (a syntax error), the same contract -applies — `exitCode: 1`, `statusMessage: "Failed to compile: ..."` — pushed -by the container entrypoint directly, since a malformed script never reaches -the sandboxed worker at all. - -A run-level **timeout or out-of-memory kill** is a different, third outcome: -the container is killed before it can push anything, so this dataset item may -not exist for that run at all. That case is signaled by the Actor run's own -status (`SUCCEEDED` vs `FAILED`/`TIMED-OUT`), not by this item's absence — -see [Limits & failure modes](#limits--failure-modes). - -## Limits & failure modes - -- Default `defaultRunOptions`: `timeoutSecs: 900`, `memoryMbytes: 1024` - (`.actor/actor.json`). Override per call — e.g. the MCP `call-actor` tool's - `callOptions.timeout`/`callOptions.memory`, or the API's `timeout`/`memory` - run options — for scripts that chain several long-running Actor calls. -- `exitCode`/`statusMessage` signal the **script's** outcome only (returned / - threw / failed to compile). A resource-limit kill is a **run-level** - outcome instead — check the Actor run's own `status`, not this dataset - item, for that case (see [Output](#output) above). +| Outcome | `exitCode` | `statusMessage` | +|---|---|---| +| Script returned | `0` | `Script completed` | +| Script threw | `1` | `Script threw: ...` | +| Failed to compile (syntax error) | `1` | `Failed to compile: ...` | +| Run-level timeout / OOM kill | — | item may not exist for this run at all | + +Check `exitCode`/`statusMessage`, not `stderr` content, to detect a failed +script — `stderr` also carries `console.error`/`console.warn` output, so its +presence alone isn't failure. A timeout/OOM kill is signaled by the Actor +run's own status (`SUCCEEDED` vs `FAILED`/`ABORTED`/`TIMED-OUT`), not by this +item's absence. ## Permissions & safety -- Runs with **limited permissions**: the sandbox has no filesystem and - outbound `fetch` (including through redirects, which are re-validated per - hop) is limited to the Apify API (`*.apify.com`). -- **No imports.** The isolate runs without workerd's `nodejs_compat`, so user - code cannot import Node built-ins (`node:net`, `node:fs`, …) or npm packages. - This removes `node:net` — a raw-socket egress path that would otherwise bypass - the `fetch` allowlist — and keeps the run token out of `process.env` (which is - not defined). +- Sandbox has **no filesystem**; outbound `fetch` (redirects re-validated + per hop) is limited to the Apify API (`*.apify.com`). +- **No imports** — runs without workerd's `nodejs_compat`, so no Node + built-ins (`node:net`, `node:fs`, …) or npm packages. This also removes + `node:net` (a raw-socket path that would bypass the `fetch` allowlist) and + keeps the run token out of `process.env` (undefined here). - Each run is an isolated, single-use container — nothing persists between runs. -- This closes off **direct fetch-based exfil** from the container — it does not - stop every path to move data out (e.g. `actor.start({ input })` on an Actor - with its own open internet access, or writing to a dataset/key-value store). +- This closes **direct fetch-based exfil** — it does not close every path to + move data out (e.g. `actor.start({ input })` on an Actor with its own + internet access, or writing to a dataset/key-value store). ## Recipes @@ -158,10 +128,10 @@ console.log(JSON.stringify(pages.slice(0, 3).map((p) => p.url))); ### Bounded parallel fan-out -This Actor's clearest win: run several independent Actors (or the same Actor -over several inputs) concurrently, then reduce before returning. Chunk the -fan-out (e.g. 5–10 at a time) — an unbounded `Promise.all` over many inputs -can hit your account's concurrent-run or memory limits. +This Actor's clearest win: run several Actors (or the same Actor over +several inputs) concurrently, then reduce before returning. Chunk it (5–10 +at a time) — an unbounded `Promise.all` can hit your account's +concurrent-run or memory limits. ```js const inputs = [{ query: 'a' }, { query: 'b' }, { query: 'c' } /* ... */]; @@ -179,9 +149,8 @@ console.log(JSON.stringify(results.slice(0, 5))); // small summary, not the full ### Read an entire dataset without managing offsets -`dataset.listItems` (and `store`) return a value that's both a `Promise` (one -page) and an `AsyncIterable` (every item, auto-paginated) — pick whichever -you need: +`dataset.listItems`/`store` work two ways — `await` for one page, `for +await` to auto-paginate every item (see [the apify binding](#the-apify-binding)): ```js // One page — e.g. a quick peek @@ -208,56 +177,35 @@ while (!TERMINAL.includes(run.status)) { } ``` -## Limitations - -Sub-runs started from inside the sandbox (via `apify.actor.start/call/callAndGetItems`) -get `meta.origin: 'MCP'` on the Apify platform, same as this Actor's own run, when -this Actor was itself started via the Apify MCP Server — this Actor reads its own -`APIFY_META_ORIGIN` env var (platform-set, not spoofable from inside the sandbox) -and forwards `X-Apify-Request-Origin: MCP` on its own API calls only when that's -`MCP`. Every sub-run also gets a platform-native `meta.actorRunId` link back to -this run regardless of origin (set automatically from the run-scoped token, no -code needed here) — so even a fully generic query can already walk sub-run → -`meta.actorRunId` → parent `meta.origin` to reconstruct the chain; the origin -forwarding above just makes single-field origin queries work without that join. - -What's still not attributed: the specific MCP *session* (which client, which -conversation) that triggered this Actor's own run in the first place — that -context isn't part of the platform's Run schema at all, on or off Code Mode. - ## The `apify` binding Every method takes one options object and returns parsed JSON — except -`store` and `dataset.listItems`, which return a value that's both a `Promise` -(one page) and an `AsyncIterable` (every match/item, auto-paginated) — see -their own lines below (`?` = optional, `= x` = default). Full API -documentation is available -[here](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). +`store` and `dataset.listItems`, which return a value that's both a +`Promise` (one page) and an `AsyncIterable` (every match/item, +auto-paginated). Full API docs: +[API.md](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). +(`?` = optional, `= x` = default) ```js // Store — GET /v2/store, a top-level Apify API resource (not an Actor method) -// `await` for one page, `for await` to walk every match (same dual nature as -// dataset.listItems below). -apify.store({ search, limit?, offset?, category? }) // → { items, count, offset, limit } +apify.store({ search, limit?, offset?, category? }) // → { items, count, offset, limit }; dual Promise/AsyncIterable, see above // Actors apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run -apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits; may return non-terminal READY/RUNNING if the 60s cap elapses first — not an error, poll run.waitForFinish) +apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (may be non-terminal READY/RUNNING past the 60s cap — not an error, see Recipes) apify.actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } // Runs apify.run.get({ runId }) // → run -apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run (same non-terminal caveat as actor.call above) +apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run (same non-terminal caveat) apify.run.abort({ runId }) // → run apify.run.getLog({ runId, limit? }) // → string // Datasets apify.dataset.create({ name? }) // → dataset apify.dataset.pushItems({ datasetId, items }) // → void -// `await` for one page: { items, count, offset, limit, desc }. `for await` auto-paginates -// through the whole dataset, one item at a time — no separate iterate() method needed. -apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) +apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) // → { items, count, offset, limit, desc }; dual Promise/AsyncIterable, see above apify.dataset.inferFields({ datasetId, sample = 5 }) // → { itemCount, fields[] } // Key-value stores From 2302300db5fb72fcb6490da23e18de544803bc73 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 15:14:41 +0200 Subject: [PATCH 36/46] docs: trim Code Mode/API-plumbing mentions from README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - What it does: drop 'Code Mode' naming and the TypeScript-SyntaxError aside (JS-only is already stated in the Input field description). - Calling this Actor: drop the raw-API POST/nextStep alternative — the call-actor example is the one path that matters here. - Learn more: drop the Code Mode design PR link, keep just the MCP Server link. - Also dropped the earlier free-text-judgment table row per feedback. --- README.md | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a764b23..c8e2886 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,12 @@ ## What it does Executes one JS script that an AI agent submits through the Apify MCP -Server's **Code Mode**, then returns whatever the script printed. JS only — -a TypeScript annotation is a SyntaxError at load, nothing transpiles it. +Server, then returns whatever the script printed. -Code Mode lets an agent do many Apify operations in **one call** — search -the Store, run an Actor, read its dataset, filter and aggregate — instead of -sending every intermediate result back through the model. This Actor is the -sandbox that runs that script. +Lets an agent do many Apify operations in **one call** — search the Store, +run an Actor, read its dataset, filter and aggregate — instead of sending +every intermediate result back through the model. This Actor is the sandbox +that runs that script. **Worth it only for bulk work** (measured, A/B eval vs. calling Actor tools directly): @@ -23,7 +22,6 @@ directly): | Filter/sort/aggregate 50+ dataset records | Modest win — ~20-35% less time, ~20% fewer tokens | | Fan out over 10+ sub-resources with a sizeable payload each (visit many pages, chain Actors) | Decisive win — ~60% less time, ~75% fewer tokens | | Under 10 items, no fan-out | **Don't use this Actor** — ~20K-token sandbox overhead isn't paid back | -| Reading/judging free text (pick a fact, choose a search term) | Keep the model in the loop — a wrong guess fails silently until the script ends | ## Calling this Actor @@ -35,11 +33,6 @@ default tools: call-actor({ actor: "apify/code-runtime", input: { code: "..." } }) ``` -Or via the raw API: `POST /v2/acts/apify~code-runtime/runs` with `{ code }` -as the body. Results land in the run's default dataset, same as any Actor -call — follow the response's `nextStep` (or call `get-dataset-items`/ -`GET /v2/datasets/{datasetId}/items`) to read it. - Default `timeoutSecs: 900`, `memoryMbytes: 1024` (`.actor/actor.json`) — override per call for scripts chaining several long Actor runs (MCP `call-actor`'s `callOptions.timeout`/`callOptions.memory`, or the API's @@ -218,4 +211,3 @@ apify.keyValueStore.list({ storeId, limit?, exclusiveStartKey? }) // → { item ## Learn more - Apify MCP Server: -- Code Mode design: [apify/apify-mcp-server#794](https://github.com/apify/apify-mcp-server/pull/794) From 2812c9ca3695a855f3e94e533caf4509f063bde7 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 16:10:21 +0200 Subject: [PATCH 37/46] chore: remove fullReadmeOnly actor.json flag MCP server now hardcodes this Actor's ID to always return the full README, instead of trusting a self-declared actor.json flag any Actor could set to opt itself out of the auto-generated summary. --- .actor/actor.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.actor/actor.json b/.actor/actor.json index cf3d551..a17a10b 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -6,7 +6,6 @@ "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, - "fullReadmeOnly": true, "defaultRunOptions": { "timeoutSecs": 900, "memoryMbytes": 1024 From 8930b95acc4aa46299c7da56309ccb2e224a2a8d Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 22 Jul 2026 09:57:06 +0200 Subject: [PATCH 38/46] fix: gate realFetch claim on request handling, not import order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #1 review comment (2026-07-21, issuecomment-5037390847) found the claimRealFetch() one-shot handoff added in 5461aee didn't guarantee runner.ts claims first. entrypoint.sh splices `code` verbatim (no escaping) into `export async function run(apify, console) { }`; a bare `}` in `code` closes that function early, and everything after runs as ordinary MODULE-SCOPE code in usercode.js. ES module evaluation order puts that ahead of runner.ts's own top-level code (usercode.js is the import evaluated immediately before runner.ts's own body runs), so injected top-level code using `(await import('./guard.js')).claimRealFetch()` reliably claimed the unrestricted fetch first, making runner.ts's own claim get null and throw -- crashing the whole Actor run. Verified live against the deployed Actor before this fix: workerd exits with 'Uncaught Error: realFetch already claimed — guard.js imported out of order', run FAILS. Root cause isn't import order, it's that claimRealFetch() was claimable at all during module evaluation, before any request. Fix: gate it on genuine request handling instead. guard.ts adds requestHandlingStarted (false until markRequestHandlingStarted() is called) and claimRealFetch() refuses (returns null without consuming the resource) until that flag is set. runner.ts calls markRequestHandlingStarted() + claims realFetch as the first, synchronous statements of the /run path in its fetch handler -- before any `await`, so no attacker-scheduled microtask from usercode.js's module scope can race it. Module evaluation (guard.js, usercode.js, runner.ts's own top level) always completes before workerd dispatches the first request, so this holds regardless of any module-graph ordering, unlike the previous design. realFetch itself moved from a module-top-level const (claimed at runner.ts's own top-level, which is what lost the race) to a module-level definite-assigned `let`, assigned inside the /run handler before any of the functions that close over it run. Also fixes the stale comments at guard.ts (claimRealFetch) and runner.ts (top-of-file + requireRealFetch) asserting import order made runner.ts claim first -- disproved by the live PoC above. Adds tests/fixtures/realfetch-escape.js: a regression probe using the exact escape shape from the review comment (real top-level await, not an async IIFE -- top-level await is what forces usercode.js's evaluation to fully settle before runner.ts's own top-level code runs; an IIFE's internal await doesn't carry that guarantee). Wired into test.sh as a new live check asserting `apify call`'s own exit status (this probe's run() body is an empty no-op post-escape, so it has no captured console to report a sentinel through, unlike the existing binding-smoke/sandbox-isolation probes). Verified against the real deployed Actor, before/after, same probe: - before (reverted worker/*.ts, same probe file): Actor run FAILS, workerd crash log matches the predicted 'guard.js imported out of order' message. - after (this commit): Actor run SUCCEEDS, exitCode 0, statusMessage 'Script completed' -- same as running an empty script. Full suite (binding-smoke, sandbox-isolation, this new regression probe) run live via ./test.sh: 19/19, 17/17, and the regression probe all pass. --- test.sh | 18 ++++++++++++ tests/fixtures/realfetch-escape.js | 43 +++++++++++++++++++++++++++++ worker/guard.ts | 44 ++++++++++++++++++++++++------ worker/runner.ts | 38 ++++++++++++++++---------- 4 files changed, 120 insertions(+), 23 deletions(-) create mode 100644 tests/fixtures/realfetch-escape.js diff --git a/test.sh b/test.sh index 2184a65..650b042 100755 --- a/test.sh +++ b/test.sh @@ -40,3 +40,21 @@ done [ "$failed" -eq 0 ] || { echo "==> some probes FAILED" >&2; exit 1; } echo "==> all probes passed" + +# Regression probe for the realFetch claim-ordering bug (PR #1 review, +# 2026-07-21): tests/fixtures/realfetch-escape.js escapes usercode.js's +# wrapper into module scope and tries to steal the internal-only realFetch +# before runner.ts's own claim. It has no captured console to report through +# (see the file's own comment), so success/failure is the Actor run itself +# succeeding vs. failing -- not a printed sentinel like the probes above. +echo "==> apify call: tests/fixtures/realfetch-escape.js (regression: realFetch claim ordering)" +jq -n --arg code "$(cat tests/fixtures/realfetch-escape.js)" '{ code: $code }' > "$input_json" +if apify call -f "$input_json" -o; then + echo "==> realfetch-escape passed (run succeeded — module-scope steal attempt did not hijack/crash the internal claim)" +else + echo "==> realfetch-escape FAILED (run crashed — realFetch claim-ordering regression, see guard.ts's requestHandlingStarted gate)" >&2 + failed=1 +fi + +[ "$failed" -eq 0 ] || { echo "==> some probes FAILED" >&2; exit 1; } +echo "==> all probes (including regressions) passed" diff --git a/tests/fixtures/realfetch-escape.js b/tests/fixtures/realfetch-escape.js new file mode 100644 index 0000000..5d591f5 --- /dev/null +++ b/tests/fixtures/realfetch-escape.js @@ -0,0 +1,43 @@ +// Regression probe for the realFetch claim-ordering bug found in PR #1's +// review (2026-07-21): +// https://github.com/apify/actor-code-runtime/pull/1#issuecomment-5037390847 +// +// entrypoint.sh splices `code` verbatim (no escaping) into +// `export async function run(apify, console) { }`. The bare `}` right +// after this comment block closes that function early -- deliberately, to +// reproduce the escape -- so `run` becomes a harmless no-op (nothing is left +// in its body once the comments end) and everything after runs as ordinary +// MODULE-SCOPE code in usercode.js: workerd evaluates that unconditionally, +// before this worker ever calls runner.ts's request handler. That used to be +// enough to `await import('./guard.js')` and call claimRealFetch() directly, +// stealing the unrestricted, un-allowlisted fetch before runner.ts's own +// claim (previously made at runner.ts's own module top level) ever ran -- +// which made THAT claim get null, throw, and crash the whole Actor run +// (self-DoS; the real exploit payoff for an attacker would be using the +// stolen fetch directly, not reported here). +// +// Not valid JS on its own (it opens with an unbalanced `}`) -- intentionally, +// same shape as the reported PoC. Not TypeScript, not compiled, not +// typechecked (lives under tests/fixtures/, outside tsconfig's `include` and +// outside the `tests/*.js` build-artifact glob): see test.sh, which pushes +// this file's raw content directly as the `code` input. +// +// Expected result with the fix (guard.ts's requestHandlingStarted gate): +// claimRealFetch() called from module scope returns null without consuming +// the resource, so nothing crashes -- this Actor run completes normally +// (exitCode 0, "Script completed"), same as any run of an empty script. +// Before the fix: this run FAILS outright (workerd crashes during module +// evaluation; entrypoint.sh can't tell that apart from a real infra failure +// and fails the whole Actor run). test.sh asserts on `apify call`'s own exit +// status, not a printed sentinel -- this probe's `run` body never executes +// any of its own code, so it has no captured console to report through. +// +// Must be genuine top-level await, not an async IIFE: a module containing +// top-level await defers the evaluation of modules that depend on it (here, +// runner.ts) until that await settles (see MDN/TC39 "Top-level await", +// Asynchronous Module Evaluation) -- an IIFE's internal await does not carry +// that guarantee, so it would not reliably win the race this probe exists to +// exercise. +} +globalThis.__stolenRealFetch = (await import('./guard.js')).claimRealFetch(); +;{ diff --git a/worker/guard.ts b/worker/guard.ts index 17840ef..46c62ab 100644 --- a/worker/guard.ts +++ b/worker/guard.ts @@ -3,7 +3,7 @@ // runs at module-evaluation time. Our own Apify API calls use the real, // unrestricted fetch (the internal API is a private IP, not *.apify.com) — // see claimRealFetch() below for how runner.js gets it without leaving it -// reachable from user code. +// usable by user code. // // Egress surface (workerd, no nodejs_compat): the only JS-reachable outbound // primitives are fetch, WebSocket, and EventSource. Raw sockets (node:net, @@ -15,16 +15,42 @@ const realFetch = globalThis.fetch.bind(globalThis); -// One-shot handoff of the unrestricted fetch to runner.js. ES modules are -// evaluated once and cached, so `guard.js` is the same module instance no -// matter who imports it. runner.js imports this module (and calls -// claimRealFetch()) before usercode.js is ever imported, so it always claims -// first. If the sandboxed script later does `await import('./guard.js')` to -// try to recover the unrestricted fetch, it gets this same cached instance — -// but the value is already gone. A standing `export { realFetch }` would hand -// it to that later import too; don't reintroduce one. +// One-shot handoff of the unrestricted fetch to runner.js — gated on genuine +// request handling, not on import order between guard.js and usercode.js. +// +// entrypoint.sh splices the `code` input verbatim (no escaping) into +// `export async function run(apify, console) { }`. A bare `}` in +// `code` closes that function early; everything after it runs as ordinary +// MODULE-SCOPE code in usercode.js, executed during module evaluation — +// i.e. unconditionally, before workerd ever calls this worker's own +// `fetch(request, env)` handler. That escaped code can `await +// import('./guard.js')` and call claimRealFetch() itself. +// +// This module previously assumed import order made runner.js's own claim run +// first (guard.js has no dependency on usercode.js, so it evaluates before +// it) — that protects nothing here, because it's usercode.js's top-level code +// racing runner.ts's top-level code, and usercode.js is the import evaluated +// immediately before runner.ts's own body runs. Verified live (PR #1 review, +// 2026-07-21): a `code` input using the escape above reliably wins that race, +// so runner.ts's own claim got null and threw, crashing the whole Actor run. +// +// The actual invariant that holds regardless of any module-graph ordering: +// usercode.js's module scope never runs *inside* a `fetch(request, env)` +// call — module evaluation always finishes before workerd dispatches the +// first request. So claimRealFetch() refuses to hand anything out until +// markRequestHandlingStarted() has been called; runner.ts calls it (and +// claims) as the first, synchronous statement of its `/run` handling, before +// any `await`, so no attacker-scheduled microtask can race it either. A claim +// attempted before that (legitimate or injected) gets null without consuming +// the resource, so the real claim still succeeds afterward. +let requestHandlingStarted = false; +export function markRequestHandlingStarted(): void { + requestHandlingStarted = true; +} + let unclaimedRealFetch: typeof realFetch | null = realFetch; export function claimRealFetch(): typeof realFetch | null { + if (!requestHandlingStarted) return null; // too early to be a trusted caller const fetchFn = unclaimedRealFetch; unclaimedRealFetch = null; return fetchFn; diff --git a/worker/runner.ts b/worker/runner.ts index f90acbf..ad3d1ec 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -7,26 +7,31 @@ // Single-tenant: one run = one container = one program = one token. No Worker // Loader / per-request isolate is needed — the program runs in this worker, // which is itself the sandbox (no filesystem, restricted outbound network). -// guard.js must be imported before usercode.js: it overrides globalThis.fetch -// to allow only apify.com, and hands us the real, unrestricted fetch via a -// one-shot claimRealFetch() for our own (internal) API calls — see guard.js -// for why this is a claim, not a standing export. -import { claimRealFetch } from './guard.js'; +// guard.js overrides globalThis.fetch to allow only apify.com, and hands us +// the real, unrestricted fetch via a one-shot claimRealFetch() for our own +// (internal) API calls. That claim is gated on markRequestHandlingStarted() +// (called below, first thing in the `fetch` handler's `/run` path), not on +// import order — usercode.js's module scope can run attacker-controlled code +// before this module's own top-level code does, so the claim must not +// happen at this module's top level either. See guard.ts's comment on +// claimRealFetch for the full reasoning. +import { claimRealFetch, markRequestHandlingStarted } from './guard.js'; import { run } from './usercode.js'; -// Must run before usercode.js's `run()` is ever invoked (it does, here — module -// evaluation order puts this ahead of any dynamic import from inside `run()`). -// Factored into a function (rather than a bare `const` + `if (!x) throw`) so the -// non-null guarantee is encoded in the return type once, here — TS doesn't carry -// a narrowed-from-null check across the later function declarations that close -// over `realFetch`, but a return type with the `null` branch already thrown away -// needs no further narrowing anywhere downstream. +// Called as the first, synchronous statement of the `/run` path in the +// `fetch` handler below — before any `await`, so no attacker-scheduled +// microtask from usercode.js's module scope can call claimRealFetch() in +// between. See guard.ts. function requireRealFetch(): typeof globalThis.fetch { const fetchFn = claimRealFetch(); - if (!fetchFn) throw new Error('realFetch already claimed — guard.js imported out of order.'); + if (!fetchFn) throw new Error('realFetch already claimed.'); return fetchFn; } -const realFetch = requireRealFetch(); +// Assigned inside the `fetch` handler's `/run` path (the only call path into +// the functions below) before any of them run — definite-assignment +// asserted rather than left `| undefined` so call sites here don't need +// null checks. +let realFetch!: typeof globalThis.fetch; const DEFAULT_GET_SCHEMA_SAMPLE = 5; @@ -540,6 +545,11 @@ export default { if (url.pathname === '/health') return new Response('ok'); if (url.pathname !== '/run') return new Response('Not found', { status: 404 }); + // First, synchronous statements of the /run path — see guard.ts and the + // requireRealFetch comment above for why this can't happen any earlier. + markRequestHandlingStarted(); + realFetch = requireRealFetch(); + const token = env.APIFY_TOKEN; if (!token) throw new Error('APIFY_TOKEN missing from Actor run environment.'); // APIFY_API_BASE_URL is the platform-internal API (may have a trailing slash). From 100481cd11943c2688fdeca8cbb5611318cdf9ed Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 4 Aug 2026 10:41:32 +0200 Subject: [PATCH 39/46] fix: remove guard.js capability-theft path, add execution safeguards + tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PR #1 review (jirispilka, 2026-08-01): - worker/guard.ts, worker/runner.ts, worker/config.capnp: the previous fix (8930b95) gated runner's internal-API fetch behind an exported markRequestHandlingStarted/claimRealFetch pair. That gate was itself an export, so escaped usercode.js top-level code could call it directly and steal the unrestricted fetch before runner's own claim ran. Root fix: move internal-API access off any module export entirely, onto a workerd env binding (env.INTERNAL_API, wired to its own outbound network service). `env` only ever reaches the genuinely-dispatched fetch(request, env) call — nothing at module-evaluation time can obtain a reference to it, exported or not. guard.js now exports only pure, safe allowlist helpers. Verified live against the original exploit PoC plus two additional attack variants in a local workerd sandbox: all now fail closed with no capability leak. - config.capnp: split ambient outbound (public-only, for guarded fetch) from the internal-API outbound (public/private/local, internal use only) as a second, independent defense layer against SSRF even if the JS-level guard had a bug. - worker/runner.ts: abort Actor runs a script started but left non-terminal when the script itself throws (best-effort, failures reported not thrown). Extracted DEFAULT_WAIT_FOR_FINISH_SECS. Added optional execution-level safeguards (maxActorRuns, maxTotalChargeUsd, defaultTimeoutSecs) so a script can't start unbounded runs or authorize unbounded spend. - tests/unit/guard.test.ts (new, vitest): token-free CI coverage for isAllowedHost/validateUrl/nextRedirectInit/guardedFetch's redirect re-validation, plus a regression test that guard.js exports nothing beyond known-safe helpers. CI now runs typecheck + this suite on every PR. - tests/binding-smoke.ts: every check now asserts the actual returned value instead of only checking the callback didn't throw. - tests/fixtures/realfetch-escape.js, test.sh: updated regression probe and comments for the new architecture (the old exploit call now throws a plain TypeError at module eval, reported as a normal compile-failure diagnostic). - docs/API.md, README.md, .actor/actor.json: documented the new execution limits, and that callAndGetItems/actor.call/run.waitForFinish can return partial/non-terminal results — waitForFinishSecs bounds the API wait only, not the child run's cost or duration. --- .actor/actor.json | 18 + .github/workflows/typecheck.yml | 18 +- .gitignore | 1 + README.md | 8 +- docs/API.md | 28 ++ package.json | 6 +- pnpm-lock.yaml | 740 +++++++++++++++++++++++++++++ test.sh | 21 +- tests/binding-smoke.ts | 61 ++- tests/fixtures/realfetch-escape.js | 48 +- tests/unit/guard.test.ts | 193 ++++++++ tsconfig.json | 2 +- worker/config.capnp | 26 +- worker/entrypoint.sh | 8 + worker/guard.ts | 73 +-- worker/runner.ts | 204 +++++--- 16 files changed, 1301 insertions(+), 154 deletions(-) create mode 100644 tests/unit/guard.test.ts diff --git a/.actor/actor.json b/.actor/actor.json index a17a10b..ed96cb3 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -21,6 +21,24 @@ "type": "string", "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Before writing code that calls a specific Actor, check its real input field names first — via fetch-actor-details (outside this script, before you write it) or apify.actor.get({ actorId }) (inside it, for an Actor picked at runtime). Do not guess field names from memory; a wrong one throws a fast 400, but costs a wasted round trip. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape. If a prior attempt already logged a nested run's defaultDatasetId/defaultKeyValueStoreId (visible in your own earlier turns), reuse it — do NOT re-run the same Actor call with identical input, that wastes compute on a call that already succeeded. apify.actor.call/run.waitForFinish may return non-terminal (READY/RUNNING) once the 60s wait cap elapses — that is NOT a failure, poll again instead of throwing.", "editor": "javascript" + }, + "maxActorRuns": { + "title": "Max Actor runs", + "type": "integer", + "description": "Caps how many Actor runs this script may start in total across actor.start/actor.call/actor.callAndGetItems. Starting one more once the limit is reached throws inside the script. Omit for no limit.", + "minimum": 1 + }, + "maxTotalChargeUsd": { + "title": "Max total charge (USD)", + "type": "number", + "description": "Execution-wide spending budget across every Actor run this script starts — distinct from a single call's own maxTotalChargeUsd, which only caps that one run. Each run's own cap is clamped so the combined total never exceeds this budget; starting a run once it's exhausted throws inside the script. Omit for no limit.", + "minimum": 0 + }, + "defaultTimeoutSecs": { + "title": "Default Actor run timeout (secs)", + "type": "integer", + "description": "Applied as timeoutSecs to actor.start/actor.call/actor.callAndGetItems calls that don't specify their own. Omit to use the Apify API's own default.", + "minimum": 1 } }, "required": ["code"] diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index ccac2fd..a6ab6bb 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -1,4 +1,4 @@ -name: Typecheck +name: Typecheck and test on: push: @@ -17,3 +17,19 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm run typecheck + + # Token-free unit tests (guard.ts's allowlist/redirect logic) — no `apify push`/`apify + # call`, no live Actor run, no APIFY_TOKEN. See tests/unit/guard.test.ts's own header + # comment for what this covers and why it exists (PR #1 review, 2026-07-21: the security + # boundary itself had zero CI coverage before this). + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm run test diff --git a/.gitignore b/.gitignore index 39046c7..63842f2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ worker/usercode.js worker/runner.js worker/guard.js tests/*.js +tests/unit/*.js *.log .DS_Store diff --git a/README.md b/README.md index c8e2886..7ae9eef 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,9 @@ override per call for scripts chaining several long Actor runs (MCP re-run an identical call, it wastes cost. - Print a small JSON summary, never a full dataset — only `console.log`/ `console.info` output comes back; a top-level `return` is **not** captured. +- `callAndGetItems` reads the dataset once, right after its (max 60s) wait — + if the child run is still `RUNNING` at that point, `items` may be empty or + partial. Check the returned `run.status` before treating it as final. ## Input @@ -69,6 +72,9 @@ override per call for scripts chaining several long Actor runs (MCP | Field | Type | Description | |---|---|---| | `code` | string | The JavaScript script to run (JS only, not transpiled). It receives the `apify` binding and `console`. | +| `maxActorRuns` | number | *Optional.* Caps how many Actor runs the script may start in total; exceeding it throws inside the script. | +| `maxTotalChargeUsd` | number | *Optional.* Execution-level spending budget across all runs the script starts (distinct from a single run's own `maxTotalChargeUsd`); exhausting it throws inside the script. | +| `defaultTimeoutSecs` | number | *Optional.* Default `timeoutSecs` for child runs that don't set their own. | ## Output @@ -187,7 +193,7 @@ apify.store({ search, limit?, offset?, category? }) // → { items, count, offs apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (may be non-terminal READY/RUNNING past the 60s cap — not an error, see Recipes) -apify.actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } +apify.actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } (items may be partial if run is still RUNNING — check run.status) // Runs apify.run.get({ runId }) // → run diff --git a/docs/API.md b/docs/API.md index 691ea88..b157831 100644 --- a/docs/API.md +++ b/docs/API.md @@ -152,6 +152,11 @@ dataset via `dataset.listItems`. } ``` +**May return partial results** — if the run is still `RUNNING` when the 60s +wait (`waitForFinishSecs`) elapses, `items` is read from the dataset at that +moment and may be empty or a partial subset of the eventual total. Check the +returned `run.status`; a non-terminal status means `items` is a snapshot, not +the final result. **Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) then [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) @@ -166,6 +171,29 @@ console.log(run.status, items.length); --- +## Execution limits + +Three optional Actor input fields bound a script's ability to start child +Actor runs (`actor.start`/`actor.call`/`actor.callAndGetItems`), independent +of any single call's own `waitForFinishSecs`/`timeoutSecs`/`maxTotalChargeUsd` +(which each bound only that one run): + +| Field | Type | Description | +|---|---|---| +| `maxActorRuns` | `number` | Caps the total number of Actor runs this script may start. Exceeding it throws inside the script. | +| `maxTotalChargeUsd` | `number` | Execution-level spending budget across every run the script starts — distinct from a single run's own `maxTotalChargeUsd`. Each new run's own cap is clamped down so the sum of all committed per-run caps never exceeds this budget; starting a run once the budget is exhausted throws inside the script. | +| `defaultTimeoutSecs` | `number` | Used as a child run's `timeoutSecs` when the script's own `actor.start`/`actor.call`/`actor.callAndGetItems` call didn't specify one. | + +All three are optional — unset means no limit beyond the Apify API's own defaults. + +**`waitForFinishSecs` is not a cost or time limit on the child run.** It only +bounds how long the API request itself waits before returning (see the +non-terminal notes on `actor.call` and `run.waitForFinish` above) — the child +run keeps running, and spending, regardless. `defaultTimeoutSecs` and +`maxTotalChargeUsd` above are what actually bound a run's duration and cost. + +--- + ## `apify.run` ### `run.get({ runId })` → `Run` diff --git a/package.json b/package.json index 4bd7ecd..87fbfaa 100644 --- a/package.json +++ b/package.json @@ -12,9 +12,11 @@ }, "scripts": { "build": "tsc -p tsconfig.json && sed -i '/^export {};$/d' tests/*.js", - "typecheck": "tsc --noEmit -p tsconfig.json" + "typecheck": "tsc --noEmit -p tsconfig.json", + "test": "vitest run" }, "devDependencies": { - "typescript": "6.0.3" + "typescript": "6.0.3", + "vitest": "4.1.10" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8d7fee7..b0b946f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,6 +15,9 @@ importers: typescript: specifier: 6.0.3 version: 6.0.3 + vitest: + specifier: 4.1.10 + version: 4.1.10(vite@8.2.0) packages: @@ -48,11 +51,438 @@ packages: cpu: [x64] os: [win32] + '@emnapi/core@2.0.0-alpha.3': + resolution: {integrity: sha512-AZypUeJ/yByuxyS7BlSNRDOMLMlROYtjYdIAuBmJssVz1UJDSeYxLrdizhXCFYhedC5bqd/ASy8EuNXbVVXp9g==} + + '@emnapi/runtime@2.0.0-alpha.3': + resolution: {integrity: sha512-hFPAhMUjJD9BSyCANEISPOogeXC9Zo9ZQl7L6vKnaVsMkCtzznaW/naYypeyl0Gv5rYfWYsZbpixTMpjDJzQeA==} + + '@emnapi/wasi-threads@2.0.1': + resolution: {integrity: sha512-9DsSk+o5NBX0CCJT8s0EROGSGxjR/tKu6aBTaVyq+SjAEQH4XcdcRxPBRzsBLizTTJ49MJjF+jgu3qnO9GLQcQ==} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@napi-rs/wasm-runtime@1.2.2': + resolution: {integrity: sha512-JfB4kuJQjaoHuCTseIINHtHWeJnvgEcxjwA5t/Y00ZgaOO1Crz3fjT/p8kT28zA/Caz7oiUMn3d6H2yOVCVwuw==} + engines: {node: ^20.19.0 || ^22.13.0 || >=23.5.0} + peerDependencies: + '@emnapi/core': ^1.7.1 || ^2.0.0-alpha.3 + '@emnapi/runtime': ^1.7.1 || ^2.0.0-alpha.3 + + '@oxc-project/types@0.142.0': + resolution: {integrity: sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ==} + + '@rolldown/binding-android-arm64@1.2.1': + resolution: {integrity: sha512-02hOeOSryYxVrOIphmLAsqnCJWxwlzFk+pEt/N/i6OgT3lShHO7xGCU5cpgchRDHboAEbSjzgGh+O/u1GswQmA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@rolldown/binding-darwin-arm64@1.2.1': + resolution: {integrity: sha512-fMsTOnN0OjFm3CyppWPitKnc8UlliVARUULW6cfU6AIqjdtgmSFWSk9vecHzZduv/yMWIHDlRhM1e8Iff9uAfA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@rolldown/binding-darwin-x64@1.2.1': + resolution: {integrity: sha512-1wjKdz/XLGKHaTNHjQveQ/B23TKx4ItAqm1JbyVuvNPc4Ze0Fb48s49TAd/2zcplPl8okE/UbTgmlVfwT7eFeQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@rolldown/binding-freebsd-x64@1.2.1': + resolution: {integrity: sha512-Fa0jHR07E7YBN4vOEsbVf2briYNsuOowfLJaXULZM0ldMlaCaj2LJgLMbMe4iacRyZmvR8efFhgR9wKuGclQUg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@rolldown/binding-linux-arm-gnueabihf@1.2.1': + resolution: {integrity: sha512-pzkgu1SSHGgRRyRZ4fbmSgmajbVt+epaLP99NDjFft69v/ypfTi6swBMiVdh2EkQ0OSnHE1lZDM7DRGkyAzUpA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@rolldown/binding-linux-arm64-gnu@1.2.1': + resolution: {integrity: sha512-QI5SEDY8cbiYWHx0VO4vIc3UlS6a32vXHjU8Qy/17adEmZIPuByJg13UEvo9c/UCiUkdcVWY83C+b+JrwnNyUg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-arm64-musl@1.2.1': + resolution: {integrity: sha512-Sm41FyCeXqmYcERoYOCbGIL5hNfd8w9LQ7Y61Bev48HkcjaJqV/iiVOaiDxjVTRMS+QKrZmD8cfPt4uMVnvM+A==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rolldown/binding-linux-ppc64-gnu@1.2.1': + resolution: {integrity: sha512-2x+WhXTGl9yJYPbltW/BSEPTVz9OIWQyER4N+gJEDWkkn904eRcBzELqh/Hf7K0w/ubGbKNMv0ZC+94QK/IFEg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-s390x-gnu@1.2.1': + resolution: {integrity: sha512-eEjmQpuRQayHPWWnywaWHkFT3ToPbP3RYy42VVd/B9aBGDA+Ol25EIWHxKQST3IiWJjikCWUF7KtbfqwZrzVwQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-gnu@1.2.1': + resolution: {integrity: sha512-/Orga1fZYkLc/56jBICcHrKchl8Z2UKdDSr3LG9ToWO1lQ6a4Livk9Xz+9WN91zsz5QR3XQz2NNoSDEvP6qadw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-musl@1.2.1': + resolution: {integrity: sha512-xxBJRL+0q0Kce7orznGWLuylHDY65vuARXZRpX+hPdv+DqK2c3NlCsVA98tlWzWNEE7yPqA/1NQ5nnCrj49Y5A==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rolldown/binding-openharmony-arm64@1.2.1': + resolution: {integrity: sha512-M6AdXIXw3s+/8XpKMzdGDEXGS1S7kwUsy+rcTIUIOx5Ge4nXKCtAFHFV9YKkXvGcC5WMoTjAteLzlsQROVI0Yw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@rolldown/binding-wasm32-wasi@1.2.1': + resolution: {integrity: sha512-/TX0SoRGojHzSAHpfVBbavRVSazg5U3h3Y3VXfcc0cdugq6kxdqw8LPGFiPr+/7gE/60zRcsOY2Vi9b9eT0jww==} + engines: {node: ^20.19.0 || ^22.13.0 || >=23.5.0} + + '@rolldown/binding-win32-arm64-msvc@1.2.1': + resolution: {integrity: sha512-EvRrivJieyHG+AO9lleZWgq+g0+S7oV2C51yuqlcyU/R9net+sI4Pj0F+lUoP2bEr6TWX3SqFaaS0SzfLxSzkw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@rolldown/binding-win32-x64-msvc@1.2.1': + resolution: {integrity: sha512-Z4eCmn5QJ/5+azF9knpLWKfVd9aidn0mAe9TpJgvBLId9Ax3t0+JVxBmT25Bv7NBbVW1TZyKjQjQReouMeH5UQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@rolldown/pluginutils@1.0.1': + resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@tybys/wasm-util@0.10.3': + resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} + + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@vitest/expect@4.1.10': + resolution: {integrity: sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==} + + '@vitest/mocker@4.1.10': + resolution: {integrity: sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@4.1.10': + resolution: {integrity: sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==} + + '@vitest/runner@4.1.10': + resolution: {integrity: sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==} + + '@vitest/snapshot@4.1.10': + resolution: {integrity: sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==} + + '@vitest/spy@4.1.10': + resolution: {integrity: sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==} + + '@vitest/utils@4.1.10': + resolution: {integrity: sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + chai@6.2.2: + resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} + engines: {node: '>=18'} + + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + es-module-lexer@2.3.1: + resolution: {integrity: sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==} + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + lightningcss-android-arm64@1.33.0: + resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.33.0: + resolution: {integrity: sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.33.0: + resolution: {integrity: sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.33.0: + resolution: {integrity: sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.33.0: + resolution: {integrity: sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.33.0: + resolution: {integrity: sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + lightningcss-linux-arm64-musl@1.33.0: + resolution: {integrity: sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + lightningcss-linux-x64-gnu@1.33.0: + resolution: {integrity: sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + lightningcss-linux-x64-musl@1.33.0: + resolution: {integrity: sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + lightningcss-win32-arm64-msvc@1.33.0: + resolution: {integrity: sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.33.0: + resolution: {integrity: sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.33.0: + resolution: {integrity: sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==} + engines: {node: '>= 12.0.0'} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + nanoid@3.3.16: + resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + obug@2.1.4: + resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} + engines: {node: '>=12.20.0'} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.5: + resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} + engines: {node: '>=12'} + + postcss@8.5.25: + resolution: {integrity: sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==} + engines: {node: ^10 || ^12 || >=14} + + rolldown@1.2.1: + resolution: {integrity: sha512-4FKJhg8d3OiyQOA6Q1Q0hoFFpW9/OoX+VsHzpECsdsIZoOArrAK90gl59YK/Z+gnDel45bgJZK03ozH/9bCqEw==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@4.2.0: + resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@1.3.0: + resolution: {integrity: sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==} + engines: {node: '>=18'} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tinyrainbow@3.1.1: + resolution: {integrity: sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==} + engines: {node: '>=14.0.0'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} hasBin: true + vite@8.2.0: + resolution: {integrity: sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + '@vitejs/devtools': ^0.4.0 + esbuild: ^0.27.0 || ^0.28.0 + jiti: '>=1.21.0' + less: ^4.0.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + '@vitejs/devtools': + optional: true + esbuild: + optional: true + jiti: + optional: true + less: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@4.1.10: + resolution: {integrity: sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.10 + '@vitest/browser-preview': 4.1.10 + '@vitest/browser-webdriverio': 4.1.10 + '@vitest/coverage-istanbul': 4.1.10 + '@vitest/coverage-v8': 4.1.10 + '@vitest/ui': 4.1.10 + happy-dom: '*' + jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + workerd@1.20260402.1: resolution: {integrity: sha512-Cg+OUlukdcCHrTTg0MBCIMFRE6XO3yGVGiWCnJPvfffy2Ga2girrEq3qF/YlHSTmbIyEE5ebCFxBYYYZueQ/Mg==} engines: {node: '>=16'} @@ -75,8 +505,318 @@ snapshots: '@cloudflare/workerd-windows-64@1.20260402.1': optional: true + '@emnapi/core@2.0.0-alpha.3': + dependencies: + '@emnapi/wasi-threads': 2.0.1 + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@2.0.0-alpha.3': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@2.0.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@napi-rs/wasm-runtime@1.2.2(@emnapi/core@2.0.0-alpha.3)(@emnapi/runtime@2.0.0-alpha.3)': + dependencies: + '@emnapi/core': 2.0.0-alpha.3 + '@emnapi/runtime': 2.0.0-alpha.3 + '@tybys/wasm-util': 0.10.3 + optional: true + + '@oxc-project/types@0.142.0': {} + + '@rolldown/binding-android-arm64@1.2.1': + optional: true + + '@rolldown/binding-darwin-arm64@1.2.1': + optional: true + + '@rolldown/binding-darwin-x64@1.2.1': + optional: true + + '@rolldown/binding-freebsd-x64@1.2.1': + optional: true + + '@rolldown/binding-linux-arm-gnueabihf@1.2.1': + optional: true + + '@rolldown/binding-linux-arm64-gnu@1.2.1': + optional: true + + '@rolldown/binding-linux-arm64-musl@1.2.1': + optional: true + + '@rolldown/binding-linux-ppc64-gnu@1.2.1': + optional: true + + '@rolldown/binding-linux-s390x-gnu@1.2.1': + optional: true + + '@rolldown/binding-linux-x64-gnu@1.2.1': + optional: true + + '@rolldown/binding-linux-x64-musl@1.2.1': + optional: true + + '@rolldown/binding-openharmony-arm64@1.2.1': + optional: true + + '@rolldown/binding-wasm32-wasi@1.2.1': + dependencies: + '@emnapi/core': 2.0.0-alpha.3 + '@emnapi/runtime': 2.0.0-alpha.3 + '@napi-rs/wasm-runtime': 1.2.2(@emnapi/core@2.0.0-alpha.3)(@emnapi/runtime@2.0.0-alpha.3) + optional: true + + '@rolldown/binding-win32-arm64-msvc@1.2.1': + optional: true + + '@rolldown/binding-win32-x64-msvc@1.2.1': + optional: true + + '@rolldown/pluginutils@1.0.1': {} + + '@standard-schema/spec@1.1.0': {} + + '@tybys/wasm-util@0.10.3': + dependencies: + tslib: 2.8.1 + optional: true + + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + + '@types/estree@1.0.9': {} + + '@vitest/expect@4.1.10': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.3 + '@vitest/spy': 4.1.10 + '@vitest/utils': 4.1.10 + chai: 6.2.2 + tinyrainbow: 3.1.1 + + '@vitest/mocker@4.1.10(vite@8.2.0)': + dependencies: + '@vitest/spy': 4.1.10 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 8.2.0 + + '@vitest/pretty-format@4.1.10': + dependencies: + tinyrainbow: 3.1.1 + + '@vitest/runner@4.1.10': + dependencies: + '@vitest/utils': 4.1.10 + pathe: 2.0.3 + + '@vitest/snapshot@4.1.10': + dependencies: + '@vitest/pretty-format': 4.1.10 + '@vitest/utils': 4.1.10 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@4.1.10': {} + + '@vitest/utils@4.1.10': + dependencies: + '@vitest/pretty-format': 4.1.10 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.1 + + assertion-error@2.0.1: {} + + chai@6.2.2: {} + + convert-source-map@2.0.0: {} + + detect-libc@2.1.2: {} + + es-module-lexer@2.3.1: {} + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + expect-type@1.4.0: {} + + fdir@6.5.0(picomatch@4.0.5): + optionalDependencies: + picomatch: 4.0.5 + + fsevents@2.3.3: + optional: true + + lightningcss-android-arm64@1.33.0: + optional: true + + lightningcss-darwin-arm64@1.33.0: + optional: true + + lightningcss-darwin-x64@1.33.0: + optional: true + + lightningcss-freebsd-x64@1.33.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.33.0: + optional: true + + lightningcss-linux-arm64-gnu@1.33.0: + optional: true + + lightningcss-linux-arm64-musl@1.33.0: + optional: true + + lightningcss-linux-x64-gnu@1.33.0: + optional: true + + lightningcss-linux-x64-musl@1.33.0: + optional: true + + lightningcss-win32-arm64-msvc@1.33.0: + optional: true + + lightningcss-win32-x64-msvc@1.33.0: + optional: true + + lightningcss@1.33.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.33.0 + lightningcss-darwin-arm64: 1.33.0 + lightningcss-darwin-x64: 1.33.0 + lightningcss-freebsd-x64: 1.33.0 + lightningcss-linux-arm-gnueabihf: 1.33.0 + lightningcss-linux-arm64-gnu: 1.33.0 + lightningcss-linux-arm64-musl: 1.33.0 + lightningcss-linux-x64-gnu: 1.33.0 + lightningcss-linux-x64-musl: 1.33.0 + lightningcss-win32-arm64-msvc: 1.33.0 + lightningcss-win32-x64-msvc: 1.33.0 + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + nanoid@3.3.16: {} + + obug@2.1.4: {} + + pathe@2.0.3: {} + + picocolors@1.1.1: {} + + picomatch@4.0.5: {} + + postcss@8.5.25: + dependencies: + nanoid: 3.3.16 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + rolldown@1.2.1: + dependencies: + '@oxc-project/types': 0.142.0 + '@rolldown/pluginutils': 1.0.1 + optionalDependencies: + '@rolldown/binding-android-arm64': 1.2.1 + '@rolldown/binding-darwin-arm64': 1.2.1 + '@rolldown/binding-darwin-x64': 1.2.1 + '@rolldown/binding-freebsd-x64': 1.2.1 + '@rolldown/binding-linux-arm-gnueabihf': 1.2.1 + '@rolldown/binding-linux-arm64-gnu': 1.2.1 + '@rolldown/binding-linux-arm64-musl': 1.2.1 + '@rolldown/binding-linux-ppc64-gnu': 1.2.1 + '@rolldown/binding-linux-s390x-gnu': 1.2.1 + '@rolldown/binding-linux-x64-gnu': 1.2.1 + '@rolldown/binding-linux-x64-musl': 1.2.1 + '@rolldown/binding-openharmony-arm64': 1.2.1 + '@rolldown/binding-wasm32-wasi': 1.2.1 + '@rolldown/binding-win32-arm64-msvc': 1.2.1 + '@rolldown/binding-win32-x64-msvc': 1.2.1 + + siginfo@2.0.0: {} + + source-map-js@1.2.1: {} + + stackback@0.0.2: {} + + std-env@4.2.0: {} + + tinybench@2.9.0: {} + + tinyexec@1.3.0: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.5) + picomatch: 4.0.5 + + tinyrainbow@3.1.1: {} + + tslib@2.8.1: + optional: true + typescript@6.0.3: {} + vite@8.2.0: + dependencies: + lightningcss: 1.33.0 + picomatch: 4.0.5 + postcss: 8.5.25 + rolldown: 1.2.1 + tinyglobby: 0.2.17 + optionalDependencies: + fsevents: 2.3.3 + + vitest@4.1.10(vite@8.2.0): + dependencies: + '@vitest/expect': 4.1.10 + '@vitest/mocker': 4.1.10(vite@8.2.0) + '@vitest/pretty-format': 4.1.10 + '@vitest/runner': 4.1.10 + '@vitest/snapshot': 4.1.10 + '@vitest/spy': 4.1.10 + '@vitest/utils': 4.1.10 + es-module-lexer: 2.3.1 + expect-type: 1.4.0 + magic-string: 0.30.21 + obug: 2.1.4 + pathe: 2.0.3 + picomatch: 4.0.5 + std-env: 4.2.0 + tinybench: 2.9.0 + tinyexec: 1.3.0 + tinyglobby: 0.2.17 + tinyrainbow: 3.1.1 + vite: 8.2.0 + why-is-node-running: 2.3.0 + transitivePeerDependencies: + - msw + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + workerd@1.20260402.1: optionalDependencies: '@cloudflare/workerd-darwin-64': 1.20260402.1 diff --git a/test.sh b/test.sh index 650b042..baf61dc 100755 --- a/test.sh +++ b/test.sh @@ -41,18 +41,21 @@ done [ "$failed" -eq 0 ] || { echo "==> some probes FAILED" >&2; exit 1; } echo "==> all probes passed" -# Regression probe for the realFetch claim-ordering bug (PR #1 review, -# 2026-07-21): tests/fixtures/realfetch-escape.js escapes usercode.js's -# wrapper into module scope and tries to steal the internal-only realFetch -# before runner.ts's own claim. It has no captured console to report through -# (see the file's own comment), so success/failure is the Actor run itself -# succeeding vs. failing -- not a printed sentinel like the probes above. -echo "==> apify call: tests/fixtures/realfetch-escape.js (regression: realFetch claim ordering)" +# Regression probe for the guard.js capability-theft bug (PR #1 review, +# 2026-07-21 and 2026-08-01): tests/fixtures/realfetch-escape.js escapes +# usercode.js's wrapper into module scope and tries to steal an unrestricted +# fetch by calling guard.js's (now-removed) claimRealFetch export directly. +# It has no captured console to report through (see the file's own comment), +# so success/failure is the Actor run itself succeeding vs. failing -- not a +# printed sentinel like the probes above. The run is expected to SUCCEED +# (with a "Failed to compile" diagnostic item, since claimRealFetch no longer +# exists to call) -- see the fixture's own comment for the full reasoning. +echo "==> apify call: tests/fixtures/realfetch-escape.js (regression: guard.js capability theft)" jq -n --arg code "$(cat tests/fixtures/realfetch-escape.js)" '{ code: $code }' > "$input_json" if apify call -f "$input_json" -o; then - echo "==> realfetch-escape passed (run succeeded — module-scope steal attempt did not hijack/crash the internal claim)" + echo "==> realfetch-escape passed (run succeeded — module-scope steal attempt found nothing to steal)" else - echo "==> realfetch-escape FAILED (run crashed — realFetch claim-ordering regression, see guard.ts's requestHandlingStarted gate)" >&2 + echo "==> realfetch-escape FAILED (run crashed — capability-theft regression, see guard.ts/runner.ts/config.capnp's INTERNAL_API binding)" >&2 failed=1 fi diff --git a/tests/binding-smoke.ts b/tests/binding-smoke.ts index c43ab83..a805010 100644 --- a/tests/binding-smoke.ts +++ b/tests/binding-smoke.ts @@ -23,20 +23,30 @@ async function check(name: string, fn: () => Promise): Promise { } const ACTOR = 'apify/hello-world'; +const [ACTOR_USERNAME, ACTOR_NAME] = ACTOR.split('/'); + +// Every status the Apify API can return for a run. Used to check a returned status is a +// real value, not just any truthy string -- mirrors TERMINAL_STATUSES in worker/runner.ts. +const RUN_STATUSES = new Set(['READY', 'RUNNING', 'SUCCEEDED', 'FAILED', 'ABORTING', 'ABORTED', 'TIMING-OUT', 'TIMED-OUT']); // ---- actor (read) ---- await check('store', async () => { const page = await apify.store({ search: 'hello world', limit: 3 }); - if (!Array.isArray(page.items)) throw new Error('expected items array'); + if (!Array.isArray(page.items) || page.items.length === 0) throw new Error(`expected non-empty items array, got ${JSON.stringify(page.items)}`); + if (typeof page.items[0].name !== 'string') throw new Error(`expected item.name string, got ${JSON.stringify(page.items[0])}`); return `${page.count} actors`; }); await check('store (for await)', async () => { let n = 0; for await (const _ of apify.store({ search: 'hello world', limit: 3 })) n++; + if (n === 0) throw new Error('expected at least 1 actor iterated, got 0'); return `${n} actors iterated`; }); await check('actor.get', async () => { const d = await apify.actor.get({ actorId: ACTOR }); + if (d.username !== ACTOR_USERNAME || d.name !== ACTOR_NAME) { + throw new Error(`actor.get returned username=${d.username} name=${d.name}, expected ${ACTOR_USERNAME}/${ACTOR_NAME}`); + } return `${d.username}/${d.name}`; }); @@ -44,6 +54,7 @@ await check('actor.get', async () => { let datasetId = ''; await check('dataset.create', async () => { datasetId = (await apify.dataset.create()).id as string; + if (!datasetId) throw new Error('dataset.create returned no id'); return datasetId; }); await check('dataset.pushItems', async () => { @@ -52,15 +63,25 @@ await check('dataset.pushItems', async () => { }); await check('dataset.listItems', async () => { const page = await apify.dataset.listItems({ datasetId }); + if (page.count !== 2 || page.items[0]?.a !== 1 || page.items[0]?.b !== 'x' || page.items[1]?.a !== 2 || page.items[1]?.b !== 'y') { + throw new Error(`expected 2 pushed items round-tripped, got ${JSON.stringify(page.items)}`); + } return `${page.count} items, offset=${page.offset}, limit=${page.limit}`; }); await check('dataset.inferFields', async () => { const s = await apify.dataset.inferFields({ datasetId }); - return `itemCount=${s.itemCount} fields=${s.fields.map((f) => f.name).join(',')}`; + const fieldNames = s.fields.map((f) => f.name).sort(); + if (fieldNames.join(',') !== 'a,b') throw new Error(`expected fields a,b, got ${fieldNames.join(',')}`); + const a = s.fields.find((f) => f.name === 'a'); + const b = s.fields.find((f) => f.name === 'b'); + if (!a?.types.includes('number')) throw new Error(`expected field a to include type number, got ${JSON.stringify(a)}`); + if (!b?.types.includes('string')) throw new Error(`expected field b to include type string, got ${JSON.stringify(b)}`); + return `itemCount=${s.itemCount} fields=${fieldNames.join(',')}`; }); await check('dataset.listItems (for await)', async () => { let n = 0; for await (const _ of apify.dataset.listItems({ datasetId, limit: 1 })) n++; + if (n !== 2) throw new Error(`expected 2 pushed items iterated, got ${n}`); return `${n} iterated`; }); @@ -76,13 +97,18 @@ await check('keyValueStore.set', async () => { return 'set obj + txt'; }); await check('keyValueStore.get', async () => { - const obj = await apify.keyValueStore.get({ storeId, key: 'obj' }) as { hello: string }; + const obj = await apify.keyValueStore.get({ storeId, key: 'obj' }) as { hello?: unknown } | null; + if (obj?.hello !== 'world') throw new Error(`kvs.get returned ${JSON.stringify(obj)}`); const txt = await apify.keyValueStore.get({ storeId, key: 'txt' }); + if (txt !== 'plain') throw new Error(`kvs.get txt returned ${JSON.stringify(txt)}`); const missing = await apify.keyValueStore.get({ storeId, key: 'nope' }); + if (missing !== null) throw new Error(`kvs.get missing key returned ${JSON.stringify(missing)}, expected null`); return `obj.hello=${obj.hello} txt=${txt} missing=${missing}`; }); await check('keyValueStore.list', async () => { - const l = await apify.keyValueStore.list({ storeId }) as { items: unknown[] }; + const l = await apify.keyValueStore.list({ storeId }) as { items: { key: string }[] }; + const keys = l.items.map((i) => i.key); + if (!keys.includes('obj') || !keys.includes('txt')) throw new Error(`expected keys obj+txt in list, got ${JSON.stringify(keys)}`); return `${l.items.length} keys`; }); @@ -91,31 +117,48 @@ let runId = ''; await check('actor.start', async () => { const run = await apify.actor.start({ actorId: ACTOR }); runId = run.id as string; + if (!runId) throw new Error('actor.start returned no run id'); + if (!RUN_STATUSES.has(run.status)) throw new Error(`actor.start returned unexpected status: ${JSON.stringify(run.status)}`); return `runId=${runId} status=${run.status}`; }); await check('run.get', async () => { - return `status=${(await apify.run.get({ runId })).status}`; + const run = await apify.run.get({ runId }); + if (run.id !== runId) throw new Error(`run.get returned id=${run.id}, expected ${runId}`); + if (!RUN_STATUSES.has(run.status)) throw new Error(`run.get returned unexpected status: ${JSON.stringify(run.status)}`); + return `status=${run.status}`; }); await check('run.waitForFinish', async () => { - return `status=${(await apify.run.waitForFinish({ runId, waitForFinishSecs: 60 })).status}`; + const run = await apify.run.waitForFinish({ runId, waitForFinishSecs: 60 }); + if (!RUN_STATUSES.has(run.status)) throw new Error(`run.waitForFinish returned unexpected status: ${JSON.stringify(run.status)}`); + return `status=${run.status}`; }); await check('run.getLog', async () => { - return `${(await apify.run.getLog({ runId, limit: 200 })).length} chars`; + const log = await apify.run.getLog({ runId, limit: 200 }); + if (typeof log !== 'string' || log.length === 0) throw new Error(`run.getLog returned ${JSON.stringify(log)}`); + return `${log.length} chars`; }); // ---- run + get items (sync) ---- await check('actor.call', async () => { - return `status=${(await apify.actor.call({ actorId: ACTOR, waitForFinishSecs: 60 })).status}`; + const run = await apify.actor.call({ actorId: ACTOR, waitForFinishSecs: 60 }); + if (!RUN_STATUSES.has(run.status)) throw new Error(`actor.call returned unexpected status: ${JSON.stringify(run.status)}`); + return `status=${run.status}`; }); await check('actor.callAndGetItems', async () => { const { run, items } = await apify.actor.callAndGetItems({ actorId: ACTOR, limit: 5, waitForFinishSecs: 60 }); + if (!RUN_STATUSES.has(run.status)) throw new Error(`actor.callAndGetItems returned unexpected status: ${JSON.stringify(run.status)}`); + if (!Array.isArray(items)) throw new Error(`actor.callAndGetItems returned non-array items: ${JSON.stringify(items)}`); return `status=${run.status} items=${items.length}`; }); // ---- abort ---- await check('run.abort', async () => { const run = await apify.actor.start({ actorId: ACTOR }); - return `status=${(await apify.run.abort({ runId: run.id as string })).status}`; + const aborted = await apify.run.abort({ runId: run.id as string }); + if (aborted.status !== 'ABORTING' && aborted.status !== 'ABORTED') { + throw new Error(`run.abort returned unexpected status: ${JSON.stringify(aborted.status)}`); + } + return `status=${aborted.status}`; }); const passed = results.filter(Boolean).length; diff --git a/tests/fixtures/realfetch-escape.js b/tests/fixtures/realfetch-escape.js index 5d591f5..829f903 100644 --- a/tests/fixtures/realfetch-escape.js +++ b/tests/fixtures/realfetch-escape.js @@ -1,6 +1,6 @@ -// Regression probe for the realFetch claim-ordering bug found in PR #1's -// review (2026-07-21): -// https://github.com/apify/actor-code-runtime/pull/1#issuecomment-5037390847 +// Regression probe for the guard.js capability-theft bug found in PR #1 review +// (2026-07-21, and again 2026-08-01 against the first attempted fix): +// https://github.com/apify/actor-code-runtime/pull/1#discussion_r3707244830 // // entrypoint.sh splices `code` verbatim (no escaping) into // `export async function run(apify, console) { }`. The bare `}` right @@ -8,13 +8,31 @@ // reproduce the escape -- so `run` becomes a harmless no-op (nothing is left // in its body once the comments end) and everything after runs as ordinary // MODULE-SCOPE code in usercode.js: workerd evaluates that unconditionally, -// before this worker ever calls runner.ts's request handler. That used to be -// enough to `await import('./guard.js')` and call claimRealFetch() directly, -// stealing the unrestricted, un-allowlisted fetch before runner.ts's own -// claim (previously made at runner.ts's own module top level) ever ran -- -// which made THAT claim get null, throw, and crash the whole Actor run -// (self-DoS; the real exploit payoff for an attacker would be using the -// stolen fetch directly, not reported here). +// before this worker ever calls runner.ts's request handler. +// +// The first fix attempt (guard.ts's requestHandlingStarted gate) still +// exported a setter (markRequestHandlingStarted) and getter (claimRealFetch) +// that escaped code could call directly, since usercode.js shares guard.js's +// module graph -- any export is equally reachable from both. This probe's +// payload (below) calls exactly that: `claimRealFetch()`, expecting it back +// as a callable, unrestricted fetch function. +// +// The actual fix (see guard.ts/runner.ts/config.capnp) removes that export +// surface entirely: this worker's own internal-API access is now a workerd +// env binding (env.INTERNAL_API), which only ever reaches the genuinely- +// dispatched fetch(request, env) call -- nothing at module-evaluation time +// receives a reference to it, exported or otherwise. guard.js now only +// exports pure allowlist helpers (isAllowedHost, validateUrl, nextRedirectInit, +// guardedFetch -- see tests/unit/guard.test.ts's "never exports a raw/ +// unrestricted fetch capability" regression test for that invariant directly). +// +// So `claimRealFetch` no longer exists on the imported module: this line now +// throws a plain TypeError ("claimRealFetch is not a function") during module +// evaluation -- an uncaught exception at that point crashes workerd's own +// startup, which entrypoint.sh's push_compile_failure() already detects and +// reports as a normal, SUCCEEDED Actor run with a "Failed to compile: ..." +// diagnostic item (exitCode 1) -- not a hard Actor run failure, and no +// capability is exposed either way. See test.sh for how this is asserted. // // Not valid JS on its own (it opens with an unbalanced `}`) -- intentionally, // same shape as the reported PoC. Not TypeScript, not compiled, not @@ -22,16 +40,6 @@ // outside the `tests/*.js` build-artifact glob): see test.sh, which pushes // this file's raw content directly as the `code` input. // -// Expected result with the fix (guard.ts's requestHandlingStarted gate): -// claimRealFetch() called from module scope returns null without consuming -// the resource, so nothing crashes -- this Actor run completes normally -// (exitCode 0, "Script completed"), same as any run of an empty script. -// Before the fix: this run FAILS outright (workerd crashes during module -// evaluation; entrypoint.sh can't tell that apart from a real infra failure -// and fails the whole Actor run). test.sh asserts on `apify call`'s own exit -// status, not a printed sentinel -- this probe's `run` body never executes -// any of its own code, so it has no captured console to report through. -// // Must be genuine top-level await, not an async IIFE: a module containing // top-level await defers the evaluation of modules that depend on it (here, // runner.ts) until that await settles (see MDN/TC39 "Top-level await", diff --git a/tests/unit/guard.test.ts b/tests/unit/guard.test.ts new file mode 100644 index 0000000..2a47d05 --- /dev/null +++ b/tests/unit/guard.test.ts @@ -0,0 +1,193 @@ +// Token-free, CI-runnable unit tests for worker/guard.ts's allowlist logic — no workerd, +// no `apify push`/`apify call`, no live Actor run. Fills the gap flagged in PR #1 review +// (2026-07-21): CI only ran `pnpm run typecheck`; every behavioral test required a live +// token, so `isAllowedHost`/`validateUrl`'s allowlist paths and `guardedFetch`'s redirect +// re-validation (the entire reason that function exists) had no test of any kind. +// +// guard.ts overrides `globalThis.fetch` as a side effect of being imported, and captures +// whatever `globalThis.fetch` was *at that moment* as its own internal `realFetch` (used by +// guardedFetch to perform the actual, pre-validated request). So: stub `globalThis.fetch` +// with a controllable mock BEFORE importing guard.ts, then call the exported `guardedFetch` +// directly — it runs against the mock, no network I/O, fully deterministic. +import { describe, expect, it, vi, beforeAll } from 'vitest'; + +let guard: typeof import('../../worker/guard.js'); +let mockFetch: ReturnType; + +beforeAll(async () => { + mockFetch = vi.fn(); + vi.stubGlobal('fetch', mockFetch); + guard = await import('../../worker/guard.js'); +}); + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }); +} + +function redirectResponse(location: string, status: number): Response { + return new Response(null, { status, headers: { location } }); +} + +describe('isAllowedHost', () => { + it.each([ + ['apify.com', true], + ['api.apify.com', true], + ['deeply.nested.apify.com', true], + ['APIFY.COM', true], // case-insensitive + ['apify.com.', true], // trailing FQDN dot stripped + ['evilapify.com', false], // suffix without the separating dot + ['apify.com.evil.com', false], // real host is evil.com + ['notapify.com', false], + ['example.com', false], + ])('%s -> %s', (hostname, expected) => { + expect(guard.isAllowedHost(hostname)).toBe(expected); + }); +}); + +describe('validateUrl', () => { + it('accepts an allowed https URL', () => { + expect(guard.validateUrl('https://api.apify.com/v2/foo').hostname).toBe('api.apify.com'); + }); + + it('accepts an allowed http URL', () => { + expect(guard.validateUrl('http://apify.com/').hostname).toBe('apify.com'); + }); + + it('rejects a disallowed host', () => { + expect(() => guard.validateUrl('https://example.com/')).toThrow(/only apify\.com and its subdomains/); + }); + + it('rejects the userinfo trick (real host is evil.com)', () => { + expect(() => guard.validateUrl('https://apify.com@evil.com/')).toThrow(/evil\.com/); + }); + + it('rejects the path trick (real host is evil.com)', () => { + expect(() => guard.validateUrl('https://evil.com/apify.com')).toThrow(/evil\.com/); + }); + + it('rejects a non-http(s) protocol', () => { + expect(() => guard.validateUrl('ftp://apify.com/')).toThrow(/protocol/); + }); + + it('rejects an unparseable URL', () => { + expect(() => guard.validateUrl('not a url')).toThrow(/only absolute http\(s\) URLs/); + }); + + it('resolves a Request object by its .url', () => { + expect(guard.validateUrl(new Request('https://apify.com/x')).hostname).toBe('apify.com'); + }); +}); + +describe('nextRedirectInit', () => { + it('303 downgrades GET regardless of original method', () => { + expect(guard.nextRedirectInit({ method: 'POST', body: 'x' }, 303)).toEqual({ method: 'GET', body: undefined }); + }); + + it('301 downgrades POST to GET', () => { + expect(guard.nextRedirectInit({ method: 'POST', body: 'x' }, 301)).toEqual({ method: 'GET', body: undefined }); + }); + + it('302 downgrades POST to GET', () => { + expect(guard.nextRedirectInit({ method: 'POST', body: 'x' }, 302)).toEqual({ method: 'GET', body: undefined }); + }); + + it('301 preserves a GET request unchanged', () => { + const init = { method: 'GET' }; + expect(guard.nextRedirectInit(init, 301)).toBe(init); + }); + + it('307 preserves method and body', () => { + const init = { method: 'POST', body: 'x' }; + expect(guard.nextRedirectInit(init, 307)).toBe(init); + }); + + it('308 preserves method and body', () => { + const init = { method: 'POST', body: 'x' }; + expect(guard.nextRedirectInit(init, 308)).toBe(init); + }); + + it('defaults to GET when no method given', () => { + expect(guard.nextRedirectInit(undefined, 303)).toEqual({ method: 'GET', body: undefined }); + }); +}); + +describe('guardedFetch', () => { + it('rejects a disallowed URL before making any request', async () => { + mockFetch.mockClear(); + await expect(guard.guardedFetch('https://example.com/', undefined)).rejects.toThrow(/only apify\.com/); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('performs the request when the URL is allowed', async () => { + mockFetch.mockClear(); + mockFetch.mockResolvedValueOnce(jsonResponse({ ok: true })); + const response = await guard.guardedFetch('https://api.apify.com/v2/browser-info', undefined); + expect(response.status).toBe(200); + expect(mockFetch).toHaveBeenCalledTimes(1); + const [, init] = mockFetch.mock.calls[0]; + expect(init.redirect).toBe('manual'); // never lets the underlying fetch auto-follow + }); + + it('follows a redirect to another allowed host', async () => { + mockFetch.mockClear(); + mockFetch + .mockResolvedValueOnce(redirectResponse('https://sub.apify.com/next', 302)) + .mockResolvedValueOnce(jsonResponse({ ok: true })); + const response = await guard.guardedFetch('https://apify.com/start', undefined); + expect(response.status).toBe(200); + expect(mockFetch).toHaveBeenCalledTimes(2); + expect(mockFetch.mock.calls[1][0]).toBe('https://sub.apify.com/next'); + }); + + it('re-validates each redirect hop and blocks a hop to a disallowed host', async () => { + mockFetch.mockClear(); + mockFetch.mockResolvedValueOnce(redirectResponse('https://evil.com/steal', 302)); + await expect(guard.guardedFetch('https://apify.com/start', undefined)).rejects.toThrow(/only apify\.com/); + expect(mockFetch).toHaveBeenCalledTimes(1); // never followed the malicious hop + }); + + it('resolves a relative redirect Location against the current URL', async () => { + mockFetch.mockClear(); + mockFetch + .mockResolvedValueOnce(redirectResponse('/v2/next', 302)) + .mockResolvedValueOnce(jsonResponse({ ok: true })); + await guard.guardedFetch('https://api.apify.com/v2/start', undefined); + expect(mockFetch.mock.calls[1][0]).toBe('https://api.apify.com/v2/next'); + }); + + it('downgrades a POST to GET on a 302, dropping the body', async () => { + mockFetch.mockClear(); + mockFetch + .mockResolvedValueOnce(redirectResponse('https://apify.com/next', 302)) + .mockResolvedValueOnce(jsonResponse({ ok: true })); + await guard.guardedFetch('https://apify.com/start', { method: 'POST', body: 'payload' }); + const [, secondInit] = mockFetch.mock.calls[1]; + expect(secondInit.method).toBe('GET'); + expect(secondInit.body).toBeUndefined(); + }); + + it('gives up after MAX_REDIRECT_HOPS redirects to allowed hosts', async () => { + mockFetch.mockClear(); + for (let i = 0; i < 10; i++) mockFetch.mockResolvedValueOnce(redirectResponse('https://apify.com/loop', 302)); + await expect(guard.guardedFetch('https://apify.com/start', undefined)).rejects.toThrow(/exceeded/); + }); + + it('returns a redirect response unchanged when it carries no Location header', async () => { + mockFetch.mockClear(); + mockFetch.mockResolvedValueOnce(new Response(null, { status: 302 })); + const response = await guard.guardedFetch('https://apify.com/start', undefined); + expect(response.status).toBe(302); + }); +}); + +describe('module exports', () => { + it('never exports a raw/unrestricted fetch capability', () => { + // Regression guard for PR #1's finding: guard.js must never export anything that + // hands the caller an unwrapped fetch function or a way to bypass the allowlist. + // Every export must be one of these known-safe, pure helpers. + const knownSafeExports = new Set(['isAllowedHost', 'validateUrl', 'nextRedirectInit', 'guardedFetch']); + for (const key of Object.keys(guard)) { + expect(knownSafeExports.has(key)).toBe(true); + } + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 704b2ad..973d4b2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,5 +6,5 @@ "lib": ["es2022", "dom"], "strict": true }, - "include": ["worker/*.ts", "tests/*.ts"] + "include": ["worker/*.ts", "tests/*.ts", "tests/unit/*.ts"] } diff --git a/worker/config.capnp b/worker/config.capnp index c78ebbb..9814b82 100644 --- a/worker/config.capnp +++ b/worker/config.capnp @@ -3,11 +3,18 @@ using Workerd = import "/workerd/workerd.capnp"; const config :Workerd.Config = ( services = [ (name = "main", worker = .codeRuntime), - # Outbound for fetch(). Apify's api.apify.com may resolve to a private - # address inside the platform network, so allow private/local too. - # tlsOptions enables HTTPS egress (trust workerd's built-in CA set); - # the hostname allowlist is enforced by guard.js, not here. - (name = "internet", network = (allow = ["public", "private", "local"], tlsOptions = (trustBrowserCas = true))), + # Ambient outbound for the sandboxed script's own fetch() calls (guard.js allowlists + # the hostname on top of this; this is a second, independent layer — restricted to + # "public" so that even a JS-level guard bug can't reach private/link-local addresses + # (e.g. cloud metadata) via a stolen or unwrapped fetch reference). + (name = "internet", network = (allow = ["public"], tlsOptions = (trustBrowserCas = true))), + # This worker's own calls to the platform-internal Apify API (bound as env.INTERNAL_API + # in runner.ts, not exposed to the ambient/guarded fetch above). api.apify.com may + # resolve to a private address inside the platform network, hence the broader allowlist + # here — this service is never reachable from usercode.js: it's only ever passed as + # part of `env`, which workerd hands solely to the genuinely-dispatched + # `fetch(request, env)` call, never to any module-scope code. See runner.ts/guard.ts. + (name = "internalApi", network = (allow = ["public", "private", "local"], tlsOptions = (trustBrowserCas = true))), ], # __PORT__ is substituted by entrypoint.sh from its own $PORT before workerd # starts — single source of truth, see entrypoint.sh. @@ -37,6 +44,15 @@ const codeRuntime :Workerd.Worker = ( # This run's own meta.origin (e.g. "MCP" when apify-mcp-server started it), # forwarded to sub-runs this script starts — see PARENT_ORIGIN in runner.ts. (name = "PARENT_ORIGIN", fromEnvironment = "APIFY_META_ORIGIN"), + # Execution-level safeguards on Actor runs this script starts (all optional — + # see runner.ts's Limits and docs/API.md's "Execution limits"). Sourced from the + # Actor's own input fields, exported as env vars by entrypoint.sh. + (name = "MAX_ACTOR_RUNS", fromEnvironment = "CODE_RUNTIME_MAX_ACTOR_RUNS"), + (name = "MAX_TOTAL_CHARGE_USD", fromEnvironment = "CODE_RUNTIME_MAX_TOTAL_CHARGE_USD"), + (name = "DEFAULT_TIMEOUT_SECS", fromEnvironment = "CODE_RUNTIME_DEFAULT_TIMEOUT_SECS"), + # Unrestricted fetch to the platform-internal API, scoped to its own outbound network + # service above — see this file's "internalApi" service and runner.ts/guard.ts. + (name = "INTERNAL_API", service = "internalApi"), ], globalOutbound = "internet", compatibilityDate = "2026-01-15", diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh index bcdb6ce..fb0938a 100755 --- a/worker/entrypoint.sh +++ b/worker/entrypoint.sh @@ -37,6 +37,14 @@ fi printf '\n}\n' } > /app/worker/usercode.js +# Execution-level safeguards (all optional Actor input fields — see runner.ts's Limits and +# docs/API.md's "Execution limits"). `// empty` yields an empty string (not "0"/"null") when +# the field is absent; runner.ts's parsePositiveNumberEnv treats a blank value as "no limit +# configured". +export CODE_RUNTIME_MAX_ACTOR_RUNS="$(jq -r '.maxActorRuns // empty' < /tmp/input.json)" +export CODE_RUNTIME_MAX_TOTAL_CHARGE_USD="$(jq -r '.maxTotalChargeUsd // empty' < /tmp/input.json)" +export CODE_RUNTIME_DEFAULT_TIMEOUT_SECS="$(jq -r '.defaultTimeoutSecs // empty' < /tmp/input.json)" + # config.capnp hardcodes __PORT__ as a placeholder so the port has one source ($PORT above). sed -i "s/__PORT__/${PORT}/" /app/worker/config.capnp diff --git a/worker/guard.ts b/worker/guard.ts index 46c62ab..17d42a5 100644 --- a/worker/guard.ts +++ b/worker/guard.ts @@ -1,9 +1,6 @@ // Restrict the user program's outbound network to apify.com and its subdomains. // Imported before usercode.js so the overrides are in place even for code that -// runs at module-evaluation time. Our own Apify API calls use the real, -// unrestricted fetch (the internal API is a private IP, not *.apify.com) — -// see claimRealFetch() below for how runner.js gets it without leaving it -// usable by user code. +// runs at module-evaluation time. // // Egress surface (workerd, no nodejs_compat): the only JS-reachable outbound // primitives are fetch, WebSocket, and EventSource. Raw sockets (node:net, @@ -12,54 +9,32 @@ // because runner.js and the apify binding never use them — leaving them would // be a non-fetch egress path around the allowlist (apify/ai-team#216 finding A, // via WebSocket). If a future need arises, wrap them like fetch instead. - -const realFetch = globalThis.fetch.bind(globalThis); - -// One-shot handoff of the unrestricted fetch to runner.js — gated on genuine -// request handling, not on import order between guard.js and usercode.js. -// -// entrypoint.sh splices the `code` input verbatim (no escaping) into -// `export async function run(apify, console) { }`. A bare `}` in -// `code` closes that function early; everything after it runs as ordinary -// MODULE-SCOPE code in usercode.js, executed during module evaluation — -// i.e. unconditionally, before workerd ever calls this worker's own -// `fetch(request, env)` handler. That escaped code can `await -// import('./guard.js')` and call claimRealFetch() itself. // -// This module previously assumed import order made runner.js's own claim run -// first (guard.js has no dependency on usercode.js, so it evaluates before -// it) — that protects nothing here, because it's usercode.js's top-level code -// racing runner.ts's top-level code, and usercode.js is the import evaluated -// immediately before runner.ts's own body runs. Verified live (PR #1 review, -// 2026-07-21): a `code` input using the escape above reliably wins that race, -// so runner.ts's own claim got null and threw, crashing the whole Actor run. +// This module used to also hand runner.js an unrestricted "real fetch" for its +// own internal API calls, via a pair of exports (markRequestHandlingStarted / +// claimRealFetch). That capability-through-export design was broken: anything +// in usercode.js's module scope can `import('./guard.js')` too (ES modules +// have no notion of a "trusted" importer), so user code could call the same +// exports runner.js did and steal the unrestricted fetch before runner.js's +// own claim ran (PR #1 review, 2026-07-21 and again 2026-08-01 — the second +// round found the first fix's gate was itself still an exported, callable +// setter). Any function this module exports is equally reachable from +// usercode.js, so no export-based gate can be made sound. // -// The actual invariant that holds regardless of any module-graph ordering: -// usercode.js's module scope never runs *inside* a `fetch(request, env)` -// call — module evaluation always finishes before workerd dispatches the -// first request. So claimRealFetch() refuses to hand anything out until -// markRequestHandlingStarted() has been called; runner.ts calls it (and -// claims) as the first, synchronous statement of its `/run` handling, before -// any `await`, so no attacker-scheduled microtask can race it either. A claim -// attempted before that (legitimate or injected) gets null without consuming -// the resource, so the real claim still succeeds afterward. -let requestHandlingStarted = false; -export function markRequestHandlingStarted(): void { - requestHandlingStarted = true; -} - -let unclaimedRealFetch: typeof realFetch | null = realFetch; -export function claimRealFetch(): typeof realFetch | null { - if (!requestHandlingStarted) return null; // too early to be a trusted caller - const fetchFn = unclaimedRealFetch; - unclaimedRealFetch = null; - return fetchFn; -} +// The actual fix moves runner.js's internal API access off of a module export +// entirely and onto workerd's own env binding (`INTERNAL_API` in +// config.capnp, wired to a separate outbound network service — see there). +// `env` is a parameter workerd hands only to the genuinely-dispatched +// `fetch(request, env)` call; nothing at module-evaluation time (including an +// escaped top-level statement in usercode.js) ever receives a reference to +// it, so there is nothing here for user code to import or steal. This module +// no longer needs to capture or export a privileged fetch at all. +const realFetch = globalThis.fetch.bind(globalThis); // Match apify.com exactly or any subdomain. The leading dot in the suffix is // what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` // (ends with `.evil.com`) both fail. -function isAllowedHost(hostname: string): boolean { +export function isAllowedHost(hostname: string): boolean { const host = hostname.toLowerCase().replace(/\.$/, ''); // strip FQDN trailing dot return host === 'apify.com' || host.endsWith('.apify.com'); } @@ -73,7 +48,7 @@ function requestUrl(input: RequestInfo | URL): string { // Parses and validates one URL against the allowlist. Returns the parsed URL // (callers use it to resolve a relative redirect Location) or throws. -function validateUrl(input: RequestInfo | URL): URL { +export function validateUrl(input: RequestInfo | URL): URL { let url: URL; try { // Parse to the real host — defeats userinfo (`apify.com@evil.com`), @@ -100,14 +75,14 @@ function validateUrl(input: RequestInfo | URL): URL { const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); const MAX_REDIRECT_HOPS = 5; -function nextRedirectInit(init: RequestInit | undefined, status: number): RequestInit | undefined { +export function nextRedirectInit(init: RequestInit | undefined, status: number): RequestInit | undefined { const method = (init?.method ?? 'GET').toUpperCase(); const downgradeToGet = status === 303 || ((status === 301 || status === 302) && method === 'POST'); if (!downgradeToGet) return init; return { ...init, method: 'GET', body: undefined }; } -async function guardedFetch(input: RequestInfo | URL, init: RequestInit | undefined, hop = 0): Promise { +export async function guardedFetch(input: RequestInfo | URL, init: RequestInit | undefined, hop = 0): Promise { if (hop > MAX_REDIRECT_HOPS) { throw new Error(`Blocked fetch: exceeded ${MAX_REDIRECT_HOPS} redirects`); } diff --git a/worker/runner.ts b/worker/runner.ts index ad3d1ec..48729db 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -7,34 +7,31 @@ // Single-tenant: one run = one container = one program = one token. No Worker // Loader / per-request isolate is needed — the program runs in this worker, // which is itself the sandbox (no filesystem, restricted outbound network). -// guard.js overrides globalThis.fetch to allow only apify.com, and hands us -// the real, unrestricted fetch via a one-shot claimRealFetch() for our own -// (internal) API calls. That claim is gated on markRequestHandlingStarted() -// (called below, first thing in the `fetch` handler's `/run` path), not on -// import order — usercode.js's module scope can run attacker-controlled code -// before this module's own top-level code does, so the claim must not -// happen at this module's top level either. See guard.ts's comment on -// claimRealFetch for the full reasoning. -import { claimRealFetch, markRequestHandlingStarted } from './guard.js'; +// guard.js overrides globalThis.fetch to allow only apify.com. +// +// This worker's own (internal) API calls need an unrestricted, un-allowlisted +// fetch — the internal API is a private IP, not *.apify.com. That capability +// is bound as `env.INTERNAL_API` (config.capnp), a workerd service binding +// wired to a separate outbound network, not a shared/exported fetch reference. +// `env` is only ever handed to the genuinely-dispatched `fetch(request, env)` +// call below by workerd's own runtime — nothing at module-evaluation time +// (including attacker-controlled top-level code that escapes usercode.js's +// wrapper, see entrypoint.sh) ever receives a reference to it, so there is +// nothing for user code to import or steal. See guard.ts for why an +// export-based handoff (this worker's previous design) could not be made +// sound: usercode.js shares guard.js's module graph, so any function guard.js +// exported was equally callable by escaped user code. import { run } from './usercode.js'; -// Called as the first, synchronous statement of the `/run` path in the -// `fetch` handler below — before any `await`, so no attacker-scheduled -// microtask from usercode.js's module scope can call claimRealFetch() in -// between. See guard.ts. -function requireRealFetch(): typeof globalThis.fetch { - const fetchFn = claimRealFetch(); - if (!fetchFn) throw new Error('realFetch already claimed.'); - return fetchFn; -} -// Assigned inside the `fetch` handler's `/run` path (the only call path into -// the functions below) before any of them run — definite-assignment -// asserted rather than left `| undefined` so call sites here don't need -// null checks. -let realFetch!: typeof globalThis.fetch; +type Fetcher = { fetch(input: RequestInfo | URL, init?: RequestInit): Promise }; const DEFAULT_GET_SCHEMA_SAMPLE = 5; +// The Apify API caps a single actor.call/run.waitForFinish wait at 60s +// (a REST API limit, not this Actor's) — see docs/API.md's Recipes section +// for the poll-in-a-loop pattern for longer runs. +const DEFAULT_WAIT_FOR_FINISH_SECS = 60; + // --- Types --------------------------------------------------------------- // The Apify API returns many more fields per record than this code reads. Rather // than inventing a full schema we don't have, ApifyRecord asserts nothing beyond @@ -47,6 +44,7 @@ interface ApifyRecord { interface RunRecord extends ApifyRecord { id: string; + status: string; } type SearchParamValue = string | number | boolean | undefined | null; @@ -179,6 +177,17 @@ interface Env { // started it. Platform-injected, not user-settable: unlike an Actor input // field, a script running inside this Actor cannot spoof it. PARENT_ORIGIN?: string; + // Execution-level safeguards on Actor runs a script starts via + // actor.start/call/callAndGetItems — see makeApifyBinding's `Limits` and + // docs/API.md's "Execution limits" section. All optional; unset means + // "no limit beyond the Apify API's own defaults". + MAX_ACTOR_RUNS?: string; + MAX_TOTAL_CHARGE_USD?: string; + DEFAULT_TIMEOUT_SECS?: string; + // Unrestricted (non-allowlisted) fetch for this worker's own calls to the + // platform-internal Apify API. Bound to a separate outbound network + // service in config.capnp — see this file's header comment and guard.ts. + INTERNAL_API: Fetcher; } interface OutputItem { @@ -247,7 +256,18 @@ function makePaginatedList>( const MCP_ORIGIN = 'MCP'; const REQUEST_ORIGIN_HEADER = 'X-Apify-Request-Origin'; -function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | undefined) { +// Execution-level safeguards on Actor runs a script starts (actor.start/call/ +// callAndGetItems, which all funnel through createRun() below). Independent of +// any single run's own timeoutSecs/waitForFinishSecs/maxTotalChargeUsd, which +// only bound THAT run — nothing previously bounded how many runs one script +// could start, or their combined cost. See docs/API.md's "Execution limits". +interface Limits { + maxActorRuns?: number; + maxTotalChargeUsd?: number; + defaultTimeoutSecs?: number; +} + +function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | undefined, internalFetch: Fetcher['fetch'], limits: Limits) { // Every request this Actor makes identifies itself; requests made while THIS // run's own origin is MCP additionally forward that origin so runs started by // apify.actor.start/call/callAndGetItems() below get meta.origin: 'MCP' too, @@ -290,7 +310,7 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u headers['content-type'] = contentType ?? 'application/json'; } } - const response = await realFetch(buildUrl(path, searchParams), { method, headers, body: requestBody }); + const response = await internalFetch(buildUrl(path, searchParams), { method, headers, body: requestBody }); if (!response.ok) throw new Error(`${method} ${path} failed: ${response.status} ${await response.text()}`); return response; }; @@ -306,29 +326,54 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // Run IDs this script itself started, via actor.call() / actor.start() (and transitively // actor.callAndGetItems(), which shares createRun() below). run.abort() below is scoped to // this set — a script can only abort runs it started, not any account-wide runId it's - // handed or guesses. + // handed or guesses. Also used by abortTrackedRuns() (called from the top-level exception + // handler below) to clean up runs still going when the script itself crashed. const startedRunIds = new Set(); - - // POST /acts/:id/runs, shared by actor.call() (start+wait, waitForFinishSecs defaults to 60, - // capped at 60s per the Apify API — for longer runs use start() + apify.run.waitForFinish()) - // and actor.start() (async kickoff, no wait). Returns the run record so the caller can read - // defaultDatasetId / defaultKeyValueStoreId. Intentionally does NOT use /run-sync, which - // returns the OUTPUT KVS record (a pattern only some Actors follow) rather than the - // structured run record. - const createRun = ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }: StartOptions): Promise => - apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { + const TERMINAL_STATUSES = new Set(['SUCCEEDED', 'FAILED', 'ABORTED', 'ABORTING', 'TIMED-OUT']); + const nonTerminalRunIds = new Set(); + + // Conservative execution-level cost cap: each run's OWN maxTotalChargeUsd is a ceiling, + // not a bill, so this tracks committed ceilings (not realized spend) against + // limits.maxTotalChargeUsd and never lets a script authorize more combined ceiling than + // that budget, even though actual spend will usually be lower. + let committedChargeUsd = 0; + + // POST /acts/:id/runs, shared by actor.call() (start+wait, waitForFinishSecs defaults to + // DEFAULT_WAIT_FOR_FINISH_SECS, capped at 60s per the Apify API — for longer runs use + // start() + apify.run.waitForFinish()) and actor.start() (async kickoff, no wait). Returns + // the run record so the caller can read defaultDatasetId / defaultKeyValueStoreId. + // Intentionally does NOT use /run-sync, which returns the OUTPUT KVS record (a pattern + // only some Actors follow) rather than the structured run record. + const createRun = ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }: StartOptions): Promise => { + if (limits.maxActorRuns !== undefined && startedRunIds.size >= limits.maxActorRuns) { + throw new Error(`Blocked actor run: this script already started ${startedRunIds.size} Actor run(s), the configured limit is ${limits.maxActorRuns}`); + } + let effectiveMaxCharge = maxTotalChargeUsd; + if (limits.maxTotalChargeUsd !== undefined) { + const remaining = limits.maxTotalChargeUsd - committedChargeUsd; + if (remaining <= 0) { + throw new Error(`Blocked actor run: execution spending budget of $${limits.maxTotalChargeUsd} is exhausted`); + } + // A run without its own cap could spend the whole remaining budget; a run with + // its own cap higher than what's left gets clamped down to what's left. + effectiveMaxCharge = effectiveMaxCharge === undefined ? remaining : Math.min(effectiveMaxCharge, remaining); + } + return apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { searchParams: { waitForFinish: waitForFinishSecs, memory: memoryMbytes, - timeout: timeoutSecs, - maxTotalChargeUsd, + timeout: timeoutSecs ?? limits.defaultTimeoutSecs, + maxTotalChargeUsd: effectiveMaxCharge, maxItems, }, body: input ?? {}, }).then((runRecord: RunRecord) => { startedRunIds.add(runRecord.id); + if (!TERMINAL_STATUSES.has(runRecord.status)) nonTerminalRunIds.add(runRecord.id); + if (effectiveMaxCharge !== undefined) committedChargeUsd += effectiveMaxCharge; return runRecord; }); + }; const actor = { get: ({ actorId }: ActorIdOptions): Promise => @@ -337,16 +382,21 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether // waitForFinish is set. Records the created run's ID in startedRunIds so run.abort() // can be scoped to runs this script itself started (see the run.abort definition below). - call: (opts: StartOptions): Promise => createRun({ waitForFinishSecs: 60, ...opts }), + call: (opts: StartOptions): Promise => createRun({ waitForFinishSecs: DEFAULT_WAIT_FOR_FINISH_SECS, ...opts }), // Async kickoff. Returns immediately with a run record in READY/RUNNING state. start: (opts: StartOptions): Promise => createRun(opts), - // Runs an Actor (same as call(), waitForFinishSecs defaults to 60) and returns its - // dataset items in one call. Calls createRun() directly rather than through - // `actor.call()` — same underlying request, no self-reference to `actor` needed. + // Runs an Actor (same as call(), waitForFinishSecs defaults to DEFAULT_WAIT_FOR_FINISH_SECS) + // and returns its dataset items in one call. Calls createRun() directly rather than + // through `actor.call()` — same underlying request, no self-reference to `actor` needed. + // + // If the run is still RUNNING when the wait elapses, this reads whatever the dataset + // holds at that moment — items may be empty or a partial subset of the eventual total. + // Check `run.status` (returned alongside `items`); a non-terminal status means the + // items are a snapshot, not the final result — see docs/API.md. callAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { - const runRecord = await createRun({ actorId, input, waitForFinishSecs: 60, ...runOpts }); + const runRecord = await createRun({ actorId, input, waitForFinishSecs: DEFAULT_WAIT_FOR_FINISH_SECS, ...runOpts }); const { items } = await dataset.listItems({ datasetId: runRecord.defaultDatasetId as string, fields, limit, }); @@ -360,10 +410,13 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // Block until the run terminates or `waitForFinishSecs` elapses (whichever comes first). // The Apify API caps this at 60s per request; longer waits require a polling loop. - waitForFinish: ({ runId, waitForFinishSecs = 60 }: WaitOptions): Promise => - apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { + waitForFinish: async ({ runId, waitForFinishSecs = DEFAULT_WAIT_FOR_FINISH_SECS }: WaitOptions): Promise => { + const runRecord: RunRecord = await apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { searchParams: { waitForFinish: waitForFinishSecs }, - }), + }); + if (TERMINAL_STATUSES.has(runRecord.status)) nonTerminalRunIds.delete(runId); + return runRecord; + }, // Scoped to runs this script itself started (see startedRunIds above) — without this, // any runId a script is handed (e.g. read from a dataset item, or guessed) could abort @@ -372,6 +425,7 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u if (!startedRunIds.has(runId)) { throw new Error(`Blocked run.abort: "${runId}" was not started by this script`); } + nonTerminalRunIds.delete(runId); return apiData('POST', `/actor-runs/${encodeURIComponent(runId)}/abort`); }, @@ -455,7 +509,7 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // Returns null when the key does not exist (404), not an error — this matches the common // "lookup or default" pattern in code. get: async ({ storeId, key }: KeyValueStoreGetOptions): Promise => { - const response = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { + const response = await internalFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { headers: baseHeaders, }); if (response.status === 404) return null; @@ -511,27 +565,42 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u return makePaginatedList(fetchPage, offset, limit); }; + // Best-effort cleanup for runs this script started but left non-terminal (e.g. the + // script itself threw with a run still in progress). Not part of the frozen `apify` + // binding handed to user code — called directly by the top-level exception handler + // below. One bad abort must not stop the others (the run tracking loop's whole point is + // to catch stragglers after an error): a run that already finished on the platform + // between our last check and now is an *expected* abort failure (the API rejects + // aborting a finished run), not a bug, so failures are reported back for logging, never + // thrown. + const abortTrackedRuns = async (): Promise<{ runId: string; error: string }[]> => { + const runIds = [...nonTerminalRunIds]; + const results = await Promise.allSettled(runIds.map((runId) => run.abort({ runId }))); + return results.flatMap((result, i) => result.status === 'rejected' ? [{ runId: runIds[i], error: errorMessage(result.reason) }] : []); + }; + // Freeze every namespace (and the wrapper) so the script can't reassign a method to // corrupt its own behavior or, for `console` below, its own output capture. - return Object.freeze({ + const binding = Object.freeze({ actor: Object.freeze(actor), store, run: Object.freeze(run), dataset: Object.freeze(dataset), keyValueStore: Object.freeze(keyValueStore), }); + return { binding, abortTrackedRuns }; } // The shape handed to user code as the `apify` binding. Exported (type-only — // erased at compile time) so tests/*.ts can type-check probes against the same // surface real usercode.js runs against, without importing runner.ts at runtime. -export type ApifyBinding = ReturnType; +export type ApifyBinding = ReturnType['binding']; // Push the captured streams as a single item to the run's default dataset. -async function pushOutput(apiV2: string, token: string, env: Env, item: OutputItem): Promise { +async function pushOutput(apiV2: string, token: string, internalFetch: Fetcher['fetch'], env: Env, item: OutputItem): Promise { const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; if (!datasetId) throw new Error('Default dataset ID missing from Actor run environment.'); - const response = await realFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { + const response = await internalFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json; charset=utf-8' }, body: JSON.stringify(item), @@ -539,21 +608,33 @@ async function pushOutput(apiV2: string, token: string, env: Env, item: OutputIt if (!response.ok) throw new Error(`Failed to push dataset item: ${response.status} ${await response.text()}`); } +// Parses an optional positive-number env var (as set by entrypoint.sh from Actor input). +// Absent, blank, non-numeric, or non-positive all mean "no limit configured" — this is a +// human-edited-adjacent path (Actor input -> env var), not a write-time-validated one, so a +// malformed value fails open to "no limit" rather than crashing the run. +function parsePositiveNumberEnv(value: string | undefined): number | undefined { + if (!value) return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; +} + export default { async fetch(request: Request, env: Env): Promise { const url = new URL(request.url); if (url.pathname === '/health') return new Response('ok'); if (url.pathname !== '/run') return new Response('Not found', { status: 404 }); - // First, synchronous statements of the /run path — see guard.ts and the - // requireRealFetch comment above for why this can't happen any earlier. - markRequestHandlingStarted(); - realFetch = requireRealFetch(); - const token = env.APIFY_TOKEN; if (!token) throw new Error('APIFY_TOKEN missing from Actor run environment.'); // APIFY_API_BASE_URL is the platform-internal API (may have a trailing slash). const apiV2 = `${(env.API_BASE_URL || 'https://api.apify.com').replace(/\/+$/, '')}/v2`; + const internalFetch: Fetcher['fetch'] = (input, init) => env.INTERNAL_API.fetch(input, init); + + const limits: Limits = { + maxActorRuns: parsePositiveNumberEnv(env.MAX_ACTOR_RUNS), + maxTotalChargeUsd: parsePositiveNumberEnv(env.MAX_TOTAL_CHARGE_USD), + defaultTimeoutSecs: parsePositiveNumberEnv(env.DEFAULT_TIMEOUT_SECS), + }; const stdout: string[] = []; const stderr: string[] = []; @@ -565,6 +646,8 @@ export default { info: (...args: unknown[]) => stdout.push(args.map(stringify).join(' ')), }); + const { binding, abortTrackedRuns } = makeApifyBinding(token, apiV2, env.PARENT_ORIGIN, internalFetch, limits); + // A thrown program is a user-level failure: capture it in stderr and still // push the output, so the run SUCCEEDS with diagnostics. Infra failures // (missing env, dataset push) throw and fail the run. @@ -580,14 +663,21 @@ export default { let exitCode = 0; let statusMessage = 'Script completed'; try { - await run(makeApifyBinding(token, apiV2, env.PARENT_ORIGIN), captureConsole); + await run(binding, captureConsole); } catch (err) { stderr.push(errorDetail(err)); exitCode = 1; statusMessage = `Script threw: ${errorMessage(err)}`; + // Best-effort: a script that started Actor runs and then crashed shouldn't leave + // them running unattended. Failures here don't change exitCode/statusMessage — + // the script's own failure is the primary signal; cleanup is secondary. + const abortFailures = await abortTrackedRuns(); + for (const { runId, error } of abortFailures) { + stderr.push(`Cleanup: failed to abort run ${runId}: ${error}`); + } } - await pushOutput(apiV2, token, env, { + await pushOutput(apiV2, token, internalFetch, env, { stdout: stdout.join('\n'), stderr: stderr.join('\n'), exitCode, From 11699fd948a710c95f137fceb94165434f771d2f Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 4 Aug 2026 11:15:46 +0200 Subject: [PATCH 40/46] fix: restore guard.js import, close TOCTOU race, add real-workerd integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scout-loop iteration 1 (4 parallel reviewers: security, code-quality, coding-standards, feature-impact) found real regressions in the previous commit: - worker/runner.ts: the previous commit's rewrite DROPPED the guard.js side-effect import entirely while removing the old claimRealFetch import — guard.js was never imported by anything in the real module graph, so its fetch/WebSocket/EventSource overrides never ran. Every script's plain fetch()/WebSocket had unrestricted egress. Confirmed live (workerd), fixed by restoring `import './guard.js'` as the first import, with a comment on why the order matters and pointing at the regression test (tests/sandbox-isolation.ts's existing 'block https://example.com/' check, now backed by a new real-workerd integration test). - worker/runner.ts's createRun(): maxActorRuns/maxTotalChargeUsd were checked synchronously but only recorded after their POST resolved, so N concurrent calls (this Actor's own documented 'Bounded parallel fan-out' recipe) all read pre-reservation counters and all passed. Now reserves synchronously before the first await, rolls back on failure. Covered by a new integration test that actually races 5 concurrent actor.start() calls against a real workerd process. - worker/runner.ts: froze the default export object so escaped module-scope code can't reassign .fetch to a wrapper that captures the real request/env on the next genuine dispatch — closes a residual capability-theft vector of the same class already fixed twice in this file, found by feature-impact review of the seam. - .actor/actor.json: maxTotalChargeUsd allowed 0, which parsePositiveNumberEnv treats as 'unset' (unlimited) — the exact inverse of a user setting a bash safety budget. Schema now requires a positive value, matching its sibling fields. - worker/guard.ts: guardedFetch's redirect-hop counter was a public parameter on an exported function (any escaped-code caller could pass a pre-inflated value to defeat MAX_REDIRECT_HOPS) — split into an unexported recursive helper. - worker/runner.ts: renamed the ad-hoc TERMINAL_STATUSES (which included the non-terminal ABORTING) to DONE_TRACKING_STATUSES with a comment on why, to stop it reading as (and drifting from) docs/API.md's actual terminal-status set. - makeApifyBinding/pushOutput: switched to object parameters (5 positional args each, over this codebase's own >3-params convention). - tests/integration/ (new): boots a real workerd process against the actual compiled worker/*.js with a local mock internal-API server — closes the gap unit tests structurally can't (guard.js correct-in-isolation vs. actually-wired-in; Limits enforcement holding under real concurrency). Runs in CI, still fully offline. - Also: RunRecord.defaultDatasetId typed properly (removed an unjustified cast), Limits fields use `| undefined` per this codebase's own convention, vitest configs added (was silently double-running the suite via build byproducts), @types/node added scoped to tests/integration only (kept separate from the rest of the program, which intentionally models workerd's no-nodejs_compat environment). --- .actor/actor.json | 6 +- .github/workflows/typecheck.yml | 19 ++++ .gitignore | 1 + package.json | 7 +- pnpm-lock.yaml | 32 ++++-- tests/integration/harness.ts | 153 ++++++++++++++++++++++++++ tests/integration/workerd-e2e.test.ts | 133 ++++++++++++++++++++++ tests/unit/guard.test.ts | 11 +- tsconfig.integration.json | 11 ++ tsconfig.json | 5 + vitest.config.ts | 11 ++ vitest.integration.config.ts | 12 ++ worker/guard.ts | 11 +- worker/runner.ts | 124 +++++++++++++++------ 14 files changed, 485 insertions(+), 51 deletions(-) create mode 100644 tests/integration/harness.ts create mode 100644 tests/integration/workerd-e2e.test.ts create mode 100644 tsconfig.integration.json create mode 100644 vitest.config.ts create mode 100644 vitest.integration.config.ts diff --git a/.actor/actor.json b/.actor/actor.json index ed96cb3..461c3dc 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -31,11 +31,11 @@ "maxTotalChargeUsd": { "title": "Max total charge (USD)", "type": "number", - "description": "Execution-wide spending budget across every Actor run this script starts — distinct from a single call's own maxTotalChargeUsd, which only caps that one run. Each run's own cap is clamped so the combined total never exceeds this budget; starting a run once it's exhausted throws inside the script. Omit for no limit.", - "minimum": 0 + "description": "Execution-wide spending budget across every Actor run this script starts — distinct from a single call's own maxTotalChargeUsd, which only caps that one run. Each run's own cap is clamped so the combined total never exceeds this budget; starting a run once it's exhausted throws inside the script. Omit for no limit. Must be greater than 0 — to allow zero Actor runs, omit the code's actor.start/call/callAndGetItems calls instead.", + "exclusiveMinimum": 0 }, "defaultTimeoutSecs": { - "title": "Default Actor run timeout (secs)", + "title": "Default Actor run timeout (seconds)", "type": "integer", "description": "Applied as timeoutSecs to actor.start/actor.call/actor.callAndGetItems calls that don't specify their own. Omit to use the Apify API's own default.", "minimum": 1 diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index a6ab6bb..22bff04 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -33,3 +33,22 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm run test + + # Boots a real workerd process against the actual compiled worker/*.js (not a mock) — the + # only thing that can catch "guard.js's logic is correct but never actually wired into the + # module graph" (a real regression found in review) or "the execution-limit safeguards look + # right but don't hold under concurrent use" (another real regression found in review). See + # tests/integration/workerd-e2e.test.ts's own header comment. Still token-free/offline — the + # internal Apify API is a local mock (tests/integration/harness.ts), no live platform needed. + test-integration: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm build + - run: pnpm run test:integration diff --git a/.gitignore b/.gitignore index 63842f2..3a27ff4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ worker/runner.js worker/guard.js tests/*.js tests/unit/*.js +tests/integration/*.js *.log .DS_Store diff --git a/package.json b/package.json index 87fbfaa..9008cd2 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "0.1.0", "description": "workerd as a normal (per-run) Apify Actor: one static worker, booted once per run, runs the submitted script and exits.", "private": true, + "type": "module", "packageManager": "pnpm@11.1.3", "dependencies": { "workerd": "1.20260402.1" @@ -12,10 +13,12 @@ }, "scripts": { "build": "tsc -p tsconfig.json && sed -i '/^export {};$/d' tests/*.js", - "typecheck": "tsc --noEmit -p tsconfig.json", - "test": "vitest run" + "typecheck": "tsc --noEmit -p tsconfig.json && tsc --noEmit -p tsconfig.integration.json", + "test": "vitest run", + "test:integration": "vitest run --config vitest.integration.config.ts" }, "devDependencies": { + "@types/node": "24.13.3", "typescript": "6.0.3", "vitest": "4.1.10" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0b946f..8366a92 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,12 +12,15 @@ importers: specifier: 1.20260402.1 version: 1.20260402.1 devDependencies: + '@types/node': + specifier: 24.13.3 + version: 24.13.3 typescript: specifier: 6.0.3 version: 6.0.3 vitest: specifier: 4.1.10 - version: 4.1.10(vite@8.2.0) + version: 4.1.10(@types/node@24.13.3)(vite@8.2.0(@types/node@24.13.3)) packages: @@ -185,6 +188,9 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/node@24.13.3': + resolution: {integrity: sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==} + '@vitest/expect@4.1.10': resolution: {integrity: sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==} @@ -394,6 +400,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + vite@8.2.0: resolution: {integrity: sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -599,6 +608,10 @@ snapshots: '@types/estree@1.0.9': {} + '@types/node@24.13.3': + dependencies: + undici-types: 7.18.2 + '@vitest/expect@4.1.10': dependencies: '@standard-schema/spec': 1.1.0 @@ -608,13 +621,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.1 - '@vitest/mocker@4.1.10(vite@8.2.0)': + '@vitest/mocker@4.1.10(vite@8.2.0(@types/node@24.13.3))': dependencies: '@vitest/spy': 4.1.10 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.2.0 + vite: 8.2.0(@types/node@24.13.3) '@vitest/pretty-format@4.1.10': dependencies: @@ -777,7 +790,9 @@ snapshots: typescript@6.0.3: {} - vite@8.2.0: + undici-types@7.18.2: {} + + vite@8.2.0(@types/node@24.13.3): dependencies: lightningcss: 1.33.0 picomatch: 4.0.5 @@ -785,12 +800,13 @@ snapshots: rolldown: 1.2.1 tinyglobby: 0.2.17 optionalDependencies: + '@types/node': 24.13.3 fsevents: 2.3.3 - vitest@4.1.10(vite@8.2.0): + vitest@4.1.10(@types/node@24.13.3)(vite@8.2.0(@types/node@24.13.3)): dependencies: '@vitest/expect': 4.1.10 - '@vitest/mocker': 4.1.10(vite@8.2.0) + '@vitest/mocker': 4.1.10(vite@8.2.0(@types/node@24.13.3)) '@vitest/pretty-format': 4.1.10 '@vitest/runner': 4.1.10 '@vitest/snapshot': 4.1.10 @@ -807,8 +823,10 @@ snapshots: tinyexec: 1.3.0 tinyglobby: 0.2.17 tinyrainbow: 3.1.1 - vite: 8.2.0 + vite: 8.2.0(@types/node@24.13.3) why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 24.13.3 transitivePeerDependencies: - msw diff --git a/tests/integration/harness.ts b/tests/integration/harness.ts new file mode 100644 index 0000000..445323f --- /dev/null +++ b/tests/integration/harness.ts @@ -0,0 +1,153 @@ +// Shared harness for integration tests that boot a REAL workerd instance against the actual +// compiled worker/runner.js + worker/guard.js (not a mock, not just the pure functions vitest's +// unit suite exercises in isolation). This is what closes the gap unit tests structurally +// cannot: whether guard.js is actually wired into the module graph, whether env.INTERNAL_API +// dispatch actually works, and whether the execution-limit safeguards actually fire end to end. +// Needs `pnpm build` to have produced worker/runner.js + worker/guard.js first (the "build" +// step in .github/workflows/typecheck.yml's integration job — see there). +import { spawn, type ChildProcess } from 'node:child_process'; +import { createServer, type Server } from 'node:http'; +import { mkdtempSync, writeFileSync, readFileSync, cpSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createRequire } from 'node:module'; + +const WORKERD_STARTUP_TIMEOUT_MS = 5_000; +const WORKERD_STARTUP_POLL_INTERVAL_MS = 100; + +function workerdBinaryPath(): string { + // Same resolution Dockerfile's builder stage uses: workerd ships its binary path via the + // package's own `default` export, one level of indirection because the platform-specific + // binary lives in an optional dependency (@cloudflare/workerd-linux-64 etc). `workerd` + // itself is a CommonJS package with no ESM entry point, hence createRequire rather than a + // static import. + const require = createRequire(import.meta.url); + return require('workerd').default; +} + +// A minimal stand-in for the platform-internal Apify API — just enough to make +// actor.start/dataset operations/pushOutput resolve, so a script's real behavior (including +// safeguard rejections, which happen before any HTTP call) is observable end to end. +export interface MockApi { + server: Server; + port: number; + requests: { method: string; path: string; body: string }[]; + close: () => Promise; +} + +export async function startMockApi(): Promise { + const requests: MockApi['requests'] = []; + const server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', (chunk) => chunks.push(chunk)); + req.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8'); + requests.push({ method: req.method ?? '', path: req.url ?? '', body }); + res.setHeader('content-type', 'application/json'); + if (req.method === 'POST' && req.url?.includes('/datasets/') && req.url.endsWith('/items')) { + res.writeHead(201); + res.end('{}'); + } else if (req.method === 'POST' && req.url?.includes('/acts/') && req.url.endsWith('/runs')) { + res.writeHead(201); + res.end(JSON.stringify({ data: { id: `run-${requests.length}`, status: 'READY', defaultDatasetId: 'ds1' } })); + } else if (req.method === 'GET' && req.url?.includes('/datasets/') && req.url.includes('/items')) { + res.writeHead(200); + res.end('[]'); + } else { + res.writeHead(200); + res.end(JSON.stringify({ data: { id: 'x', status: 'SUCCEEDED' } })); + } + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('mock API failed to bind a port'); + return { + server, + port: address.port, + requests, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} + +export interface RunOptions { + /** Actor input fields beyond `code`, e.g. { maxActorRuns: 1 } — mirrors what entrypoint.sh reads. */ + inputFields?: Record; +} + +export interface RunResult { + /** The pushed dataset item, if pushOutput ran (empty object if the worker crashed at startup). */ + pushedItem: Record | null; + /** True if workerd itself started and served /health before we tore it down. */ + startedCleanly: boolean; + /** Raw stderr from the workerd process (useful for asserting startup-crash diagnostics). */ + stderr: string; + mockApi: MockApi; +} + +// Boots a fresh workerd instance with `code` wrapped exactly like entrypoint.sh does, against a +// fresh MockApi standing in for the internal Apify API, sends one /run request, and tears both +// down. Mirrors entrypoint.sh's own env var wiring (CODE_RUNTIME_* for the execution limits) +// rather than reinventing a second convention. +export async function runScript(code: string, options: RunOptions = {}): Promise { + const mockApi = await startMockApi(); + const workDir = mkdtempSync(join(tmpdir(), 'code-runtime-it-')); + const repoRoot = join(import.meta.dirname, '..', '..'); + + cpSync(join(repoRoot, 'worker', 'runner.js'), join(workDir, 'runner.js')); + cpSync(join(repoRoot, 'worker', 'guard.js'), join(workDir, 'guard.js')); + writeFileSync(join(workDir, 'usercode.js'), `export async function run(apify, console) {\n${code}\n}\n`); + + const port = 10_000 + Math.floor(Math.random() * 10_000); + // config.capnp's __PORT__ placeholder appears twice (once in a comment, once in the real + // socket address) — replaceAll, not replace, or the comment's occurrence "wins" and the + // real one is left as the literal string "__PORT__" (workerd then fails DNS-resolving it + // as a port/service name). + const configTemplate = readFileSync(join(repoRoot, 'worker', 'config.capnp'), 'utf8'); + writeFileSync(join(workDir, 'config.capnp'), configTemplate.replaceAll('__PORT__', String(port))); + + const inputFields = options.inputFields ?? {}; + const child: ChildProcess = spawn(workerdBinaryPath(), ['serve', '--experimental', join(workDir, 'config.capnp')], { + env: { + ...process.env, + APIFY_TOKEN: 'fake-token', + ACTOR_DEFAULT_DATASET_ID: 'ds-default', + APIFY_API_BASE_URL: `http://127.0.0.1:${mockApi.port}`, + APIFY_META_ORIGIN: '', + CODE_RUNTIME_MAX_ACTOR_RUNS: inputFields.maxActorRuns !== undefined ? String(inputFields.maxActorRuns) : '', + CODE_RUNTIME_MAX_TOTAL_CHARGE_USD: inputFields.maxTotalChargeUsd !== undefined ? String(inputFields.maxTotalChargeUsd) : '', + CODE_RUNTIME_DEFAULT_TIMEOUT_SECS: inputFields.defaultTimeoutSecs !== undefined ? String(inputFields.defaultTimeoutSecs) : '', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr?.on('data', (chunk) => { stderr += chunk.toString(); }); + + const deadline = Date.now() + WORKERD_STARTUP_TIMEOUT_MS; + let startedCleanly = false; + while (Date.now() < deadline) { + try { + const res = await fetch(`http://127.0.0.1:${port}/health`); + if (res.ok) { startedCleanly = true; break; } + } catch { /* not up yet, or crashed — keep polling until the deadline */ } + if (child.exitCode !== null) break; // crashed at startup, no point polling further + await new Promise((resolve) => setTimeout(resolve, WORKERD_STARTUP_POLL_INTERVAL_MS)); + } + + if (startedCleanly) { + try { + await fetch(`http://127.0.0.1:${port}/run`, { method: 'POST' }); + } catch { /* the /run call itself may crash the worker — that's a result to assert on, not a harness failure */ } + } + + child.kill(); + await mockApi.close(); + + const pushRequest = mockApi.requests.find((r) => r.method === 'POST' && r.path.endsWith('/items') && r.path.includes('ds-default')); + return { + pushedItem: pushRequest ? JSON.parse(pushRequest.body) : null, + startedCleanly, + stderr, + mockApi, + }; +} diff --git a/tests/integration/workerd-e2e.test.ts b/tests/integration/workerd-e2e.test.ts new file mode 100644 index 0000000..3c2286a --- /dev/null +++ b/tests/integration/workerd-e2e.test.ts @@ -0,0 +1,133 @@ +// Integration tests against a REAL workerd process running the actual compiled +// worker/runner.js + worker/guard.js — not a mock of guard.ts's functions (see +// tests/unit/guard.test.ts for that), but the real module graph, the real workerd request +// dispatch, and the real env.INTERNAL_API binding. This is what proves guard.js is actually +// wired in (not just correct in isolation) and that the execution-limit safeguards actually +// fire under real, concurrent use — both gaps a pure unit test structurally cannot close. +// +// Needs worker/runner.js + worker/guard.js to exist (`pnpm build` first — see +// .github/workflows/typecheck.yml's integration job). Slower and more flaky-prone than the +// unit suite (spawns a real process, real ports) — kept in its own directory/config so it can +// be run and reasoned about separately. +import { describe, expect, it } from 'vitest'; +import { runScript } from './harness.js'; + +describe('guard.js is actually enforced (not just correct in isolation)', () => { + it('blocks a disallowed host for an ordinary, non-escaping script', async () => { + const result = await runScript(` + try { + const r = await fetch('http://example.com/'); + console.log('LEAK status=' + r.status); + } catch (e) { + console.log('blocked: ' + e.message); + } + `); + expect(result.startedCleanly).toBe(true); + expect(result.pushedItem?.stdout).toMatch(/^blocked: Blocked fetch/); + expect(result.pushedItem?.stdout).not.toMatch(/LEAK/); + }); + + it('allows a fetch to an apify.com host', async () => { + // The Actor's own internal-API mock isn't apify.com, so this only proves the guard's + // ALLOW path is reachable (doesn't hang/throw before even trying) — the mock returns a + // real HTTP response for any host once workerd's own DNS/connect succeeds, which for + // a host that doesn't resolve (apify.com does, publicly) would time out instead of + // throwing a "Blocked fetch" error. Asserting the ABSENCE of the guard's own blocked- + // fetch error message is what distinguishes "guard let it through" from "guard blocked + // it" here, without depending on live internet access from the test runner. + const result = await runScript(` + try { + await fetch('https://apify.com/'); + console.log('reached apify.com (no guard rejection)'); + } catch (e) { + console.log('error: ' + e.message); + } + `); + expect(result.pushedItem?.stdout).not.toMatch(/Blocked fetch/); + }); + + it('blocks WebSocket construction', async () => { + const result = await runScript(` + try { + new WebSocket('ws://example.com/'); + console.log('LEAK: constructed'); + } catch (e) { + console.log('blocked: ' + e.message); + } + `); + expect(result.pushedItem?.stdout).toMatch(/^blocked: Blocked WebSocket/); + }); + + it('the PR #1 capability-theft PoC fails closed with no capability leak', async () => { + // Same payload as tests/fixtures/realfetch-escape.js: escapes the usercode.js wrapper + // into module scope and tries to call guard.js's (removed) claimRealFetch export. + const result = await runScript(` +} +globalThis.__stolenRealFetch = (await import('./guard.js')).claimRealFetch(); +;{ + `); + expect(result.startedCleanly).toBe(false); // crashes at module eval, same as entrypoint.sh expects + expect(result.stderr).toMatch(/claimRealFetch is not a function/); + }); +}); + +describe('execution-limit safeguards fire under real (including concurrent) use', () => { + it('maxActorRuns blocks a run past the configured limit (sequential)', async () => { + const result = await runScript(` + await apify.actor.start({ actorId: 'apify/hello-world' }); + try { + await apify.actor.start({ actorId: 'apify/hello-world' }); + console.log('LEAK: second run started'); + } catch (e) { + console.log('blocked: ' + e.message); + } + `, { inputFields: { maxActorRuns: 1 } }); + expect(result.pushedItem?.stdout).toMatch(/^blocked: Blocked actor run/); + }); + + it('maxActorRuns blocks a run past the limit even when calls race concurrently', async () => { + // Regression test for the TOCTOU race: createRun() used to only record a started run + // AFTER its POST resolved, so N concurrent calls (this Actor's own documented "Bounded + // parallel fan-out" recipe) all read the pre-reservation count and all passed the + // check. createRun() now reserves synchronously before the first await. + const result = await runScript(` + const results = await Promise.allSettled( + Array.from({ length: 5 }, () => apify.actor.start({ actorId: 'apify/hello-world' })), + ); + const started = results.filter((r) => r.status === 'fulfilled').length; + const blocked = results.filter((r) => r.status === 'rejected').length; + console.log('started=' + started + ' blocked=' + blocked); + `, { inputFields: { maxActorRuns: 1 } }); + expect(result.pushedItem?.stdout).toBe('started=1 blocked=4'); + }); + + it('maxTotalChargeUsd blocks a run once the execution budget is exhausted', async () => { + const result = await runScript(` + await apify.actor.start({ actorId: 'apify/hello-world', maxTotalChargeUsd: 5 }); + try { + await apify.actor.start({ actorId: 'apify/hello-world' }); + console.log('LEAK: second run started'); + } catch (e) { + console.log('blocked: ' + e.message); + } + `, { inputFields: { maxTotalChargeUsd: 5 } }); + expect(result.pushedItem?.stdout).toMatch(/^blocked: Blocked actor run: execution spending budget/); + }); + + it('defaultTimeoutSecs is applied to a run that does not specify its own timeoutSecs', async () => { + const result = await runScript(` + await apify.actor.start({ actorId: 'apify/hello-world' }); + `, { inputFields: { defaultTimeoutSecs: 42 } }); + const runCreateRequest = result.mockApi.requests.find((r) => r.method === 'POST' && r.path.includes('/acts/')); + expect(runCreateRequest?.path).toMatch(/timeout=42/); + }); + + it('actor.callAndGetItems does not double-count against maxActorRuns', async () => { + const result = await runScript(` + const { run, items } = await apify.actor.callAndGetItems({ actorId: 'apify/hello-world', limit: 5 }); + console.log('status=' + run.status + ' items=' + items.length); + `, { inputFields: { maxActorRuns: 1 } }); + expect(result.pushedItem?.stdout).toMatch(/^status=/); + expect(result.pushedItem?.stdout).not.toMatch(/Blocked actor run/); + }); +}); diff --git a/tests/unit/guard.test.ts b/tests/unit/guard.test.ts index 2a47d05..4fd20a9 100644 --- a/tests/unit/guard.test.ts +++ b/tests/unit/guard.test.ts @@ -166,10 +166,17 @@ describe('guardedFetch', () => { expect(secondInit.body).toBeUndefined(); }); - it('gives up after MAX_REDIRECT_HOPS redirects to allowed hosts', async () => { + it('gives up after exactly MAX_REDIRECT_HOPS redirects to allowed hosts', async () => { + // MAX_REDIRECT_HOPS is module-private (not exported — see guard.ts's own comment on + // why nothing beyond the pure allowlist helpers is), so this pins the boundary by its + // observable effect instead: guard.ts's `hop > MAX_REDIRECT_HOPS` check means calls at + // hop 0..5 each make a real fetch (6 calls, MAX_REDIRECT_HOPS=5 + the initial request) + // before hop 6 throws without calling fetch again. A change to MAX_REDIRECT_HOPS's + // value, or an off-by-one in the `>` check, changes this exact count. mockFetch.mockClear(); for (let i = 0; i < 10; i++) mockFetch.mockResolvedValueOnce(redirectResponse('https://apify.com/loop', 302)); - await expect(guard.guardedFetch('https://apify.com/start', undefined)).rejects.toThrow(/exceeded/); + await expect(guard.guardedFetch('https://apify.com/start', undefined)).rejects.toThrow(/exceeded 5 redirects/); + expect(mockFetch).toHaveBeenCalledTimes(6); }); it('returns a redirect response unchanged when it carries no Location header', async () => { diff --git a/tsconfig.integration.json b/tsconfig.integration.json new file mode 100644 index 0000000..2be4426 --- /dev/null +++ b/tsconfig.integration.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "moduleResolution": "bundler", + "lib": ["es2022"], + "strict": true, + "types": ["node"] + }, + "include": ["tests/integration/*.ts"] +} diff --git a/tsconfig.json b/tsconfig.json index 973d4b2..bf97d8d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,5 +6,10 @@ "lib": ["es2022", "dom"], "strict": true }, + // tests/integration/*.ts is genuine Node (spawns workerd, uses fs/child_process) and + // typechecks separately under tsconfig.integration.json instead: this program's other + // files model the workerd/no-nodejs_compat environment (see tests/globals.d.ts, which + // declares `process`/`require` as absent) — @types/node's real ambient `process`/`require` + // globals would collide with that if both lived in one program. "include": ["worker/*.ts", "tests/*.ts", "tests/unit/*.ts"] } diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..8fbf17c --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'vitest/config'; + +// Only the .ts sources under tests/unit/ — without this, `pnpm build`'s tsc output leaves a +// compiled tests/unit/*.js next to each *.ts (gitignored build byproduct, same as tests/*.js +// for the probe fixtures), and vitest's own default file discovery picks up both, silently +// running every unit test twice under two different module instances. +export default defineConfig({ + test: { + include: ['tests/unit/**/*.test.ts'], + }, +}); diff --git a/vitest.integration.config.ts b/vitest.integration.config.ts new file mode 100644 index 0000000..957c46e --- /dev/null +++ b/vitest.integration.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'vitest/config'; + +// Separate from vitest.config.ts (the fast, pure-function unit suite): these tests spawn a +// real workerd process per case, so they're slower and need worker/runner.js + worker/guard.js +// already compiled (`pnpm build` first — see package.json's test:integration script and +// .github/workflows/typecheck.yml's integration job). +export default defineConfig({ + test: { + include: ['tests/integration/**/*.test.ts'], + testTimeout: 15_000, + }, +}); diff --git a/worker/guard.ts b/worker/guard.ts index 17d42a5..691324d 100644 --- a/worker/guard.ts +++ b/worker/guard.ts @@ -82,7 +82,10 @@ export function nextRedirectInit(init: RequestInit | undefined, status: number): return { ...init, method: 'GET', body: undefined }; } -export async function guardedFetch(input: RequestInfo | URL, init: RequestInit | undefined, hop = 0): Promise { +// `hop` is an internal recursion counter, not part of the public contract — kept unexported +// so a caller (including escaped usercode.js, which can import and call any export of this +// module) can't pass a pre-inflated or negative value to defeat MAX_REDIRECT_HOPS. +async function guardedFetchHop(input: RequestInfo | URL, init: RequestInit | undefined, hop: number): Promise { if (hop > MAX_REDIRECT_HOPS) { throw new Error(`Blocked fetch: exceeded ${MAX_REDIRECT_HOPS} redirects`); } @@ -92,7 +95,11 @@ export async function guardedFetch(input: RequestInfo | URL, init: RequestInit | const location = response.headers.get('location'); if (!location) return response; // redirect status with no Location: nothing to follow const nextUrl = new URL(location, url); // resolves a relative Location against the current URL - return guardedFetch(nextUrl.href, nextRedirectInit(init, response.status), hop + 1); + return guardedFetchHop(nextUrl.href, nextRedirectInit(init, response.status), hop + 1); +} + +export function guardedFetch(input: RequestInfo | URL, init: RequestInit | undefined): Promise { + return guardedFetchHop(input, init, 0); } // writable:false + configurable:false, matching blockGlobal() below — a plain diff --git a/worker/runner.ts b/worker/runner.ts index 48729db..3932dec 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -21,6 +21,18 @@ // export-based handoff (this worker's previous design) could not be made // sound: usercode.js shares guard.js's module graph, so any function guard.js // exported was equally callable by escaped user code. +// +// guard.js is a side-effect import — nothing here binds a name from it — but it MUST be +// the first import in this file. Import declarations are hoisted and dependencies +// evaluate in the order first encountered; guard.js has to install its fetch/WebSocket/ +// EventSource overrides before usercode.js's module body runs, including any +// attacker-controlled top-level statement that escapes usercode.js's wrapper (see +// entrypoint.sh). Reversing this order (or dropping the import) silently makes the +// entire allowlist dead code — `globalThis.fetch` stays the real, unrestricted fetch for +// every script this Actor runs. tests/sandbox-isolation.ts is the regression test for +// this: it fails loudly (`allow https://example.com/` check reports NOT blocked) if this +// import is ever missing or reordered. +import './guard.js'; import { run } from './usercode.js'; type Fetcher = { fetch(input: RequestInfo | URL, init?: RequestInit): Promise }; @@ -45,6 +57,7 @@ interface ApifyRecord { interface RunRecord extends ApifyRecord { id: string; status: string; + defaultDatasetId: string; } type SearchParamValue = string | number | boolean | undefined | null; @@ -262,12 +275,21 @@ const REQUEST_ORIGIN_HEADER = 'X-Apify-Request-Origin'; // only bound THAT run — nothing previously bounded how many runs one script // could start, or their combined cost. See docs/API.md's "Execution limits". interface Limits { - maxActorRuns?: number; - maxTotalChargeUsd?: number; - defaultTimeoutSecs?: number; + // Always constructed with all three keys present (the fetch handler below never omits + // one) — `| undefined` documents "may be undefined" without implying a caller can leave + // the key out entirely, per this codebase's own `?` vs `| undefined` convention. + maxActorRuns: number | undefined; + maxTotalChargeUsd: number | undefined; + defaultTimeoutSecs: number | undefined; } -function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | undefined, internalFetch: Fetcher['fetch'], limits: Limits) { +function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: { + token: string; + apiV2: string; + parentOrigin: string | undefined; + internalFetch: Fetcher['fetch']; + limits: Limits; +}) { // Every request this Actor makes identifies itself; requests made while THIS // run's own origin is MCP additionally forward that origin so runs started by // apify.actor.start/call/callAndGetItems() below get meta.origin: 'MCP' too, @@ -329,7 +351,12 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // handed or guesses. Also used by abortTrackedRuns() (called from the top-level exception // handler below) to clean up runs still going when the script itself crashed. const startedRunIds = new Set(); - const TERMINAL_STATUSES = new Set(['SUCCEEDED', 'FAILED', 'ABORTED', 'ABORTING', 'TIMED-OUT']); + // Statuses after which abortTrackedRuns() (below) should NOT attempt to abort a run again. + // Deliberately broader than the API's own terminal-status set (docs/API.md's + // SUCCEEDED/FAILED/ABORTED/TIMED-OUT) by one: ABORTING means a run is already mid-abort + // (e.g. this script already called run.abort() on it), so re-aborting it would just be a + // redundant API call, not a real cleanup action. + const DONE_TRACKING_STATUSES = new Set(['SUCCEEDED', 'FAILED', 'ABORTED', 'ABORTING', 'TIMED-OUT']); const nonTerminalRunIds = new Set(); // Conservative execution-level cost cap: each run's OWN maxTotalChargeUsd is a ceiling, @@ -337,6 +364,11 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // limits.maxTotalChargeUsd and never lets a script authorize more combined ceiling than // that budget, even though actual spend will usually be lower. let committedChargeUsd = 0; + // Separate from startedRunIds.size: reserved synchronously (see createRun below) so two + // concurrent createRun() calls — e.g. docs/API.md's own "Bounded parallel fan-out" recipe, + // `Promise.all(batch.map(() => apify.actor.start(...)))` — can't both read the + // pre-reservation count before either's POST resolves and both pass the cap check. + let reservedRunCount = 0; // POST /acts/:id/runs, shared by actor.call() (start+wait, waitForFinishSecs defaults to // DEFAULT_WAIT_FOR_FINISH_SECS, capped at 60s per the Apify API — for longer runs use @@ -344,9 +376,9 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // the run record so the caller can read defaultDatasetId / defaultKeyValueStoreId. // Intentionally does NOT use /run-sync, which returns the OUTPUT KVS record (a pattern // only some Actors follow) rather than the structured run record. - const createRun = ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }: StartOptions): Promise => { - if (limits.maxActorRuns !== undefined && startedRunIds.size >= limits.maxActorRuns) { - throw new Error(`Blocked actor run: this script already started ${startedRunIds.size} Actor run(s), the configured limit is ${limits.maxActorRuns}`); + const createRun = async ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }: StartOptions): Promise => { + if (limits.maxActorRuns !== undefined && reservedRunCount >= limits.maxActorRuns) { + throw new Error(`Blocked actor run: this script already started/is starting ${reservedRunCount} Actor run(s), the configured limit is ${limits.maxActorRuns}`); } let effectiveMaxCharge = maxTotalChargeUsd; if (limits.maxTotalChargeUsd !== undefined) { @@ -358,21 +390,33 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // its own cap higher than what's left gets clamped down to what's left. effectiveMaxCharge = effectiveMaxCharge === undefined ? remaining : Math.min(effectiveMaxCharge, remaining); } - return apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { - searchParams: { - waitForFinish: waitForFinishSecs, - memory: memoryMbytes, - timeout: timeoutSecs ?? limits.defaultTimeoutSecs, - maxTotalChargeUsd: effectiveMaxCharge, - maxItems, - }, - body: input ?? {}, - }).then((runRecord: RunRecord) => { + // Reserve BEFORE the network round-trip below (nothing here awaits yet, so this runs + // to completion in one synchronous tick relative to any other createRun() call — see + // the comment on reservedRunCount above for why that matters). + reservedRunCount += 1; + if (effectiveMaxCharge !== undefined) committedChargeUsd += effectiveMaxCharge; + try { + const runRecord: RunRecord = await apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { + searchParams: { + waitForFinish: waitForFinishSecs, + memory: memoryMbytes, + timeout: timeoutSecs ?? limits.defaultTimeoutSecs, + maxTotalChargeUsd: effectiveMaxCharge, + maxItems, + }, + body: input ?? {}, + }); startedRunIds.add(runRecord.id); - if (!TERMINAL_STATUSES.has(runRecord.status)) nonTerminalRunIds.add(runRecord.id); - if (effectiveMaxCharge !== undefined) committedChargeUsd += effectiveMaxCharge; + if (!DONE_TRACKING_STATUSES.has(runRecord.status)) nonTerminalRunIds.add(runRecord.id); return runRecord; - }); + } catch (err) { + // The reservation never became a real run — release it, so a failed attempt + // (bad actorId, network error, ...) doesn't permanently eat into the script's + // run-count/budget allowance. + reservedRunCount -= 1; + if (effectiveMaxCharge !== undefined) committedChargeUsd -= effectiveMaxCharge; + throw err; + } }; const actor = { @@ -398,7 +442,7 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u callAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { const runRecord = await createRun({ actorId, input, waitForFinishSecs: DEFAULT_WAIT_FOR_FINISH_SECS, ...runOpts }); const { items } = await dataset.listItems({ - datasetId: runRecord.defaultDatasetId as string, fields, limit, + datasetId: runRecord.defaultDatasetId, fields, limit, }); return { run: runRecord, items }; }, @@ -414,7 +458,7 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u const runRecord: RunRecord = await apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { searchParams: { waitForFinish: waitForFinishSecs }, }); - if (TERMINAL_STATUSES.has(runRecord.status)) nonTerminalRunIds.delete(runId); + if (DONE_TRACKING_STATUSES.has(runRecord.status)) nonTerminalRunIds.delete(runId); return runRecord; }, @@ -597,7 +641,13 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u export type ApifyBinding = ReturnType['binding']; // Push the captured streams as a single item to the run's default dataset. -async function pushOutput(apiV2: string, token: string, internalFetch: Fetcher['fetch'], env: Env, item: OutputItem): Promise { +async function pushOutput({ apiV2, token, internalFetch, env, item }: { + apiV2: string; + token: string; + internalFetch: Fetcher['fetch']; + env: Env; + item: OutputItem; +}): Promise { const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; if (!datasetId) throw new Error('Default dataset ID missing from Actor run environment.'); const response = await internalFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { @@ -609,16 +659,22 @@ async function pushOutput(apiV2: string, token: string, internalFetch: Fetcher[' } // Parses an optional positive-number env var (as set by entrypoint.sh from Actor input). -// Absent, blank, non-numeric, or non-positive all mean "no limit configured" — this is a -// human-edited-adjacent path (Actor input -> env var), not a write-time-validated one, so a -// malformed value fails open to "no limit" rather than crashing the run. +// Absent or blank means "field omitted" -> no limit, matching .actor/actor.json's own +// description for each field. The platform validates the input schema's types/minimums +// before this code ever runs, so non-numeric/non-positive shouldn't reach here in +// practice — treating it as "no limit" rather than crashing is a deliberate fallback for +// that already-unlikely case, not a substitute for the schema validation. function parsePositiveNumberEnv(value: string | undefined): number | undefined { if (!value) return undefined; const parsed = Number(value); return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; } -export default { +// Frozen so escaped usercode.js module-scope code (which shares this module's namespace via +// `import('./runner.js')`, the same reachability every export in this file has — see guard.ts's +// header comment) can't reassign `.fetch` to a wrapper that captures the real `request`/`env` +// (APIFY_TOKEN, INTERNAL_API) the next time workerd genuinely dispatches to this worker. +export default Object.freeze({ async fetch(request: Request, env: Env): Promise { const url = new URL(request.url); if (url.pathname === '/health') return new Response('ok'); @@ -646,7 +702,7 @@ export default { info: (...args: unknown[]) => stdout.push(args.map(stringify).join(' ')), }); - const { binding, abortTrackedRuns } = makeApifyBinding(token, apiV2, env.PARENT_ORIGIN, internalFetch, limits); + const { binding, abortTrackedRuns } = makeApifyBinding({ token, apiV2, parentOrigin: env.PARENT_ORIGIN, internalFetch, limits }); // A thrown program is a user-level failure: capture it in stderr and still // push the output, so the run SUCCEEDS with diagnostics. Infra failures @@ -677,12 +733,10 @@ export default { } } - await pushOutput(apiV2, token, internalFetch, env, { - stdout: stdout.join('\n'), - stderr: stderr.join('\n'), - exitCode, - statusMessage, + await pushOutput({ + apiV2, token, internalFetch, env, + item: { stdout: stdout.join('\n'), stderr: stderr.join('\n'), exitCode, statusMessage }, }); return Response.json({ ok: true }); }, -}; +}); From 11cae8a2fb4ff34f107974f77b4753503ea4d462 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 4 Aug 2026 11:46:48 +0200 Subject: [PATCH 41/46] fix: close global-URL-hijack SSRF bypass, NaN budget corruption, harness bugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scout-loop iteration 2 (4 fresh reviewers) found real bugs in the previous round's fixes: - worker/guard.ts: validateUrl called the bare `new URL(...)`, resolving whatever globalThis.URL currently is. A plain script (no module-scope escape needed) could do `globalThis.URL = class extends URL { get hostname() { return 'apify.com'; } }` to make the allowlist check believe a disallowed host was apify.com, while the actual fetch still went to the real target string. Fixed by capturing the real URL constructor (RealURL) before usercode.js can ever run, same pattern as realFetch. Verified live against workerd, plus a new unit test regression case. - worker/runner.ts's createRun(): a script-supplied maxTotalChargeUsd of NaN (or another non-finite/non-positive value) flowed straight into committedChargeUsd's arithmetic. NaN is absorbing (NaN - x and x - NaN both stay NaN), so a single bad call permanently corrupted the running total and defeated the whole execution-level budget check for the rest of the script — the catch block's rollback can't recover it either (subtracting NaN from NaN is still NaN). Now validated (finite, > 0) before it touches any shared state. Verified live against workerd. - worker/runner.ts: runner.ts's own Object.freeze calls run (via import order) AFTER usercode.js's module body, so escaped top-level code could shadow the global Object.freeze to a no-op before any of them ever fire — defeating the 'frozen so the script can't reassign this' guarantee on the apify binding/console/default export. Fixed by exporting a pre-captured realObjectFreeze from guard.ts (same capture-before-usercode-runs pattern), used throughout runner.ts instead of the bare global. - tests/integration/harness.ts: the mock API's route matching used `req.url.endsWith('/runs')`, which breaks the moment a real request carries a query string — createRun() always attaches one (waitForFinish/timeout/memory/ maxTotalChargeUsd), so requests silently fell through to the wrong mock response branch, masking real behavior in several 'passing' tests. Fixed to match on pathname. Also: removed a test that quietly made a live network call to https://apify.com (flaky, contradicted this suite's own 'offline' claim — the allow-path is already covered offline by the unit suite); added a new test proving createRun()'s rollback actually fires on a real API rejection (via a new failNextRunCreate() hook on the mock); every acquired resource (mock server, temp dir, workerd child process) now released via try/finally on every path, including waiting for the child to actually exit before returning. - worker/runner.ts, tests/binding-smoke.ts: fixed a stale comment left after the previous round's TERMINAL_STATUSES -> DONE_TRACKING_STATUSES rename, and corrected DONE_TRACKING_STATUSES's own comment to name its real call sites (createRun/ waitForFinish, not abortTrackedRuns). - tsconfig.integration.json now extends tsconfig.json instead of duplicating its compiler options. - .actor/actor.json: dropped a restated numeric-floor sentence from maxTotalChargeUsd's description that broke the sibling fields' pattern of trusting the JSON Schema constraint alone. --- .actor/actor.json | 2 +- tests/binding-smoke.ts | 2 +- tests/integration/harness.ts | 140 +++++++++++++++++++------- tests/integration/workerd-e2e.test.ts | 51 ++++++---- tests/unit/guard.test.ts | 27 ++++- tsconfig.integration.json | 5 +- worker/guard.ts | 30 +++++- worker/runner.ts | 58 +++++++---- 8 files changed, 228 insertions(+), 87 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 461c3dc..b8e3691 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -31,7 +31,7 @@ "maxTotalChargeUsd": { "title": "Max total charge (USD)", "type": "number", - "description": "Execution-wide spending budget across every Actor run this script starts — distinct from a single call's own maxTotalChargeUsd, which only caps that one run. Each run's own cap is clamped so the combined total never exceeds this budget; starting a run once it's exhausted throws inside the script. Omit for no limit. Must be greater than 0 — to allow zero Actor runs, omit the code's actor.start/call/callAndGetItems calls instead.", + "description": "Execution-wide spending budget across every Actor run this script starts — distinct from a single call's own maxTotalChargeUsd, which only caps that one run. Each run's own cap is clamped so the combined total never exceeds this budget; starting a run once it's exhausted throws inside the script. Omit for no limit.", "exclusiveMinimum": 0 }, "defaultTimeoutSecs": { diff --git a/tests/binding-smoke.ts b/tests/binding-smoke.ts index a805010..417ab60 100644 --- a/tests/binding-smoke.ts +++ b/tests/binding-smoke.ts @@ -26,7 +26,7 @@ const ACTOR = 'apify/hello-world'; const [ACTOR_USERNAME, ACTOR_NAME] = ACTOR.split('/'); // Every status the Apify API can return for a run. Used to check a returned status is a -// real value, not just any truthy string -- mirrors TERMINAL_STATUSES in worker/runner.ts. +// real value, not just any truthy string -- mirrors DONE_TRACKING_STATUSES in worker/runner.ts. const RUN_STATUSES = new Set(['READY', 'RUNNING', 'SUCCEEDED', 'FAILED', 'ABORTING', 'ABORTED', 'TIMING-OUT', 'TIMED-OUT']); // ---- actor (read) ---- diff --git a/tests/integration/harness.ts b/tests/integration/harness.ts index 445323f..6c76683 100644 --- a/tests/integration/harness.ts +++ b/tests/integration/harness.ts @@ -7,13 +7,14 @@ // step in .github/workflows/typecheck.yml's integration job — see there). import { spawn, type ChildProcess } from 'node:child_process'; import { createServer, type Server } from 'node:http'; -import { mkdtempSync, writeFileSync, readFileSync, cpSync } from 'node:fs'; +import { mkdtempSync, writeFileSync, readFileSync, cpSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { createRequire } from 'node:module'; const WORKERD_STARTUP_TIMEOUT_MS = 5_000; const WORKERD_STARTUP_POLL_INTERVAL_MS = 100; +const WORKERD_EXIT_TIMEOUT_MS = 2_000; function workerdBinaryPath(): string { // Same resolution Dockerfile's builder stage uses: workerd ships its binary path via the @@ -25,6 +26,19 @@ function workerdBinaryPath(): string { return require('workerd').default; } +// Binds an OS-assigned ephemeral port on a throwaway listener, then releases it — the same +// "ask the OS for a free one" trick startMockApi() uses for its own port, reused here so +// workerd's own port isn't picked by guessing a range (see the historical note this replaces: +// a literal `10_000 + Math.random() * 10_000` had no collision retry). +async function reserveEphemeralPort(): Promise { + const probe = createServer(); + await new Promise((resolve) => probe.listen(0, '127.0.0.1', resolve)); + const address = probe.address(); + if (!address || typeof address === 'string') throw new Error('failed to reserve an ephemeral port'); + await new Promise((resolve) => probe.close(() => resolve())); + return address.port; +} + // A minimal stand-in for the platform-internal Apify API — just enough to make // actor.start/dataset operations/pushOutput resolve, so a script's real behavior (including // safeguard rejections, which happen before any HTTP call) is observable end to end. @@ -32,25 +46,46 @@ export interface MockApi { server: Server; port: number; requests: { method: string; path: string; body: string }[]; + /** + * Makes the NEXT `POST .../acts/:id/runs` request fail with the given status/body instead + * of succeeding — one-shot, cleared after it fires. Lets a test prove createRun()'s + * reservation rollback actually releases the run-count/budget slot on a real API + * rejection, not just on the happy path. + */ + failNextRunCreate: (status: number, body: string) => void; close: () => Promise; } export async function startMockApi(): Promise { const requests: MockApi['requests'] = []; + let pendingRunCreateFailure: { status: number; body: string } | null = null; const server = createServer((req, res) => { const chunks: Buffer[] = []; - req.on('data', (chunk) => chunks.push(chunk)); + req.on('data', (chunk: Buffer) => chunks.push(chunk)); req.on('end', () => { const body = Buffer.concat(chunks).toString('utf8'); + // req.url is path+query (no scheme/host); parse against a throwaway base so + // route matching is on the PATH alone — matching the raw string (e.g. with + // `.endsWith('/runs')`) breaks the moment a real request carries a query string + // (createRun() always attaches one: waitForFinish/timeout/memory/maxTotalChargeUsd), + // silently falling through to the wrong response branch below. + const pathname = new URL(req.url ?? '/', 'http://mock-api.internal').pathname; requests.push({ method: req.method ?? '', path: req.url ?? '', body }); res.setHeader('content-type', 'application/json'); - if (req.method === 'POST' && req.url?.includes('/datasets/') && req.url.endsWith('/items')) { + if (req.method === 'POST' && pathname.startsWith('/v2/datasets/') && pathname.endsWith('/items')) { res.writeHead(201); res.end('{}'); - } else if (req.method === 'POST' && req.url?.includes('/acts/') && req.url.endsWith('/runs')) { + } else if (req.method === 'POST' && pathname.startsWith('/v2/acts/') && pathname.endsWith('/runs')) { + if (pendingRunCreateFailure) { + const { status, body: failureBody } = pendingRunCreateFailure; + pendingRunCreateFailure = null; + res.writeHead(status); + res.end(failureBody); + return; + } res.writeHead(201); res.end(JSON.stringify({ data: { id: `run-${requests.length}`, status: 'READY', defaultDatasetId: 'ds1' } })); - } else if (req.method === 'GET' && req.url?.includes('/datasets/') && req.url.includes('/items')) { + } else if (req.method === 'GET' && pathname.startsWith('/v2/datasets/') && pathname.endsWith('/items')) { res.writeHead(200); res.end('[]'); } else { @@ -66,6 +101,7 @@ export async function startMockApi(): Promise { server, port: address.port, requests, + failNextRunCreate: (status, body) => { pendingRunCreateFailure = { status, body }; }, close: () => new Promise((resolve) => server.close(() => resolve())), }; } @@ -73,10 +109,12 @@ export async function startMockApi(): Promise { export interface RunOptions { /** Actor input fields beyond `code`, e.g. { maxActorRuns: 1 } — mirrors what entrypoint.sh reads. */ inputFields?: Record; + /** Called with the MockApi after it's started but before workerd boots — e.g. to arm failNextRunCreate(). */ + beforeStart?: (mockApi: MockApi) => void; } export interface RunResult { - /** The pushed dataset item, if pushOutput ran (empty object if the worker crashed at startup). */ + /** The pushed dataset item, or null if pushOutput never ran (e.g. workerd crashed at startup). */ pushedItem: Record | null; /** True if workerd itself started and served /health before we tore it down. */ startedCleanly: boolean; @@ -87,18 +125,33 @@ export interface RunResult { // Boots a fresh workerd instance with `code` wrapped exactly like entrypoint.sh does, against a // fresh MockApi standing in for the internal Apify API, sends one /run request, and tears both -// down. Mirrors entrypoint.sh's own env var wiring (CODE_RUNTIME_* for the execution limits) -// rather than reinventing a second convention. +// down — every acquired resource (mock server, workerd process, temp dir) is released on every +// path, including when workerd never becomes healthy or a step above throws. Mirrors +// entrypoint.sh's own env var wiring (CODE_RUNTIME_* for the execution limits) rather than +// reinventing a second convention. export async function runScript(code: string, options: RunOptions = {}): Promise { const mockApi = await startMockApi(); - const workDir = mkdtempSync(join(tmpdir(), 'code-runtime-it-')); + try { + options.beforeStart?.(mockApi); + const workDir = mkdtempSync(join(tmpdir(), 'code-runtime-it-')); + try { + return await runInWorkDir(code, options, mockApi, workDir); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + } finally { + await mockApi.close(); + } +} + +async function runInWorkDir(code: string, options: RunOptions, mockApi: MockApi, workDir: string): Promise { const repoRoot = join(import.meta.dirname, '..', '..'); cpSync(join(repoRoot, 'worker', 'runner.js'), join(workDir, 'runner.js')); cpSync(join(repoRoot, 'worker', 'guard.js'), join(workDir, 'guard.js')); writeFileSync(join(workDir, 'usercode.js'), `export async function run(apify, console) {\n${code}\n}\n`); - const port = 10_000 + Math.floor(Math.random() * 10_000); + const port = await reserveEphemeralPort(); // config.capnp's __PORT__ placeholder appears twice (once in a comment, once in the real // socket address) — replaceAll, not replace, or the comment's occurrence "wins" and the // real one is left as the literal string "__PORT__" (workerd then fails DNS-resolving it @@ -120,34 +173,47 @@ export async function runScript(code: string, options: RunOptions = {}): Promise }, stdio: ['ignore', 'pipe', 'pipe'], }); - let stderr = ''; - child.stderr?.on('data', (chunk) => { stderr += chunk.toString(); }); + try { + let stderr = ''; + child.stderr?.on('data', (chunk: Buffer) => { stderr += chunk.toString(); }); - const deadline = Date.now() + WORKERD_STARTUP_TIMEOUT_MS; - let startedCleanly = false; - while (Date.now() < deadline) { - try { - const res = await fetch(`http://127.0.0.1:${port}/health`); - if (res.ok) { startedCleanly = true; break; } - } catch { /* not up yet, or crashed — keep polling until the deadline */ } - if (child.exitCode !== null) break; // crashed at startup, no point polling further - await new Promise((resolve) => setTimeout(resolve, WORKERD_STARTUP_POLL_INTERVAL_MS)); - } + const deadline = Date.now() + WORKERD_STARTUP_TIMEOUT_MS; + let startedCleanly = false; + while (Date.now() < deadline) { + try { + const res = await fetch(`http://127.0.0.1:${port}/health`); + if (res.ok) { startedCleanly = true; break; } + } catch { /* not up yet, or crashed — keep polling until the deadline */ } + if (child.exitCode !== null) break; // crashed at startup, no point polling further + await new Promise((resolve) => setTimeout(resolve, WORKERD_STARTUP_POLL_INTERVAL_MS)); + } - if (startedCleanly) { - try { - await fetch(`http://127.0.0.1:${port}/run`, { method: 'POST' }); - } catch { /* the /run call itself may crash the worker — that's a result to assert on, not a harness failure */ } - } + if (startedCleanly) { + try { + await fetch(`http://127.0.0.1:${port}/run`, { method: 'POST' }); + } catch { /* the /run call itself may crash the worker — that's a result to assert on, not a harness failure */ } + } - child.kill(); - await mockApi.close(); - - const pushRequest = mockApi.requests.find((r) => r.method === 'POST' && r.path.endsWith('/items') && r.path.includes('ds-default')); - return { - pushedItem: pushRequest ? JSON.parse(pushRequest.body) : null, - startedCleanly, - stderr, - mockApi, - }; + const pushRequest = mockApi.requests.find((r) => { + const pathname = new URL(r.path, 'http://mock-api.internal').pathname; + return r.method === 'POST' && pathname === '/v2/datasets/ds-default/items'; + }); + return { + pushedItem: pushRequest ? JSON.parse(pushRequest.body) : null, + startedCleanly, + stderr, + mockApi, + }; + } finally { + child.kill(); + // Wait for the process to actually exit (with a SIGKILL escalation) before returning — + // otherwise a workerd process slow to honor SIGTERM can still be running (and holding + // its port) when the next test in this file starts spawning its own. + if (child.exitCode === null) { + await Promise.race([ + new Promise((resolve) => child.once('exit', () => resolve())), + new Promise((resolve) => setTimeout(() => { child.kill('SIGKILL'); resolve(); }, WORKERD_EXIT_TIMEOUT_MS)), + ]); + } + } } diff --git a/tests/integration/workerd-e2e.test.ts b/tests/integration/workerd-e2e.test.ts index 3c2286a..0081ff7 100644 --- a/tests/integration/workerd-e2e.test.ts +++ b/tests/integration/workerd-e2e.test.ts @@ -27,24 +27,13 @@ describe('guard.js is actually enforced (not just correct in isolation)', () => expect(result.pushedItem?.stdout).not.toMatch(/LEAK/); }); - it('allows a fetch to an apify.com host', async () => { - // The Actor's own internal-API mock isn't apify.com, so this only proves the guard's - // ALLOW path is reachable (doesn't hang/throw before even trying) — the mock returns a - // real HTTP response for any host once workerd's own DNS/connect succeeds, which for - // a host that doesn't resolve (apify.com does, publicly) would time out instead of - // throwing a "Blocked fetch" error. Asserting the ABSENCE of the guard's own blocked- - // fetch error message is what distinguishes "guard let it through" from "guard blocked - // it" here, without depending on live internet access from the test runner. - const result = await runScript(` - try { - await fetch('https://apify.com/'); - console.log('reached apify.com (no guard rejection)'); - } catch (e) { - console.log('error: ' + e.message); - } - `); - expect(result.pushedItem?.stdout).not.toMatch(/Blocked fetch/); - }); + // The allow-path (a request TO apify.com actually going through) is deliberately NOT + // covered here: it would require either live internet access from the test runner (flaky, + // and not actually offline despite this suite's other claims) or mocking DNS/TLS for a real + // host, neither of which this harness does. tests/unit/guard.test.ts's "performs the + // request when the URL is allowed" case covers that path fully offline, against a mocked + // fetch — this file only needs to prove the DISALLOW path is wired into the real worker + // (see the previous test), which is the part a unit test can't reach. it('blocks WebSocket construction', async () => { const result = await runScript(` @@ -122,6 +111,32 @@ describe('execution-limit safeguards fire under real (including concurrent) use' expect(runCreateRequest?.path).toMatch(/timeout=42/); }); + it('a rejected actor.start() releases its reservation (rollback actually fires)', async () => { + // Regression test for createRun()'s try/catch rollback: without it, a run that fails + // AFTER being synchronously reserved (bad actorId, API rejection, ...) would + // permanently eat into maxActorRuns's budget for a run that never actually started. + const result = await runScript(` + let firstFailed = false; + try { + await apify.actor.start({ actorId: 'apify/hello-world' }); + } catch (e) { + firstFailed = true; + } + // If the failed attempt above wasn't rolled back, this would be blocked too + // (maxActorRuns: 1 already "spent" by the failed one). + let secondSucceeded = false; + try { + await apify.actor.start({ actorId: 'apify/hello-world' }); + secondSucceeded = true; + } catch (e) { /* would mean rollback didn't happen */ } + console.log('firstFailed=' + firstFailed + ' secondSucceeded=' + secondSucceeded); + `, { + inputFields: { maxActorRuns: 1 }, + beforeStart: (mockApi) => mockApi.failNextRunCreate(400, JSON.stringify({ error: { message: 'bad actorId' } })), + }); + expect(result.pushedItem?.stdout).toBe('firstFailed=true secondSucceeded=true'); + }); + it('actor.callAndGetItems does not double-count against maxActorRuns', async () => { const result = await runScript(` const { run, items } = await apify.actor.callAndGetItems({ actorId: 'apify/hello-world', limit: 5 }); diff --git a/tests/unit/guard.test.ts b/tests/unit/guard.test.ts index 4fd20a9..edf1c61 100644 --- a/tests/unit/guard.test.ts +++ b/tests/unit/guard.test.ts @@ -191,10 +191,33 @@ describe('module exports', () => { it('never exports a raw/unrestricted fetch capability', () => { // Regression guard for PR #1's finding: guard.js must never export anything that // hands the caller an unwrapped fetch function or a way to bypass the allowlist. - // Every export must be one of these known-safe, pure helpers. - const knownSafeExports = new Set(['isAllowedHost', 'validateUrl', 'nextRedirectInit', 'guardedFetch']); + // Every export must be one of these known-safe, pure helpers (realObjectFreeze is + // safe by the same reasoning: it's still just Object.freeze, the concept carries no + // capability — see its own comment in guard.ts for why it needs to be exported at all). + const knownSafeExports = new Set(['isAllowedHost', 'validateUrl', 'nextRedirectInit', 'guardedFetch', 'realObjectFreeze']); for (const key of Object.keys(guard)) { expect(knownSafeExports.has(key)).toBe(true); } }); }); + +describe('validateUrl resists a hijacked global URL constructor', () => { + // Regression test for a real bypass found in review: validateUrl used to call the bare + // `new URL(...)`, which resolves whatever `globalThis.URL` currently is. A script that + // replaces it with a lying implementation (real .href, faked .hostname) could make + // validateUrl believe a disallowed host was apify.com, with no module-scope-escape trick + // needed at all — see guard.ts's RealURL comment for the fix (capture URL before + // usercode.js can ever run). + it('still rejects a disallowed host after globalThis.URL is replaced with a lying constructor', () => { + const OriginalURL = globalThis.URL; + class LyingURL extends OriginalURL { + get hostname() { return 'apify.com'; } + } + globalThis.URL = LyingURL; + try { + expect(() => guard.validateUrl('http://example.com/')).toThrow(/only apify\.com/); + } finally { + globalThis.URL = OriginalURL; + } + }); +}); diff --git a/tsconfig.integration.json b/tsconfig.integration.json index 2be4426..d566980 100644 --- a/tsconfig.integration.json +++ b/tsconfig.integration.json @@ -1,10 +1,7 @@ { + "extends": "./tsconfig.json", "compilerOptions": { - "target": "es2022", - "module": "es2022", - "moduleResolution": "bundler", "lib": ["es2022"], - "strict": true, "types": ["node"] }, "include": ["tests/integration/*.ts"] diff --git a/worker/guard.ts b/worker/guard.ts index 691324d..fe174ea 100644 --- a/worker/guard.ts +++ b/worker/guard.ts @@ -31,6 +31,27 @@ // no longer needs to capture or export a privileged fetch at all. const realFetch = globalThis.fetch.bind(globalThis); +// Captured before usercode.js's module body ever runs (see runner.ts's import-order +// comment — guard.js's own top-level code, including this line, always finishes first). +// `new URL(...)` and `instanceof URL` below resolve the identifier `URL` from scope, which +// is just `globalThis.URL` — an ordinary, reassignable global. A script that replaces it +// (`globalThis.URL = class FakeURL extends URL { get hostname() { return 'apify.com'; } }`) +// would make every validation call below trust a lying parser while the *string* actually +// handed to realFetch is untouched, defeating the allowlist with no module-scope-escape +// trick needed at all — this file's own `Object.defineProperty(globalThis, 'fetch', ...)` +// below is exactly this same "capture the real one before a script can swap it" pattern, +// applied here to the other builtin this file's security check depends on. +const RealURL = globalThis.URL; + +// Same capture-before-usercode-runs reasoning, for a different consumer: runner.ts's own +// `Object.freeze(...)` calls (on the `apify` binding, `console`, and its default export) are +// top-level code that runs AFTER usercode.js's module body (see runner.ts's import-order +// comment), so escaped top-level code could reassign the global `Object.freeze` to a no-op +// before runner.ts ever calls it — silently defeating every "frozen so the script can't +// reassign this" guarantee there. Exporting a pre-captured reference is safe (it's still +// just Object.freeze; the concept itself carries no capability) and closes that gap. +export const realObjectFreeze: typeof Object.freeze = Object.freeze.bind(Object); + // Match apify.com exactly or any subdomain. The leading dot in the suffix is // what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` // (ends with `.evil.com`) both fail. @@ -41,7 +62,7 @@ export function isAllowedHost(hostname: string): boolean { function requestUrl(input: RequestInfo | URL): string { if (typeof input === 'string') return input; - if (input instanceof URL) return input.href; + if (input instanceof RealURL) return input.href; if (input && typeof input.url === 'string') return input.url; // Request return String(input); } @@ -52,8 +73,9 @@ export function validateUrl(input: RequestInfo | URL): URL { let url: URL; try { // Parse to the real host — defeats userinfo (`apify.com@evil.com`), - // path/query/fragment (`evil.com/apify.com`) and similar tricks. - url = new URL(requestUrl(input)); + // path/query/fragment (`evil.com/apify.com`) and similar tricks. Uses RealURL + // (see above), not the bare global, so a hijacked globalThis.URL can't lie here. + url = new RealURL(requestUrl(input)); } catch { throw new Error('Blocked fetch: only absolute http(s) URLs to apify.com are allowed'); } @@ -94,7 +116,7 @@ async function guardedFetchHop(input: RequestInfo | URL, init: RequestInit | und if (!REDIRECT_STATUSES.has(response.status)) return response; const location = response.headers.get('location'); if (!location) return response; // redirect status with no Location: nothing to follow - const nextUrl = new URL(location, url); // resolves a relative Location against the current URL + const nextUrl = new RealURL(location, url); // resolves a relative Location against the current URL return guardedFetchHop(nextUrl.href, nextRedirectInit(init, response.status), hop + 1); } diff --git a/worker/runner.ts b/worker/runner.ts index 3932dec..8c7e589 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -22,17 +22,23 @@ // sound: usercode.js shares guard.js's module graph, so any function guard.js // exported was equally callable by escaped user code. // -// guard.js is a side-effect import — nothing here binds a name from it — but it MUST be -// the first import in this file. Import declarations are hoisted and dependencies -// evaluate in the order first encountered; guard.js has to install its fetch/WebSocket/ -// EventSource overrides before usercode.js's module body runs, including any -// attacker-controlled top-level statement that escapes usercode.js's wrapper (see -// entrypoint.sh). Reversing this order (or dropping the import) silently makes the -// entire allowlist dead code — `globalThis.fetch` stays the real, unrestricted fetch for -// every script this Actor runs. tests/sandbox-isolation.ts is the regression test for -// this: it fails loudly (`allow https://example.com/` check reports NOT blocked) if this -// import is ever missing or reordered. -import './guard.js'; +// guard.js MUST be the first import in this file, whether or not a name is bound from it. +// Import declarations are hoisted and dependencies evaluate in the order first +// encountered; guard.js has to install its fetch/WebSocket/EventSource overrides (and +// capture RealURL/realObjectFreeze — see guard.ts) before usercode.js's module body runs, +// including any attacker-controlled top-level statement that escapes usercode.js's +// wrapper (see entrypoint.sh). Reversing this order (or dropping the import) silently +// makes the entire allowlist dead code — `globalThis.fetch` stays the real, unrestricted +// fetch for every script this Actor runs. tests/sandbox-isolation.ts is the regression +// test for this: it fails loudly (`allow https://example.com/` check reports NOT blocked) +// if this import is ever missing or reordered. +// +// realObjectFreeze (used below instead of the bare global `Object.freeze`) is guard.js's +// own pre-captured reference, for the same reason: this file's Object.freeze calls are +// top-level code that runs AFTER usercode.js's module body, so a script could otherwise +// shadow the global `Object.freeze` to a no-op before any of them ever run. See guard.ts's +// comment on realObjectFreeze. +import { realObjectFreeze } from './guard.js'; import { run } from './usercode.js'; type Fetcher = { fetch(input: RequestInfo | URL, init?: RequestInit): Promise }; @@ -351,8 +357,11 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: // handed or guesses. Also used by abortTrackedRuns() (called from the top-level exception // handler below) to clean up runs still going when the script itself crashed. const startedRunIds = new Set(); - // Statuses after which abortTrackedRuns() (below) should NOT attempt to abort a run again. - // Deliberately broader than the API's own terminal-status set (docs/API.md's + // Gates nonTerminalRunIds membership: createRun() (below) adds a run's id here unless its + // status is already one of these; waitForFinish() removes it once the status becomes one + // of these. A status in this set means this script no longer needs to track (and + // therefore abortTrackedRuns(), below, no longer needs to abort) that run. Deliberately + // broader than the API's own terminal-status set (docs/API.md's // SUCCEEDED/FAILED/ABORTED/TIMED-OUT) by one: ABORTING means a run is already mid-abort // (e.g. this script already called run.abort() on it), so re-aborting it would just be a // redundant API call, not a real cleanup action. @@ -380,6 +389,15 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: if (limits.maxActorRuns !== undefined && reservedRunCount >= limits.maxActorRuns) { throw new Error(`Blocked actor run: this script already started/is starting ${reservedRunCount} Actor run(s), the configured limit is ${limits.maxActorRuns}`); } + // Reject a malformed script-supplied maxTotalChargeUsd before it ever reaches + // committedChargeUsd's arithmetic below: NaN in particular is silently absorbing — + // `NaN - anything` and `anything - NaN` both stay NaN, so a single bad call would + // permanently corrupt the running total and (since `NaN <= 0` is false) defeat the + // whole execution-level budget check for the rest of the script, with no way to + // recover it via the catch block's rollback (subtracting NaN from NaN is still NaN). + if (maxTotalChargeUsd !== undefined && !(Number.isFinite(maxTotalChargeUsd) && maxTotalChargeUsd > 0)) { + throw new Error(`Invalid maxTotalChargeUsd: ${maxTotalChargeUsd} (must be a finite number greater than 0)`); + } let effectiveMaxCharge = maxTotalChargeUsd; if (limits.maxTotalChargeUsd !== undefined) { const remaining = limits.maxTotalChargeUsd - committedChargeUsd; @@ -625,12 +643,12 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: // Freeze every namespace (and the wrapper) so the script can't reassign a method to // corrupt its own behavior or, for `console` below, its own output capture. - const binding = Object.freeze({ - actor: Object.freeze(actor), + const binding = realObjectFreeze({ + actor: realObjectFreeze(actor), store, - run: Object.freeze(run), - dataset: Object.freeze(dataset), - keyValueStore: Object.freeze(keyValueStore), + run: realObjectFreeze(run), + dataset: realObjectFreeze(dataset), + keyValueStore: realObjectFreeze(keyValueStore), }); return { binding, abortTrackedRuns }; } @@ -674,7 +692,7 @@ function parsePositiveNumberEnv(value: string | undefined): number | undefined { // `import('./runner.js')`, the same reachability every export in this file has — see guard.ts's // header comment) can't reassign `.fetch` to a wrapper that captures the real `request`/`env` // (APIFY_TOKEN, INTERNAL_API) the next time workerd genuinely dispatches to this worker. -export default Object.freeze({ +export default realObjectFreeze({ async fetch(request: Request, env: Env): Promise { const url = new URL(request.url); if (url.pathname === '/health') return new Response('ok'); @@ -695,7 +713,7 @@ export default Object.freeze({ const stdout: string[] = []; const stderr: string[] = []; // Frozen so the script can't reassign e.g. console.log to corrupt its own capture. - const captureConsole: ConsoleLike = Object.freeze({ + const captureConsole: ConsoleLike = realObjectFreeze({ log: (...args: unknown[]) => stdout.push(args.map(stringify).join(' ')), error: (...args: unknown[]) => stderr.push(args.map(stringify).join(' ')), warn: (...args: unknown[]) => stderr.push(args.map(stringify).join(' ')), From a491443c433245059942af45087b7a172c00ded5 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 4 Aug 2026 12:16:45 +0200 Subject: [PATCH 42/46] fix: harden every security check against builtin prototype/static-method poisoning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scout-loop iteration 3 (4 fresh reviewers, all converging independently on the same root cause): rounds 1-2 fixed *reassignment* of security-relevant globals (globalThis.URL, globalThis.Object.freeze) by capturing a reference before usercode.js can run. That does NOT protect a shared PROTOTYPE method or static function — a captured constructor's .prototype IS the same mutable object as the live global's .prototype, and a captured static function reference only helps if it's captured before usercode.js runs, which several call sites weren't. Three real, empirically-verified bypasses, no module-scope escape needed at all — ordinary script-body code: - worker/guard.ts's isAllowedHost called hostname.toLowerCase()/.replace()/.endsWith() directly. `String.prototype.endsWith = () => true;` in a normal script defeated the entire *.apify.com allowlist for every fetch() call. Verified live: fetch to http://example.com/ succeeded. - worker/runner.ts's run.abort ownership check did `startedRunIds.has(runId)`. `Set.prototype.has = () => true;` let a script abort ANY run, not just ones it started — a real capability escalation, not just an SSRF variant. Also affected DONE_TRACKING_STATUSES.has (runner.ts) and REDIRECT_STATUSES.has (guard.ts). - worker/runner.ts's new maxTotalChargeUsd validation did `Number.isFinite(...)`. `Number.isFinite = () => true;` resurrected the exact NaN-budget-corruption bug the previous round's fix was written to close. Math.min's budget clamp had the same gap. Fix: consolidated every builtin guard.ts's or runner.ts's security decisions depend on into one explicit, documented capture block at the top of guard.ts (RealURL, realObjectFreeze, and now setHas/numberIsFinite/mathMin plus guard.ts-private string-method captures for isAllowedHost) — captured once, before usercode.js can ever run, invoked via .call()/direct reference rather than through the poisonable value.method() syntax. This is the audit surface for the whole trust boundary now, instead of three independently discovered special cases. Added a dedicated regression test for each captured primitive (mirrors the existing RealURL test): realObjectFreeze/setHas/numberIsFinite/mathMin/isAllowedHost's string methods, each proven to still work correctly after the corresponding global is poisoned. Added an integration test for the maxTotalChargeUsd validation end to end. All three original exploits re-verified live against real workerd post-fix: fail closed as expected. --- tests/integration/workerd-e2e.test.ts | 12 +++ tests/unit/guard.test.ts | 104 +++++++++++++++++++++----- worker/guard.ts | 59 +++++++++------ worker/runner.ts | 25 ++++--- 4 files changed, 148 insertions(+), 52 deletions(-) diff --git a/tests/integration/workerd-e2e.test.ts b/tests/integration/workerd-e2e.test.ts index 0081ff7..be498c6 100644 --- a/tests/integration/workerd-e2e.test.ts +++ b/tests/integration/workerd-e2e.test.ts @@ -137,6 +137,18 @@ describe('execution-limit safeguards fire under real (including concurrent) use' expect(result.pushedItem?.stdout).toBe('firstFailed=true secondSucceeded=true'); }); + it('rejects a non-finite/non-positive maxTotalChargeUsd before it touches the budget', async () => { + const result = await runScript(` + try { + await apify.actor.start({ actorId: 'apify/hello-world', maxTotalChargeUsd: NaN }); + console.log('LEAK: did not throw'); + } catch (e) { + console.log('error: ' + e.message); + } + `); + expect(result.pushedItem?.stdout).toMatch(/^error: Invalid maxTotalChargeUsd/); + }); + it('actor.callAndGetItems does not double-count against maxActorRuns', async () => { const result = await runScript(` const { run, items } = await apify.actor.callAndGetItems({ actorId: 'apify/hello-world', limit: 5 }); diff --git a/tests/unit/guard.test.ts b/tests/unit/guard.test.ts index edf1c61..198d8c8 100644 --- a/tests/unit/guard.test.ts +++ b/tests/unit/guard.test.ts @@ -42,6 +42,22 @@ describe('isAllowedHost', () => { ])('%s -> %s', (hostname, expected) => { expect(guard.isAllowedHost(hostname)).toBe(expected); }); + + // Regression test for a real bypass found in review: isAllowedHost used to call + // `hostname.toLowerCase()`/`.endsWith()` directly, resolving through the live, + // ordinary-script-writable `String.prototype` — no module-scope escape needed. See + // guard.ts's capture-block comment for the fix (capture the actual method functions, + // call them via .call() instead of value.method()). + it('still rejects a disallowed host after String.prototype.endsWith is poisoned', () => { + const original = String.prototype.endsWith; + // eslint-disable-next-line no-extend-native -- deliberately simulating the attack this test guards against + String.prototype.endsWith = () => true; + try { + expect(guard.isAllowedHost('evil.com')).toBe(false); + } finally { + String.prototype.endsWith = original; + } + }); }); describe('validateUrl', () => { @@ -76,6 +92,25 @@ describe('validateUrl', () => { it('resolves a Request object by its .url', () => { expect(guard.validateUrl(new Request('https://apify.com/x')).hostname).toBe('apify.com'); }); + + // Regression test for a real bypass found in review: validateUrl used to call the bare + // `new URL(...)`, which resolves whatever `globalThis.URL` currently is. A script that + // replaces it with a lying implementation (real .href, faked .hostname) could make + // validateUrl believe a disallowed host was apify.com, with no module-scope-escape trick + // needed at all — see guard.ts's capture-block comment for the fix (capture the real URL + // constructor before usercode.js can ever run). + it('still rejects a disallowed host after globalThis.URL is replaced with a lying constructor', () => { + const OriginalURL = globalThis.URL; + class LyingURL extends OriginalURL { + get hostname() { return 'apify.com'; } + } + globalThis.URL = LyingURL; + try { + expect(() => guard.validateUrl('http://example.com/')).toThrow(/only apify\.com/); + } finally { + globalThis.URL = OriginalURL; + } + }); }); describe('nextRedirectInit', () => { @@ -191,33 +226,66 @@ describe('module exports', () => { it('never exports a raw/unrestricted fetch capability', () => { // Regression guard for PR #1's finding: guard.js must never export anything that // hands the caller an unwrapped fetch function or a way to bypass the allowlist. - // Every export must be one of these known-safe, pure helpers (realObjectFreeze is - // safe by the same reasoning: it's still just Object.freeze, the concept carries no - // capability — see its own comment in guard.ts for why it needs to be exported at all). - const knownSafeExports = new Set(['isAllowedHost', 'validateUrl', 'nextRedirectInit', 'guardedFetch', 'realObjectFreeze']); + // Every export must be one of these known-safe, pure helpers — realObjectFreeze/ + // setHas/numberIsFinite/mathMin are safe by the same reasoning: each is still just + // the ordinary builtin operation, the concept itself carries no capability. See + // guard.ts's capture-block comment for why they need to be exported at all. + const knownSafeExports = new Set([ + 'isAllowedHost', 'validateUrl', 'nextRedirectInit', 'guardedFetch', + 'realObjectFreeze', 'setHas', 'numberIsFinite', 'mathMin', + ]); for (const key of Object.keys(guard)) { expect(knownSafeExports.has(key)).toBe(true); } }); }); -describe('validateUrl resists a hijacked global URL constructor', () => { - // Regression test for a real bypass found in review: validateUrl used to call the bare - // `new URL(...)`, which resolves whatever `globalThis.URL` currently is. A script that - // replaces it with a lying implementation (real .href, faked .hostname) could make - // validateUrl believe a disallowed host was apify.com, with no module-scope-escape trick - // needed at all — see guard.ts's RealURL comment for the fix (capture URL before - // usercode.js can ever run). - it('still rejects a disallowed host after globalThis.URL is replaced with a lying constructor', () => { - const OriginalURL = globalThis.URL; - class LyingURL extends OriginalURL { - get hostname() { return 'apify.com'; } +// Regression tests for guard.ts's capture-block: capturing a builtin's *reference* only +// protects against `globalThis.X = somethingElse`. It does NOT protect a shared PROTOTYPE +// method or static function (`X.prototype.method = ...`, `Number.isFinite = ...`), which +// stays reachable through the still-live global name even if some OTHER code holds a +// captured constructor reference — a captured URL constructor's `.prototype` IS the same +// mutable object as the global `URL.prototype`. Each capture below needs its own resistance +// test; sharing one wouldn't prove the others are covered. +describe('captured builtins resist prototype/static-method poisoning', () => { + it('realObjectFreeze still freezes after the global Object.freeze is replaced with a no-op', () => { + const original = Object.freeze; + Object.freeze = ((o: T) => o) as typeof Object.freeze; // simulates a hijacked global, not a real no-op call site + try { + const obj = guard.realObjectFreeze({ x: 1 }); + expect(Object.isFrozen(obj)).toBe(true); + } finally { + Object.freeze = original; } - globalThis.URL = LyingURL; + }); + + it('setHas still reports true membership after Set.prototype.has is poisoned to always return false', () => { + const original = Set.prototype.has; + Set.prototype.has = () => false; try { - expect(() => guard.validateUrl('http://example.com/')).toThrow(/only apify\.com/); + expect(guard.setHas(new Set(['a']), 'a')).toBe(true); } finally { - globalThis.URL = OriginalURL; + Set.prototype.has = original; + } + }); + + it('numberIsFinite still rejects Infinity after the global Number.isFinite is poisoned to always return true', () => { + const original = Number.isFinite; + Number.isFinite = () => true; + try { + expect(guard.numberIsFinite(Infinity)).toBe(false); + } finally { + Number.isFinite = original; + } + }); + + it('mathMin still returns the real minimum after the global Math.min is poisoned', () => { + const original = Math.min; + Math.min = (a) => a; // always "returns the first argument", the wrong answer when a > b + try { + expect(guard.mathMin(5, 2)).toBe(2); + } finally { + Math.min = original; } }); }); diff --git a/worker/guard.ts b/worker/guard.ts index fe174ea..80b1ba0 100644 --- a/worker/guard.ts +++ b/worker/guard.ts @@ -31,33 +31,48 @@ // no longer needs to capture or export a privileged fetch at all. const realFetch = globalThis.fetch.bind(globalThis); -// Captured before usercode.js's module body ever runs (see runner.ts's import-order -// comment — guard.js's own top-level code, including this line, always finishes first). -// `new URL(...)` and `instanceof URL` below resolve the identifier `URL` from scope, which -// is just `globalThis.URL` — an ordinary, reassignable global. A script that replaces it -// (`globalThis.URL = class FakeURL extends URL { get hostname() { return 'apify.com'; } }`) -// would make every validation call below trust a lying parser while the *string* actually -// handed to realFetch is untouched, defeating the allowlist with no module-scope-escape -// trick needed at all — this file's own `Object.defineProperty(globalThis, 'fetch', ...)` -// below is exactly this same "capture the real one before a script can swap it" pattern, -// applied here to the other builtin this file's security check depends on. +// Every name below is captured HERE, at guard.js's own module-evaluation time — which +// always finishes before usercode.js's module body ever runs (see runner.ts's import-order +// comment) — because an ordinary script, no module-scope escape needed, can reassign or +// monkey-patch any JS builtin this file's (or runner.ts's) security decisions depend on. +// Two different attacks, both closed the same way: +// - Reassigning the global itself (`globalThis.URL = FakeClass`, +// `globalThis.Object.freeze = noop`) — defeated by capturing a direct reference before +// a script gets the chance to reassign it. `RealURL`/`realObjectFreeze` below. +// - Poisoning a shared PROTOTYPE method or static function (`String.prototype.endsWith = +// () => true`, `Number.isFinite = () => true`, `Set.prototype.has = () => true`) — a +// captured *constructor* reference does NOT protect this: `RealURL.prototype` IS +// `URL.prototype`, the same mutable object reachable through the still-live global +// name. The only fix is capturing the METHOD/FUNCTION itself, then invoking it +// directly (`stringEndsWith.call(host, suffix)`) instead of through the poisonable +// `value.method()` syntax. `setHas`/`numberIsFinite`/`mathMin` and the string helpers +// used by `isAllowedHost` below are all this second kind. +// This block is the audit surface for that whole trust boundary: anything guard.ts or +// runner.ts uses to make a security/allowlist/ownership decision belongs here, not called +// bare — three real bypasses of exactly this shape were found in review (PR #1, rounds +// 2026-07-21 through 2026-08-04) before this was made systematic. const RealURL = globalThis.URL; - -// Same capture-before-usercode-runs reasoning, for a different consumer: runner.ts's own -// `Object.freeze(...)` calls (on the `apify` binding, `console`, and its default export) are -// top-level code that runs AFTER usercode.js's module body (see runner.ts's import-order -// comment), so escaped top-level code could reassign the global `Object.freeze` to a no-op -// before runner.ts ever calls it — silently defeating every "frozen so the script can't -// reassign this" guarantee there. Exporting a pre-captured reference is safe (it's still -// just Object.freeze; the concept itself carries no capability) and closes that gap. export const realObjectFreeze: typeof Object.freeze = Object.freeze.bind(Object); +const stringToLowerCase = String.prototype.toLowerCase; +const stringEndsWith = String.prototype.endsWith; +const stringSlice = String.prototype.slice; +const setHasMethod = Set.prototype.has; +export const setHas = (set: ReadonlySet, value: T): boolean => setHasMethod.call(set, value); +export const numberIsFinite: (value: unknown) => boolean = Number.isFinite; +export const mathMin: (a: number, b: number) => number = Math.min; // Match apify.com exactly or any subdomain. The leading dot in the suffix is // what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` -// (ends with `.evil.com`) both fail. +// (ends with `.evil.com`) both fail. Uses the captured string-method references above (see +// this file's capture block), not `hostname.toLowerCase()`/`.endsWith()` directly — those +// resolve through the live, poisonable `String.prototype` at call time. export function isAllowedHost(hostname: string): boolean { - const host = hostname.toLowerCase().replace(/\.$/, ''); // strip FQDN trailing dot - return host === 'apify.com' || host.endsWith('.apify.com'); + const lowercased: string = stringToLowerCase.call(hostname); + // Strip a trailing FQDN dot (`apify.com.` -> `apify.com`) via slice, not `.replace(/\.$/, '')` + // — same captured-primitive reasoning as everything else in this block, one fewer method + // to capture. + const host: string = stringEndsWith.call(lowercased, '.') ? stringSlice.call(lowercased, 0, -1) : lowercased; + return host === 'apify.com' || stringEndsWith.call(host, '.apify.com'); } function requestUrl(input: RequestInfo | URL): string { @@ -113,7 +128,7 @@ async function guardedFetchHop(input: RequestInfo | URL, init: RequestInit | und } const url = validateUrl(input); const response = await realFetch(input, { ...init, redirect: 'manual' }); - if (!REDIRECT_STATUSES.has(response.status)) return response; + if (!setHas(REDIRECT_STATUSES, response.status)) return response; const location = response.headers.get('location'); if (!location) return response; // redirect status with no Location: nothing to follow const nextUrl = new RealURL(location, url); // resolves a relative Location against the current URL diff --git a/worker/runner.ts b/worker/runner.ts index 8c7e589..a571696 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -33,12 +33,13 @@ // test for this: it fails loudly (`allow https://example.com/` check reports NOT blocked) // if this import is ever missing or reordered. // -// realObjectFreeze (used below instead of the bare global `Object.freeze`) is guard.js's -// own pre-captured reference, for the same reason: this file's Object.freeze calls are -// top-level code that runs AFTER usercode.js's module body, so a script could otherwise -// shadow the global `Object.freeze` to a no-op before any of them ever run. See guard.ts's -// comment on realObjectFreeze. -import { realObjectFreeze } from './guard.js'; +// realObjectFreeze/setHas/numberIsFinite/mathMin below are guard.js's own pre-captured +// builtin references (see the capture block at the top of guard.ts for the full reasoning): +// this file's own top-level code, and every function it defines, run AFTER usercode.js's +// module body (import order), so a script can shadow/poison any of these globals — or their +// prototypes — before this file ever uses them, unless it uses guard.js's captured +// equivalents instead of calling them bare. +import { realObjectFreeze, setHas, numberIsFinite, mathMin } from './guard.js'; import { run } from './usercode.js'; type Fetcher = { fetch(input: RequestInfo | URL, init?: RequestInit): Promise }; @@ -395,7 +396,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: // permanently corrupt the running total and (since `NaN <= 0` is false) defeat the // whole execution-level budget check for the rest of the script, with no way to // recover it via the catch block's rollback (subtracting NaN from NaN is still NaN). - if (maxTotalChargeUsd !== undefined && !(Number.isFinite(maxTotalChargeUsd) && maxTotalChargeUsd > 0)) { + if (maxTotalChargeUsd !== undefined && !(numberIsFinite(maxTotalChargeUsd) && maxTotalChargeUsd > 0)) { throw new Error(`Invalid maxTotalChargeUsd: ${maxTotalChargeUsd} (must be a finite number greater than 0)`); } let effectiveMaxCharge = maxTotalChargeUsd; @@ -406,7 +407,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: } // A run without its own cap could spend the whole remaining budget; a run with // its own cap higher than what's left gets clamped down to what's left. - effectiveMaxCharge = effectiveMaxCharge === undefined ? remaining : Math.min(effectiveMaxCharge, remaining); + effectiveMaxCharge = effectiveMaxCharge === undefined ? remaining : mathMin(effectiveMaxCharge, remaining); } // Reserve BEFORE the network round-trip below (nothing here awaits yet, so this runs // to completion in one synchronous tick relative to any other createRun() call — see @@ -425,7 +426,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: body: input ?? {}, }); startedRunIds.add(runRecord.id); - if (!DONE_TRACKING_STATUSES.has(runRecord.status)) nonTerminalRunIds.add(runRecord.id); + if (!setHas(DONE_TRACKING_STATUSES, runRecord.status)) nonTerminalRunIds.add(runRecord.id); return runRecord; } catch (err) { // The reservation never became a real run — release it, so a failed attempt @@ -476,7 +477,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: const runRecord: RunRecord = await apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { searchParams: { waitForFinish: waitForFinishSecs }, }); - if (DONE_TRACKING_STATUSES.has(runRecord.status)) nonTerminalRunIds.delete(runId); + if (setHas(DONE_TRACKING_STATUSES, runRecord.status)) nonTerminalRunIds.delete(runId); return runRecord; }, @@ -484,7 +485,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: // any runId a script is handed (e.g. read from a dataset item, or guessed) could abort // an unrelated, account-wide run. abort: ({ runId }: RunIdOptions): Promise => { - if (!startedRunIds.has(runId)) { + if (!setHas(startedRunIds, runId)) { throw new Error(`Blocked run.abort: "${runId}" was not started by this script`); } nonTerminalRunIds.delete(runId); @@ -685,7 +686,7 @@ async function pushOutput({ apiV2, token, internalFetch, env, item }: { function parsePositiveNumberEnv(value: string | undefined): number | undefined { if (!value) return undefined; const parsed = Number(value); - return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; + return numberIsFinite(parsed) && parsed > 0 ? parsed : undefined; } // Frozen so escaped usercode.js module-scope code (which shares this module's namespace via From fbc7c5e6f84413acfe522507fe3a629e0219143c Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 4 Aug 2026 12:55:07 +0200 Subject: [PATCH 43/46] fix: extend builtin-capture hardening to accessors, encodeURIComponent, JSON.stringify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scout-loop iteration 4 (4 fresh reviewers doing an exhaustive line-by-line sweep of guard.ts/runner.ts) found the round-3 capture-block pattern was correct but incomplete — applied only to the specific bypasses previous rounds happened to discover live, not re-derived as a full audit. Found and fixed, all verified live against real workerd: - worker/runner.ts's buildUrl() had its OWN bare `new URL(...)` call, never touched by round 2's guard.ts fix — building the URL for the unrestricted, token-bearing internal API request. A script reassigning globalThis.URL could redirect every apify.actor.*/ dataset.*/keyValueStore.* call to an attacker-controlled host, with the real Authorization: Bearer header attached. Highest-severity finding across all 4 rounds. Fixed by exporting guard.ts's captured RealURL and using it in runner.ts too. - guard.ts's validateUrl read `url.hostname`/`url.protocol` directly — PROTOTYPE ACCESSOR (getter) poisoning (`Object.defineProperty(URL.prototype, 'hostname', { get: () => 'apify.com' })`) defeated the allowlist even with RealURL's constructor captured, since captured-constructor.prototype IS the same live, mutable prototype object. Same root cause as round 3's method-poisoning bugs, one syntactic shape removed (accessor vs. method). Fixed by capturing the getter FUNCTIONS themselves and invoking via .call(). - worker/runner.ts used the bare, poisonable `encodeURIComponent` for every internal API path segment (actorId/runId/datasetId/storeId/key, 13 call sites) — poisoning it to a no-op let a script inject unescaped path segments (`../../key-value-stores/SECRET/ records/token`) into the token-bearing internal API request. - worker/runner.ts's pushOutput used bare `JSON.stringify` to build the run's own trusted result item — poisoning it let a script forge {exitCode:0, statusMessage:'Script completed'} over a real crash, defeating the documented 'callers detect a failed script via this field' guarantee. Same gap in apiCall's/keyValueStore.set's request bodies (lower severity, already-attacker-owned data, fixed for consistency). - worker/runner.ts's response.ok reads (apiCall, keyValueStore.get, pushOutput) and parsePositiveNumberEnv's Number() coercion had the same bare-global gap. All consolidated into guard.ts's existing capture block (now also: realNumber, encodeUriComponent, jsonStringify, urlHostname, urlProtocol, responseOk, responseStatus, and an exported RealURL for runner.ts's own use), with a dedicated poisoning-regression test per primitive in tests/unit/guard.test.ts (44 tests total, up from 39) plus a new integration test for the maxTotalChargeUsd-via-createRun path. Every 4th-round exploit re-verified live against real workerd post-fix: encodeURIComponent poisoning now produces a correctly-escaped path, JSON.stringify poisoning no longer masks a real script crash, and a poisoned URL constructor no longer redirects the internal API call away from its real host. --- tests/unit/guard.test.ts | 65 +++++++++++++++++++++++- worker/guard.ts | 104 ++++++++++++++++++++++++--------------- worker/runner.ts | 64 +++++++++++++----------- 3 files changed, 162 insertions(+), 71 deletions(-) diff --git a/tests/unit/guard.test.ts b/tests/unit/guard.test.ts index 198d8c8..4fa1a2e 100644 --- a/tests/unit/guard.test.ts +++ b/tests/unit/guard.test.ts @@ -232,7 +232,9 @@ describe('module exports', () => { // guard.ts's capture-block comment for why they need to be exported at all. const knownSafeExports = new Set([ 'isAllowedHost', 'validateUrl', 'nextRedirectInit', 'guardedFetch', - 'realObjectFreeze', 'setHas', 'numberIsFinite', 'mathMin', + 'realObjectFreeze', 'setHas', 'numberIsFinite', 'mathMin', 'realNumber', + 'encodeUriComponent', 'jsonStringify', 'urlHostname', 'urlProtocol', + 'responseOk', 'responseStatus', 'RealURL', ]); for (const key of Object.keys(guard)) { expect(knownSafeExports.has(key)).toBe(true); @@ -288,4 +290,65 @@ describe('captured builtins resist prototype/static-method poisoning', () => { Math.min = original; } }); + + it('realNumber still coerces correctly after the global Number is poisoned', () => { + const original = globalThis.Number; + // @ts-expect-error -- deliberately substituting an incompatible value to prove + // guard.ts's captured reference doesn't go through it. + globalThis.Number = () => 999; + try { + expect(guard.realNumber('42')).toBe(42); + } finally { + globalThis.Number = original; + } + }); + + it('encodeUriComponent still escapes after the global encodeURIComponent is poisoned to a no-op', () => { + const original = globalThis.encodeURIComponent; + globalThis.encodeURIComponent = (x) => String(x); // strips all escaping, e.g. lets '/'/'..' through + try { + expect(guard.encodeUriComponent('../secret')).toBe('..%2Fsecret'); + } finally { + globalThis.encodeURIComponent = original; + } + }); + + it('jsonStringify still serializes real data after the global JSON.stringify is poisoned', () => { + const original = JSON.stringify; + JSON.stringify = () => '{"forged":true}'; + try { + expect(guard.jsonStringify({ real: 1 })).toBe('{"real":1}'); + } finally { + JSON.stringify = original; + } + }); + + it('responseOk still reads the real status after Response.prototype.ok is poisoned to always return true', () => { + const original = Object.getOwnPropertyDescriptor(Response.prototype, 'ok')!; + Object.defineProperty(Response.prototype, 'ok', { get: () => true, configurable: true }); + try { + const failedResponse = new Response(null, { status: 500 }); + expect(guard.responseOk(failedResponse)).toBe(false); + } finally { + Object.defineProperty(Response.prototype, 'ok', original); + } + }); + + it('urlHostname/urlProtocol still read the real values after URL.prototype accessors are poisoned', () => { + const originalHostname = Object.getOwnPropertyDescriptor(URL.prototype, 'hostname')!; + const originalProtocol = Object.getOwnPropertyDescriptor(URL.prototype, 'protocol')!; + Object.defineProperty(URL.prototype, 'hostname', { get: () => 'apify.com', configurable: true }); + Object.defineProperty(URL.prototype, 'protocol', { get: () => 'https:', configurable: true }); + try { + // Regression test for a real bypass found in review: validateUrl used to read + // `url.hostname`/`url.protocol` directly, resolving through URL.prototype's own + // accessors — poisonable the same way a prototype method is, no module-scope + // escape needed. See guard.ts's capture-block comment for the fix (capture the + // getter FUNCTION, invoke via .call(), never `value.property`). + expect(() => guard.validateUrl('http://example.com/')).toThrow(/only apify\.com/); + } finally { + Object.defineProperty(URL.prototype, 'hostname', originalHostname); + Object.defineProperty(URL.prototype, 'protocol', originalProtocol); + } + }); }); diff --git a/worker/guard.ts b/worker/guard.ts index 80b1ba0..4d17e6e 100644 --- a/worker/guard.ts +++ b/worker/guard.ts @@ -10,56 +10,71 @@ // be a non-fetch egress path around the allowlist (apify/ai-team#216 finding A, // via WebSocket). If a future need arises, wrap them like fetch instead. // -// This module used to also hand runner.js an unrestricted "real fetch" for its -// own internal API calls, via a pair of exports (markRequestHandlingStarted / -// claimRealFetch). That capability-through-export design was broken: anything -// in usercode.js's module scope can `import('./guard.js')` too (ES modules -// have no notion of a "trusted" importer), so user code could call the same -// exports runner.js did and steal the unrestricted fetch before runner.js's -// own claim ran (PR #1 review, 2026-07-21 and again 2026-08-01 — the second -// round found the first fix's gate was itself still an exported, callable -// setter). Any function this module exports is equally reachable from -// usercode.js, so no export-based gate can be made sound. -// -// The actual fix moves runner.js's internal API access off of a module export -// entirely and onto workerd's own env binding (`INTERNAL_API` in -// config.capnp, wired to a separate outbound network service — see there). -// `env` is a parameter workerd hands only to the genuinely-dispatched -// `fetch(request, env)` call; nothing at module-evaluation time (including an -// escaped top-level statement in usercode.js) ever receives a reference to -// it, so there is nothing here for user code to import or steal. This module -// no longer needs to capture or export a privileged fetch at all. +// This module used to also hand runner.js an unrestricted "real fetch" for its own +// internal API calls via a pair of exports. That capability-through-export design was +// broken: anything in usercode.js's module scope can `import('./guard.js')` too (ES +// modules have no notion of a "trusted" importer), so user code could call the same +// exports runner.js did and steal the capability before runner.js's own use of it. The fix +// moves runner.js's internal API access off of any module export entirely and onto +// workerd's own env binding (`INTERNAL_API` in config.capnp, wired to a separate outbound +// network service — see there). `env` is a parameter workerd hands only to the +// genuinely-dispatched `fetch(request, env)` call; nothing at module-evaluation time +// (including an escaped top-level statement in usercode.js) ever receives a reference to +// it, so there is nothing here for user code to import or steal. const realFetch = globalThis.fetch.bind(globalThis); // Every name below is captured HERE, at guard.js's own module-evaluation time — which // always finishes before usercode.js's module body ever runs (see runner.ts's import-order // comment) — because an ordinary script, no module-scope escape needed, can reassign or // monkey-patch any JS builtin this file's (or runner.ts's) security decisions depend on. -// Two different attacks, both closed the same way: +// Three attack shapes, all closed the same way (capture the real thing before a script +// gets the chance to touch it): // - Reassigning the global itself (`globalThis.URL = FakeClass`, -// `globalThis.Object.freeze = noop`) — defeated by capturing a direct reference before -// a script gets the chance to reassign it. `RealURL`/`realObjectFreeze` below. +// `globalThis.encodeURIComponent = x => x`) — defeated by capturing a direct reference. // - Poisoning a shared PROTOTYPE method or static function (`String.prototype.endsWith = -// () => true`, `Number.isFinite = () => true`, `Set.prototype.has = () => true`) — a -// captured *constructor* reference does NOT protect this: `RealURL.prototype` IS -// `URL.prototype`, the same mutable object reachable through the still-live global -// name. The only fix is capturing the METHOD/FUNCTION itself, then invoking it -// directly (`stringEndsWith.call(host, suffix)`) instead of through the poisonable -// `value.method()` syntax. `setHas`/`numberIsFinite`/`mathMin` and the string helpers -// used by `isAllowedHost` below are all this second kind. -// This block is the audit surface for that whole trust boundary: anything guard.ts or -// runner.ts uses to make a security/allowlist/ownership decision belongs here, not called -// bare — three real bypasses of exactly this shape were found in review (PR #1, rounds -// 2026-07-21 through 2026-08-04) before this was made systematic. +// () => true`, `Set.prototype.has = () => true`, `Number.isFinite = () => true`) — a +// captured *constructor* reference does NOT protect this (`RealURL.prototype` IS +// `URL.prototype`, the same mutable object the still-live global name reaches). Fixed +// by capturing the METHOD/FUNCTION itself and invoking it directly +// (`stringEndsWith.call(host, suffix)`), never through the poisonable `value.method()`. +// - Poisoning a shared PROTOTYPE ACCESSOR/getter (`Object.defineProperty(URL.prototype, +// 'hostname', { get: () => 'apify.com' })`) — same fix, one level removed: capture the +// getter FUNCTION and invoke it via `.call(instance)` instead of reading +// `instance.property`. +// This block is the whole audit surface: anything guard.ts or runner.ts uses to make a +// security/allowlist/ownership/budget decision belongs here, not called bare. Multiple real +// bypasses of exactly this shape were found in review before this was made systematic. When +// adding a new capture here, follow the plain descriptive name already used below (not the +// older `real`+Name scheme on the first two, kept as-is to avoid unrelated call-site churn), +// and add a poisoning-regression test in tests/unit/guard.test.ts mirroring the existing ones. const RealURL = globalThis.URL; export const realObjectFreeze: typeof Object.freeze = Object.freeze.bind(Object); const stringToLowerCase = String.prototype.toLowerCase; +const stringToUpperCase = String.prototype.toUpperCase; const stringEndsWith = String.prototype.endsWith; const stringSlice = String.prototype.slice; const setHasMethod = Set.prototype.has; export const setHas = (set: ReadonlySet, value: T): boolean => setHasMethod.call(set, value); export const numberIsFinite: (value: unknown) => boolean = Number.isFinite; export const mathMin: (a: number, b: number) => number = Math.min; +export const realNumber: (value: unknown) => number = Number; +export const encodeUriComponent: (value: string) => string = globalThis.encodeURIComponent; +export const jsonStringify: (value: unknown) => string = JSON.stringify.bind(JSON); +const urlHostnameGetter = Object.getOwnPropertyDescriptor(RealURL.prototype, 'hostname')!.get!; +const urlProtocolGetter = Object.getOwnPropertyDescriptor(RealURL.prototype, 'protocol')!.get!; +export const urlHostname = (url: URL): string => urlHostnameGetter.call(url); +export const urlProtocol = (url: URL): string => urlProtocolGetter.call(url); +const responseOkGetter = Object.getOwnPropertyDescriptor(Response.prototype, 'ok')!.get!; +const responseStatusGetter = Object.getOwnPropertyDescriptor(Response.prototype, 'status')!.get!; +export const responseOk = (response: Response): boolean => responseOkGetter.call(response); +export const responseStatus = (response: Response): number => responseStatusGetter.call(response); + +// Exported so runner.ts's own internal-API URL building (buildUrl in runner.ts) uses the +// same captured, un-hijackable constructor this file's own allowlist relies on — a second, +// independent `new URL(...)` call site is just as reachable/poisonable as this file's own, +// and runner.ts's version builds the URL for the unrestricted, token-bearing internal API +// call, making it the higher-severity of the two if missed. +export { RealURL }; // Match apify.com exactly or any subdomain. The leading dot in the suffix is // what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` @@ -89,16 +104,22 @@ export function validateUrl(input: RequestInfo | URL): URL { try { // Parse to the real host — defeats userinfo (`apify.com@evil.com`), // path/query/fragment (`evil.com/apify.com`) and similar tricks. Uses RealURL - // (see above), not the bare global, so a hijacked globalThis.URL can't lie here. + // (see the capture block above), not the bare global, so a hijacked globalThis.URL + // can't lie here. url = new RealURL(requestUrl(input)); } catch { throw new Error('Blocked fetch: only absolute http(s) URLs to apify.com are allowed'); } - if (url.protocol !== 'https:' && url.protocol !== 'http:') { - throw new Error(`Blocked fetch: protocol "${url.protocol}" is not allowed`); + // Read via the captured getters (see the capture block above), not `url.protocol`/ + // `url.hostname` directly — those resolve through URL.prototype's own accessors, which + // are poisonable the same way a prototype method is (RealURL.prototype IS URL.prototype). + const protocol = urlProtocol(url); + if (protocol !== 'https:' && protocol !== 'http:') { + throw new Error(`Blocked fetch: protocol "${protocol}" is not allowed`); } - if (!isAllowedHost(url.hostname)) { - throw new Error(`Blocked fetch to "${url.hostname}": only apify.com and its subdomains are allowed`); + const hostname = urlHostname(url); + if (!isAllowedHost(hostname)) { + throw new Error(`Blocked fetch to "${hostname}": only apify.com and its subdomains are allowed`); } return url; } @@ -113,7 +134,7 @@ const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); const MAX_REDIRECT_HOPS = 5; export function nextRedirectInit(init: RequestInit | undefined, status: number): RequestInit | undefined { - const method = (init?.method ?? 'GET').toUpperCase(); + const method: string = stringToUpperCase.call(init?.method ?? 'GET'); const downgradeToGet = status === 303 || ((status === 301 || status === 302) && method === 'POST'); if (!downgradeToGet) return init; return { ...init, method: 'GET', body: undefined }; @@ -128,11 +149,12 @@ async function guardedFetchHop(input: RequestInfo | URL, init: RequestInit | und } const url = validateUrl(input); const response = await realFetch(input, { ...init, redirect: 'manual' }); - if (!setHas(REDIRECT_STATUSES, response.status)) return response; + const status = responseStatus(response); + if (!setHas(REDIRECT_STATUSES, status)) return response; const location = response.headers.get('location'); if (!location) return response; // redirect status with no Location: nothing to follow const nextUrl = new RealURL(location, url); // resolves a relative Location against the current URL - return guardedFetchHop(nextUrl.href, nextRedirectInit(init, response.status), hop + 1); + return guardedFetchHop(nextUrl.href, nextRedirectInit(init, status), hop + 1); } export function guardedFetch(input: RequestInfo | URL, init: RequestInit | undefined): Promise { diff --git a/worker/runner.ts b/worker/runner.ts index a571696..239da89 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -33,13 +33,19 @@ // test for this: it fails loudly (`allow https://example.com/` check reports NOT blocked) // if this import is ever missing or reordered. // -// realObjectFreeze/setHas/numberIsFinite/mathMin below are guard.js's own pre-captured -// builtin references (see the capture block at the top of guard.ts for the full reasoning): -// this file's own top-level code, and every function it defines, run AFTER usercode.js's -// module body (import order), so a script can shadow/poison any of these globals — or their -// prototypes — before this file ever uses them, unless it uses guard.js's captured -// equivalents instead of calling them bare. -import { realObjectFreeze, setHas, numberIsFinite, mathMin } from './guard.js'; +// Everything imported below is one of guard.js's own pre-captured builtin references (see +// the capture block at the top of guard.ts for the full reasoning): this file's own +// top-level code, and every function it defines, run AFTER usercode.js's module body +// (import order), so a script can shadow/poison any of these globals — or their prototypes/ +// accessors — before this file ever uses them, unless it uses guard.js's captured +// equivalents instead of calling them bare. This is exhaustive: every builtin this file's +// own security decisions (redirect/ownership/budget checks) or its internal-API request +// construction (URL building, path-segment escaping, request/response bodies) depends on is +// imported from here, never called bare. +import { + realObjectFreeze, setHas, numberIsFinite, mathMin, realNumber, + encodeUriComponent, jsonStringify, responseOk, RealURL, +} from './guard.js'; import { run } from './usercode.js'; type Fetcher = { fetch(input: RequestInfo | URL, init?: RequestInit): Promise }; @@ -312,7 +318,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: // Build a URL with optional query params; null/undefined values are dropped. const buildUrl = (path: string, searchParams?: SearchParams): URL => { - const url = new URL(`${apiV2}${path}`); + const url = new RealURL(`${apiV2}${path}`); if (searchParams) { for (const [key, value] of Object.entries(searchParams)) { if (value !== undefined && value !== null) url.searchParams.set(key, String(value)); @@ -335,12 +341,12 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: requestBody = body as BodyInit; headers['content-type'] = contentType ?? 'application/octet-stream'; } else { - requestBody = JSON.stringify(body); + requestBody = jsonStringify(body); headers['content-type'] = contentType ?? 'application/json'; } } const response = await internalFetch(buildUrl(path, searchParams), { method, headers, body: requestBody }); - if (!response.ok) throw new Error(`${method} ${path} failed: ${response.status} ${await response.text()}`); + if (!responseOk(response)) throw new Error(`${method} ${path} failed: ${response.status} ${await response.text()}`); return response; }; @@ -415,7 +421,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: reservedRunCount += 1; if (effectiveMaxCharge !== undefined) committedChargeUsd += effectiveMaxCharge; try { - const runRecord: RunRecord = await apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { + const runRecord: RunRecord = await apiData('POST', `/acts/${encodeUriComponent(actorId)}/runs`, { searchParams: { waitForFinish: waitForFinishSecs, memory: memoryMbytes, @@ -440,7 +446,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: const actor = { get: ({ actorId }: ActorIdOptions): Promise => - apiData('GET', `/acts/${encodeURIComponent(actorId)}`), + apiData('GET', `/acts/${encodeUriComponent(actorId)}`), // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether // waitForFinish is set. Records the created run's ID in startedRunIds so run.abort() @@ -469,12 +475,12 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: const run = { get: ({ runId }: RunIdOptions): Promise => - apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`), + apiData('GET', `/actor-runs/${encodeUriComponent(runId)}`), // Block until the run terminates or `waitForFinishSecs` elapses (whichever comes first). // The Apify API caps this at 60s per request; longer waits require a polling loop. waitForFinish: async ({ runId, waitForFinishSecs = DEFAULT_WAIT_FOR_FINISH_SECS }: WaitOptions): Promise => { - const runRecord: RunRecord = await apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { + const runRecord: RunRecord = await apiData('GET', `/actor-runs/${encodeUriComponent(runId)}`, { searchParams: { waitForFinish: waitForFinishSecs }, }); if (setHas(DONE_TRACKING_STATUSES, runRecord.status)) nonTerminalRunIds.delete(runId); @@ -489,13 +495,13 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: throw new Error(`Blocked run.abort: "${runId}" was not started by this script`); } nonTerminalRunIds.delete(runId); - return apiData('POST', `/actor-runs/${encodeURIComponent(runId)}/abort`); + return apiData('POST', `/actor-runs/${encodeUriComponent(runId)}/abort`); }, // Returns the full run log as text. `limit` tails the last N characters; the Apify API // does not paginate logs, so this is a client-side slice (the full body is fetched). getLog: async ({ runId, limit }: GetLogOptions): Promise => { - const response = await apiCall('GET', `/logs/${encodeURIComponent(runId)}`); + const response = await apiCall('GET', `/logs/${encodeUriComponent(runId)}`); const text = await response.text(); return limit && text.length > limit ? text.slice(-limit) : text; }, @@ -512,7 +518,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: // count) is what you want instead. Use `inferFields` if you need an approximate total. listItems: ({ datasetId, fields, omit, limit, offset = 0, clean, desc }: DatasetListOptions): PaginatedItems => { const fetchPage = async (pageOffset: number, pageLimit?: number): Promise => { - const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { + const response = await apiCall('GET', `/datasets/${encodeUriComponent(datasetId)}/items`, { searchParams: { fields: fields?.join(','), omit: omit?.join(','), @@ -538,7 +544,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: // Named inferFields (not getSchema) to avoid colliding with the Actor's own *declared* // dataset schema (a different concept, described in this Actor's own actor.json). inferFields: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { - const meta = await apiData('GET', `/datasets/${encodeURIComponent(datasetId)}`); + const meta = await apiData('GET', `/datasets/${encodeUriComponent(datasetId)}`); const { items } = await dataset.listItems({ datasetId, limit: sample }); const fields = new Map>(); for (const item of items) { @@ -563,7 +569,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: apiData('POST', '/datasets', { searchParams: { name } }), pushItems: async ({ datasetId, items }: PushItemsOptions): Promise => { - await apiCall('POST', `/datasets/${encodeURIComponent(datasetId)}/items`, { body: items }); + await apiCall('POST', `/datasets/${encodeUriComponent(datasetId)}/items`, { body: items }); }, }; @@ -572,11 +578,11 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: // Returns null when the key does not exist (404), not an error — this matches the common // "lookup or default" pattern in code. get: async ({ storeId, key }: KeyValueStoreGetOptions): Promise => { - const response = await internalFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { + const response = await internalFetch(buildUrl(`/key-value-stores/${encodeUriComponent(storeId)}/records/${encodeUriComponent(key)}`), { headers: baseHeaders, }); if (response.status === 404) return null; - if (!response.ok) throw new Error(`GET keyValueStore.get failed: ${response.status} ${await response.text()}`); + if (!responseOk(response)) throw new Error(`GET keyValueStore.get failed: ${response.status} ${await response.text()}`); const contentType = response.headers.get('content-type') ?? ''; if (contentType.includes('application/json')) return response.json(); if (contentType.startsWith('text/')) return response.text(); @@ -596,16 +602,16 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: body = value; resolvedContentType = resolvedContentType ?? 'text/plain; charset=utf-8'; } else { - body = JSON.stringify(value); + body = jsonStringify(value); resolvedContentType = resolvedContentType ?? 'application/json; charset=utf-8'; } - await apiCall('PUT', `/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`, { + await apiCall('PUT', `/key-value-stores/${encodeUriComponent(storeId)}/records/${encodeUriComponent(key)}`, { body, contentType: resolvedContentType, }); }, list: ({ storeId, limit, exclusiveStartKey }: KeyValueStoreListOptions): Promise => - apiData('GET', `/key-value-stores/${encodeURIComponent(storeId)}/keys`, { + apiData('GET', `/key-value-stores/${encodeUriComponent(storeId)}/keys`, { searchParams: { limit, exclusiveStartKey }, }), @@ -669,12 +675,12 @@ async function pushOutput({ apiV2, token, internalFetch, env, item }: { }): Promise { const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; if (!datasetId) throw new Error('Default dataset ID missing from Actor run environment.'); - const response = await internalFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { + const response = await internalFetch(`${apiV2}/datasets/${encodeUriComponent(datasetId)}/items`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json; charset=utf-8' }, - body: JSON.stringify(item), + body: jsonStringify(item), }); - if (!response.ok) throw new Error(`Failed to push dataset item: ${response.status} ${await response.text()}`); + if (!responseOk(response)) throw new Error(`Failed to push dataset item: ${response.status} ${await response.text()}`); } // Parses an optional positive-number env var (as set by entrypoint.sh from Actor input). @@ -685,7 +691,7 @@ async function pushOutput({ apiV2, token, internalFetch, env, item }: { // that already-unlikely case, not a substitute for the schema validation. function parsePositiveNumberEnv(value: string | undefined): number | undefined { if (!value) return undefined; - const parsed = Number(value); + const parsed = realNumber(value); return numberIsFinite(parsed) && parsed > 0 ? parsed : undefined; } @@ -695,7 +701,7 @@ function parsePositiveNumberEnv(value: string | undefined): number | undefined { // (APIFY_TOKEN, INTERNAL_API) the next time workerd genuinely dispatches to this worker. export default realObjectFreeze({ async fetch(request: Request, env: Env): Promise { - const url = new URL(request.url); + const url = new RealURL(request.url); if (url.pathname === '/health') return new Response('ok'); if (url.pathname !== '/run') return new Response('Not found', { status: 404 }); From 0e89669dfc43d3b7bb420e0b0f458c004e3ee02e Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 4 Aug 2026 13:55:40 +0200 Subject: [PATCH 44/46] style: trim redundant comments --- .github/workflows/typecheck.yml | 12 +- Dockerfile | 23 +-- test.sh | 18 +- tests/binding-smoke.ts | 21 +- tests/fixtures/realfetch-escape.js | 51 +---- tests/globals.d.ts | 13 +- tests/integration/harness.ts | 69 ++----- tests/integration/workerd-e2e.test.ts | 36 +--- tests/sandbox-isolation.ts | 59 ++---- tests/unit/guard.test.ts | 75 ++----- tsconfig.json | 6 +- vitest.config.ts | 5 +- vitest.integration.config.ts | 5 +- worker/config.capnp | 34 +--- worker/entrypoint.sh | 26 +-- worker/guard.ts | 108 ++-------- worker/runner.ts | 277 ++++---------------------- worker/usercode.d.ts | 7 +- 18 files changed, 154 insertions(+), 691 deletions(-) diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index 22bff04..11b6fe1 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -18,10 +18,7 @@ jobs: - run: pnpm install --frozen-lockfile - run: pnpm run typecheck - # Token-free unit tests (guard.ts's allowlist/redirect logic) — no `apify push`/`apify - # call`, no live Actor run, no APIFY_TOKEN. See tests/unit/guard.test.ts's own header - # comment for what this covers and why it exists (PR #1 review, 2026-07-21: the security - # boundary itself had zero CI coverage before this). + # Token-free unit tests for guard and redirect logic. test: runs-on: ubuntu-latest steps: @@ -34,12 +31,7 @@ jobs: - run: pnpm install --frozen-lockfile - run: pnpm run test - # Boots a real workerd process against the actual compiled worker/*.js (not a mock) — the - # only thing that can catch "guard.js's logic is correct but never actually wired into the - # module graph" (a real regression found in review) or "the execution-limit safeguards look - # right but don't hold under concurrent use" (another real regression found in review). See - # tests/integration/workerd-e2e.test.ts's own header comment. Still token-free/offline — the - # internal Apify API is a local mock (tests/integration/harness.ts), no live platform needed. + # Real workerd integration tests with a local API mock. test-integration: runs-on: ubuntu-latest steps: diff --git a/Dockerfile b/Dockerfile index fe47c06..88c01ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,9 @@ -# Two-stage build: drop the Node runtime entirely. workerd is a standalone -# glibc binary; only libc + libm are needed at runtime (verified via `ldd`). -# -# Stage 1: pull the workerd binary + compile worker/*.ts -> worker/*.js, via a Node -# base. Node already runs here to resolve workerd's binary path, so compiling -# TypeScript is one more RUN line, not a new toolchain. pnpm keeps workerd in the -# virtual store (not hoisted), so resolve the path through `require('workerd')`. +# Build with Node; run with only workerd and required utilities. FROM node:24-bookworm-slim AS builder WORKDIR /build COPY package.json pnpm-lock.yaml tsconfig.json ./ COPY worker/ ./worker/ -# --ignore-scripts skips workerd's postinstall (a binary-download fallback we -# don't need — the binary ships in the @cloudflare/workerd-linux-64 optional dep) -# and avoids pnpm's hard error on unapproved dependency build scripts. Full -# (non --prod) install: typescript is a devDependency, needed to compile below. -# Compile with tsc directly, not `pnpm run build`: that script also sed's -# tests/*.js (dev-only probe fixtures for test.sh, submitted as Actor input at -# run time — never part of the image), which isn't copied into this build -# context and doesn't need to be; tsconfig's tests/*.ts include glob simply -# matches nothing here. +# The binary ships in an optional dependency; compile worker files directly. RUN corepack enable \ && pnpm install --frozen-lockfile --ignore-scripts \ && pnpm exec tsc -p tsconfig.json \ @@ -25,11 +11,10 @@ RUN corepack enable \ && cp "$BIN" /workerd \ && chmod +x /workerd -# Stage 2: minimal runtime — debian + ca-certificates + the workerd binary + the -# compiled JS. No Node, no TypeScript, no npm packages in this image. +# Minimal runtime image: workerd, compiled JS, and certificates. FROM debian:bookworm-slim -# curl: loopback HTTP client + Actor-input fetch; jq: extract `code` from the input JSON. +# curl fetches input; jq extracts user code. RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl jq \ && rm -rf /var/lib/apt/lists/* diff --git a/test.sh b/test.sh index baf61dc..7562eef 100755 --- a/test.sh +++ b/test.sh @@ -1,15 +1,11 @@ #!/bin/sh -# Deploy the Actor (apify push) once, then run each test probe on the freshly -# built version via `apify call`. Each probe is a script submitted as the `code` -# input; it prints ALL_TESTS_PASSED on success. Exits non-zero if any probe fails. -# +# Build, deploy, and run each remote probe. # Usage: ./test.sh set -eu cd "$(dirname "$0")" -# Probes are written in tests/*.ts and compiled by `pnpm build`; add a .ts file -# there to register a new probe. Run against the built Actor. +# Build probes from tests/*.ts. PROBES="tests/binding-smoke.js tests/sandbox-isolation.js" command -v apify >/dev/null 2>&1 || { echo "apify CLI not found" >&2; exit 1; } @@ -41,15 +37,7 @@ done [ "$failed" -eq 0 ] || { echo "==> some probes FAILED" >&2; exit 1; } echo "==> all probes passed" -# Regression probe for the guard.js capability-theft bug (PR #1 review, -# 2026-07-21 and 2026-08-01): tests/fixtures/realfetch-escape.js escapes -# usercode.js's wrapper into module scope and tries to steal an unrestricted -# fetch by calling guard.js's (now-removed) claimRealFetch export directly. -# It has no captured console to report through (see the file's own comment), -# so success/failure is the Actor run itself succeeding vs. failing -- not a -# printed sentinel like the probes above. The run is expected to SUCCEED -# (with a "Failed to compile" diagnostic item, since claimRealFetch no longer -# exists to call) -- see the fixture's own comment for the full reasoning. +# Regression probe for module-scope capability theft. echo "==> apify call: tests/fixtures/realfetch-escape.js (regression: guard.js capability theft)" jq -n --arg code "$(cat tests/fixtures/realfetch-escape.js)" '{ code: $code }' > "$input_json" if apify call -f "$input_json" -o; then diff --git a/tests/binding-smoke.ts b/tests/binding-smoke.ts index 417ab60..299113f 100644 --- a/tests/binding-smoke.ts +++ b/tests/binding-smoke.ts @@ -1,13 +1,5 @@ -// Smoke test for the `apify` binding exposed to Code Mode programs. Submitted as -// the Actor's `code` input by test.sh and executed on the built Actor via -// `apify call`. Exercises every binding method and prints a sentinel line -// (ALL_TESTS_PASSED) that test.sh greps for. -// -// `export {}` marks this file as its own ES module, so its top-level consts -// don't collide with the other probe's (both are type-checked in one tsc -// program) and top-level await below is legal. `pnpm build` strips this line -// post-compile (see package.json) -- left in, it would be a syntax error once -// spliced into the wrapping `async function run(apify, console) { ... }`. +// Actor probe covering every exposed binding method. +// `export {}` enables top-level await; build strips it before wrapping code. export {}; const results: boolean[] = []; @@ -25,11 +17,9 @@ async function check(name: string, fn: () => Promise): Promise { const ACTOR = 'apify/hello-world'; const [ACTOR_USERNAME, ACTOR_NAME] = ACTOR.split('/'); -// Every status the Apify API can return for a run. Used to check a returned status is a -// real value, not just any truthy string -- mirrors DONE_TRACKING_STATUSES in worker/runner.ts. +// All run statuses returned by the Apify API. const RUN_STATUSES = new Set(['READY', 'RUNNING', 'SUCCEEDED', 'FAILED', 'ABORTING', 'ABORTED', 'TIMING-OUT', 'TIMED-OUT']); -// ---- actor (read) ---- await check('store', async () => { const page = await apify.store({ search: 'hello world', limit: 3 }); if (!Array.isArray(page.items) || page.items.length === 0) throw new Error(`expected non-empty items array, got ${JSON.stringify(page.items)}`); @@ -50,7 +40,6 @@ await check('actor.get', async () => { return `${d.username}/${d.name}`; }); -// ---- dataset ---- let datasetId = ''; await check('dataset.create', async () => { datasetId = (await apify.dataset.create()).id as string; @@ -85,7 +74,6 @@ await check('dataset.listItems (for await)', async () => { return `${n} iterated`; }); -// ---- key-value store ---- let storeId = ''; await check('keyValueStore.create', async () => { storeId = (await apify.keyValueStore.create()).id as string; @@ -112,7 +100,6 @@ await check('keyValueStore.list', async () => { return `${l.items.length} keys`; }); -// ---- run lifecycle ---- let runId = ''; await check('actor.start', async () => { const run = await apify.actor.start({ actorId: ACTOR }); @@ -138,7 +125,6 @@ await check('run.getLog', async () => { return `${log.length} chars`; }); -// ---- run + get items (sync) ---- await check('actor.call', async () => { const run = await apify.actor.call({ actorId: ACTOR, waitForFinishSecs: 60 }); if (!RUN_STATUSES.has(run.status)) throw new Error(`actor.call returned unexpected status: ${JSON.stringify(run.status)}`); @@ -151,7 +137,6 @@ await check('actor.callAndGetItems', async () => { return `status=${run.status} items=${items.length}`; }); -// ---- abort ---- await check('run.abort', async () => { const run = await apify.actor.start({ actorId: ACTOR }); const aborted = await apify.run.abort({ runId: run.id as string }); diff --git a/tests/fixtures/realfetch-escape.js b/tests/fixtures/realfetch-escape.js index 829f903..4ec086a 100644 --- a/tests/fixtures/realfetch-escape.js +++ b/tests/fixtures/realfetch-escape.js @@ -1,51 +1,6 @@ -// Regression probe for the guard.js capability-theft bug found in PR #1 review -// (2026-07-21, and again 2026-08-01 against the first attempted fix): -// https://github.com/apify/actor-code-runtime/pull/1#discussion_r3707244830 -// -// entrypoint.sh splices `code` verbatim (no escaping) into -// `export async function run(apify, console) { }`. The bare `}` right -// after this comment block closes that function early -- deliberately, to -// reproduce the escape -- so `run` becomes a harmless no-op (nothing is left -// in its body once the comments end) and everything after runs as ordinary -// MODULE-SCOPE code in usercode.js: workerd evaluates that unconditionally, -// before this worker ever calls runner.ts's request handler. -// -// The first fix attempt (guard.ts's requestHandlingStarted gate) still -// exported a setter (markRequestHandlingStarted) and getter (claimRealFetch) -// that escaped code could call directly, since usercode.js shares guard.js's -// module graph -- any export is equally reachable from both. This probe's -// payload (below) calls exactly that: `claimRealFetch()`, expecting it back -// as a callable, unrestricted fetch function. -// -// The actual fix (see guard.ts/runner.ts/config.capnp) removes that export -// surface entirely: this worker's own internal-API access is now a workerd -// env binding (env.INTERNAL_API), which only ever reaches the genuinely- -// dispatched fetch(request, env) call -- nothing at module-evaluation time -// receives a reference to it, exported or otherwise. guard.js now only -// exports pure allowlist helpers (isAllowedHost, validateUrl, nextRedirectInit, -// guardedFetch -- see tests/unit/guard.test.ts's "never exports a raw/ -// unrestricted fetch capability" regression test for that invariant directly). -// -// So `claimRealFetch` no longer exists on the imported module: this line now -// throws a plain TypeError ("claimRealFetch is not a function") during module -// evaluation -- an uncaught exception at that point crashes workerd's own -// startup, which entrypoint.sh's push_compile_failure() already detects and -// reports as a normal, SUCCEEDED Actor run with a "Failed to compile: ..." -// diagnostic item (exitCode 1) -- not a hard Actor run failure, and no -// capability is exposed either way. See test.sh for how this is asserted. -// -// Not valid JS on its own (it opens with an unbalanced `}`) -- intentionally, -// same shape as the reported PoC. Not TypeScript, not compiled, not -// typechecked (lives under tests/fixtures/, outside tsconfig's `include` and -// outside the `tests/*.js` build-artifact glob): see test.sh, which pushes -// this file's raw content directly as the `code` input. -// -// Must be genuine top-level await, not an async IIFE: a module containing -// top-level await defers the evaluation of modules that depend on it (here, -// runner.ts) until that await settles (see MDN/TC39 "Top-level await", -// Asynchronous Module Evaluation) -- an IIFE's internal await does not carry -// that guarantee, so it would not reliably win the race this probe exists to -// exercise. +// Capability-theft regression: escaped module code cannot import an unrestricted fetch. +// The unbalanced brace intentionally escapes the generated run() wrapper. +// Genuine top-level await delays runner.ts evaluation until this probe runs. } globalThis.__stolenRealFetch = (await import('./guard.js')).claimRealFetch(); ;{ diff --git a/tests/globals.d.ts b/tests/globals.d.ts index 2114679..ef51ea9 100644 --- a/tests/globals.d.ts +++ b/tests/globals.d.ts @@ -1,19 +1,10 @@ -// Ambient declarations for probes in this directory. Each probe is compiled -// standalone by tsc, then submitted as the Actor's `code` input — entrypoint.sh -// splices that text into `export async function run(apify, console) { ... }` -// (see worker/usercode.d.ts), so `apify` and `console` are real function -// parameters at runtime, not globals. `declare global` here only satisfies the -// compiler for these standalone probe files; nothing in this file is emitted to JS. +// Ambient types for probes compiled into run(apify, console) bodies. import type { ApifyBinding } from '../worker/runner.js'; declare global { const apify: ApifyBinding; - // Sandbox-isolation probe checks `typeof process`/`typeof require` — both are - // genuinely absent at runtime (no nodejs_compat). `typeof` never throws on an - // undeclared identifier, so declaring these here doesn't change that: a - // `declare` emits no JS, so the runtime binding stays exactly as absent as - // the probe expects. + // Declared for typeof checks; workerd has no process or require globals. const process: unknown; const require: unknown; } diff --git a/tests/integration/harness.ts b/tests/integration/harness.ts index 6c76683..ef5dd04 100644 --- a/tests/integration/harness.ts +++ b/tests/integration/harness.ts @@ -1,10 +1,5 @@ -// Shared harness for integration tests that boot a REAL workerd instance against the actual -// compiled worker/runner.js + worker/guard.js (not a mock, not just the pure functions vitest's -// unit suite exercises in isolation). This is what closes the gap unit tests structurally -// cannot: whether guard.js is actually wired into the module graph, whether env.INTERNAL_API -// dispatch actually works, and whether the execution-limit safeguards actually fire end to end. -// Needs `pnpm build` to have produced worker/runner.js + worker/guard.js first (the "build" -// step in .github/workflows/typecheck.yml's integration job — see there). +// Integration harness for real workerd, module wiring, and env bindings. +// Run `pnpm build` first. import { spawn, type ChildProcess } from 'node:child_process'; import { createServer, type Server } from 'node:http'; import { mkdtempSync, writeFileSync, readFileSync, cpSync, rmSync } from 'node:fs'; @@ -17,19 +12,12 @@ const WORKERD_STARTUP_POLL_INTERVAL_MS = 100; const WORKERD_EXIT_TIMEOUT_MS = 2_000; function workerdBinaryPath(): string { - // Same resolution Dockerfile's builder stage uses: workerd ships its binary path via the - // package's own `default` export, one level of indirection because the platform-specific - // binary lives in an optional dependency (@cloudflare/workerd-linux-64 etc). `workerd` - // itself is a CommonJS package with no ESM entry point, hence createRequire rather than a - // static import. + // Match Dockerfile's workerd binary resolution. const require = createRequire(import.meta.url); return require('workerd').default; } -// Binds an OS-assigned ephemeral port on a throwaway listener, then releases it — the same -// "ask the OS for a free one" trick startMockApi() uses for its own port, reused here so -// workerd's own port isn't picked by guessing a range (see the historical note this replaces: -// a literal `10_000 + Math.random() * 10_000` had no collision retry). +// Ask the OS for a free port instead of guessing. async function reserveEphemeralPort(): Promise { const probe = createServer(); await new Promise((resolve) => probe.listen(0, '127.0.0.1', resolve)); @@ -39,19 +27,12 @@ async function reserveEphemeralPort(): Promise { return address.port; } -// A minimal stand-in for the platform-internal Apify API — just enough to make -// actor.start/dataset operations/pushOutput resolve, so a script's real behavior (including -// safeguard rejections, which happen before any HTTP call) is observable end to end. +// Minimal API stand-in for end-to-end runtime behavior. export interface MockApi { server: Server; port: number; requests: { method: string; path: string; body: string }[]; - /** - * Makes the NEXT `POST .../acts/:id/runs` request fail with the given status/body instead - * of succeeding — one-shot, cleared after it fires. Lets a test prove createRun()'s - * reservation rollback actually releases the run-count/budget slot on a real API - * rejection, not just on the happy path. - */ + /** Fail the next run-create request once. */ failNextRunCreate: (status: number, body: string) => void; close: () => Promise; } @@ -64,11 +45,7 @@ export async function startMockApi(): Promise { req.on('data', (chunk: Buffer) => chunks.push(chunk)); req.on('end', () => { const body = Buffer.concat(chunks).toString('utf8'); - // req.url is path+query (no scheme/host); parse against a throwaway base so - // route matching is on the PATH alone — matching the raw string (e.g. with - // `.endsWith('/runs')`) breaks the moment a real request carries a query string - // (createRun() always attaches one: waitForFinish/timeout/memory/maxTotalChargeUsd), - // silently falling through to the wrong response branch below. + // Match paths without query strings. const pathname = new URL(req.url ?? '/', 'http://mock-api.internal').pathname; requests.push({ method: req.method ?? '', path: req.url ?? '', body }); res.setHeader('content-type', 'application/json'); @@ -107,28 +84,23 @@ export async function startMockApi(): Promise { } export interface RunOptions { - /** Actor input fields beyond `code`, e.g. { maxActorRuns: 1 } — mirrors what entrypoint.sh reads. */ + /** Input fields beyond `code`. */ inputFields?: Record; - /** Called with the MockApi after it's started but before workerd boots — e.g. to arm failNextRunCreate(). */ + /** Configure the mock before workerd starts. */ beforeStart?: (mockApi: MockApi) => void; } export interface RunResult { - /** The pushed dataset item, or null if pushOutput never ran (e.g. workerd crashed at startup). */ + /** Pushed output, or null if output was never written. */ pushedItem: Record | null; - /** True if workerd itself started and served /health before we tore it down. */ + /** Whether workerd served /health. */ startedCleanly: boolean; - /** Raw stderr from the workerd process (useful for asserting startup-crash diagnostics). */ + /** Workerd stderr. */ stderr: string; mockApi: MockApi; } -// Boots a fresh workerd instance with `code` wrapped exactly like entrypoint.sh does, against a -// fresh MockApi standing in for the internal Apify API, sends one /run request, and tears both -// down — every acquired resource (mock server, workerd process, temp dir) is released on every -// path, including when workerd never becomes healthy or a step above throws. Mirrors -// entrypoint.sh's own env var wiring (CODE_RUNTIME_* for the execution limits) rather than -// reinventing a second convention. +// Boot workerd with wrapped code, run once, and release all resources. export async function runScript(code: string, options: RunOptions = {}): Promise { const mockApi = await startMockApi(); try { @@ -152,10 +124,7 @@ async function runInWorkDir(code: string, options: RunOptions, mockApi: MockApi, writeFileSync(join(workDir, 'usercode.js'), `export async function run(apify, console) {\n${code}\n}\n`); const port = await reserveEphemeralPort(); - // config.capnp's __PORT__ placeholder appears twice (once in a comment, once in the real - // socket address) — replaceAll, not replace, or the comment's occurrence "wins" and the - // real one is left as the literal string "__PORT__" (workerd then fails DNS-resolving it - // as a port/service name). + // Replace both config placeholders. const configTemplate = readFileSync(join(repoRoot, 'worker', 'config.capnp'), 'utf8'); writeFileSync(join(workDir, 'config.capnp'), configTemplate.replaceAll('__PORT__', String(port))); @@ -183,15 +152,15 @@ async function runInWorkDir(code: string, options: RunOptions, mockApi: MockApi, try { const res = await fetch(`http://127.0.0.1:${port}/health`); if (res.ok) { startedCleanly = true; break; } - } catch { /* not up yet, or crashed — keep polling until the deadline */ } - if (child.exitCode !== null) break; // crashed at startup, no point polling further + } catch { /* retry until timeout */ } + if (child.exitCode !== null) break; await new Promise((resolve) => setTimeout(resolve, WORKERD_STARTUP_POLL_INTERVAL_MS)); } if (startedCleanly) { try { await fetch(`http://127.0.0.1:${port}/run`, { method: 'POST' }); - } catch { /* the /run call itself may crash the worker — that's a result to assert on, not a harness failure */ } + } catch { /* assert worker failures from returned state */ } } const pushRequest = mockApi.requests.find((r) => { @@ -206,9 +175,7 @@ async function runInWorkDir(code: string, options: RunOptions, mockApi: MockApi, }; } finally { child.kill(); - // Wait for the process to actually exit (with a SIGKILL escalation) before returning — - // otherwise a workerd process slow to honor SIGTERM can still be running (and holding - // its port) when the next test in this file starts spawning its own. + // Ensure slow workerd processes cannot hold ports between tests. if (child.exitCode === null) { await Promise.race([ new Promise((resolve) => child.once('exit', () => resolve())), diff --git a/tests/integration/workerd-e2e.test.ts b/tests/integration/workerd-e2e.test.ts index be498c6..ebec9c3 100644 --- a/tests/integration/workerd-e2e.test.ts +++ b/tests/integration/workerd-e2e.test.ts @@ -1,14 +1,5 @@ -// Integration tests against a REAL workerd process running the actual compiled -// worker/runner.js + worker/guard.js — not a mock of guard.ts's functions (see -// tests/unit/guard.test.ts for that), but the real module graph, the real workerd request -// dispatch, and the real env.INTERNAL_API binding. This is what proves guard.js is actually -// wired in (not just correct in isolation) and that the execution-limit safeguards actually -// fire under real, concurrent use — both gaps a pure unit test structurally cannot close. -// -// Needs worker/runner.js + worker/guard.js to exist (`pnpm build` first — see -// .github/workflows/typecheck.yml's integration job). Slower and more flaky-prone than the -// unit suite (spawns a real process, real ports) — kept in its own directory/config so it can -// be run and reasoned about separately. +// End-to-end tests for real workerd, module wiring, and env bindings. +// Run `pnpm build` first; these tests spawn real processes and ports. import { describe, expect, it } from 'vitest'; import { runScript } from './harness.js'; @@ -27,13 +18,6 @@ describe('guard.js is actually enforced (not just correct in isolation)', () => expect(result.pushedItem?.stdout).not.toMatch(/LEAK/); }); - // The allow-path (a request TO apify.com actually going through) is deliberately NOT - // covered here: it would require either live internet access from the test runner (flaky, - // and not actually offline despite this suite's other claims) or mocking DNS/TLS for a real - // host, neither of which this harness does. tests/unit/guard.test.ts's "performs the - // request when the URL is allowed" case covers that path fully offline, against a mocked - // fetch — this file only needs to prove the DISALLOW path is wired into the real worker - // (see the previous test), which is the part a unit test can't reach. it('blocks WebSocket construction', async () => { const result = await runScript(` @@ -48,14 +32,13 @@ describe('guard.js is actually enforced (not just correct in isolation)', () => }); it('the PR #1 capability-theft PoC fails closed with no capability leak', async () => { - // Same payload as tests/fixtures/realfetch-escape.js: escapes the usercode.js wrapper - // into module scope and tries to call guard.js's (removed) claimRealFetch export. + // Module-scope capability-theft regression. const result = await runScript(` } globalThis.__stolenRealFetch = (await import('./guard.js')).claimRealFetch(); ;{ `); - expect(result.startedCleanly).toBe(false); // crashes at module eval, same as entrypoint.sh expects + expect(result.startedCleanly).toBe(false); expect(result.stderr).toMatch(/claimRealFetch is not a function/); }); }); @@ -75,10 +58,6 @@ describe('execution-limit safeguards fire under real (including concurrent) use' }); it('maxActorRuns blocks a run past the limit even when calls race concurrently', async () => { - // Regression test for the TOCTOU race: createRun() used to only record a started run - // AFTER its POST resolved, so N concurrent calls (this Actor's own documented "Bounded - // parallel fan-out" recipe) all read the pre-reservation count and all passed the - // check. createRun() now reserves synchronously before the first await. const result = await runScript(` const results = await Promise.allSettled( Array.from({ length: 5 }, () => apify.actor.start({ actorId: 'apify/hello-world' })), @@ -112,9 +91,6 @@ describe('execution-limit safeguards fire under real (including concurrent) use' }); it('a rejected actor.start() releases its reservation (rollback actually fires)', async () => { - // Regression test for createRun()'s try/catch rollback: without it, a run that fails - // AFTER being synchronously reserved (bad actorId, API rejection, ...) would - // permanently eat into maxActorRuns's budget for a run that never actually started. const result = await runScript(` let firstFailed = false; try { @@ -122,13 +98,11 @@ describe('execution-limit safeguards fire under real (including concurrent) use' } catch (e) { firstFailed = true; } - // If the failed attempt above wasn't rolled back, this would be blocked too - // (maxActorRuns: 1 already "spent" by the failed one). let secondSucceeded = false; try { await apify.actor.start({ actorId: 'apify/hello-world' }); secondSucceeded = true; - } catch (e) { /* would mean rollback didn't happen */ } + } catch (e) { /* expected only if rollback fails */ } console.log('firstFailed=' + firstFailed + ' secondSucceeded=' + secondSucceeded); `, { inputFields: { maxActorRuns: 1 }, diff --git a/tests/sandbox-isolation.ts b/tests/sandbox-isolation.ts index bf877df..00dcb2c 100644 --- a/tests/sandbox-isolation.ts +++ b/tests/sandbox-isolation.ts @@ -1,19 +1,5 @@ -// Sandbox isolation test. Submitted as the Actor's `code` input by test.sh and -// executed on the built Actor via `apify call`. Asserts the sandbox boundary -// holds and prints a sentinel line (ALL_TESTS_PASSED) that test.sh greps for. -// -// Guards github.com/apify/ai-team#216 (findings A, B, C): the isolate runs -// WITHOUT workerd's nodejs_compat, so user code cannot reach Node built-ins. -// That removes node:net (a raw-socket egress path that bypassed guard.js's -// *.apify.com fetch allowlist — finding A) and process.env (which held the -// run's APIFY_TOKEN — finding B), and makes the "no imports" docs accurate -// (finding C). fetch() and the apify binding must still work. -// -// `export {}` marks this file as its own ES module, so its top-level consts -// don't collide with the other probe's (both are type-checked in one tsc -// program) and top-level await below is legal. `pnpm build` strips this line -// post-compile (see package.json) -- left in, it would be a syntax error once -// spliced into the wrapping `async function run(apify, console) { ... }`. +// Actor probe for workerd isolation, egress guards, and binding access. +// `export {}` enables top-level await; build strips it before wrapping code. export {}; const results: boolean[] = []; @@ -27,70 +13,59 @@ function check(name: string, cond: boolean, detail = ''): void { } } -// Node built-ins must NOT be importable (no nodejs_compat). +// Node built-ins must be unavailable. for (const mod of ['node:net', 'node:fs', 'node:dns', 'node:child_process']) { let imported = false; try { await import(mod); imported = true; } catch { /* expected */ } check(`import ${mod} blocked`, !imported, imported ? 'IMPORTED (leak!)' : 'blocked'); } -// The run token lives in process.env under nodejs_compat; without it, process -// and require must be undefined so user code can't read the credential. +// No process or require means no token access through Node globals. check('process undefined', typeof process === 'undefined', `typeof process = ${typeof process}`); check('require undefined', typeof require === 'undefined', `typeof require = ${typeof require}`); -// fetch must remain — the apify binding depends on it. check('fetch available', typeof fetch === 'function', `typeof fetch = ${typeof fetch}`); -// guard.js allowlist: apify.com and *.apify.com only. A guard rejection throws -// synchronously with a "Blocked fetch" message BEFORE any network I/O; anything -// else (a real network/HTTP error) means guard let the request through. So we -// classify by the error message, not by whether the request ultimately succeeds. +// Classify guard rejection separately from network errors. async function guardBlocks(url: string): Promise { try { await fetch(url); - return false; // request went out — guard allowed it + return false; } catch (e) { - return /Blocked fetch/.test((e as Error).message); // guard rejection vs. network error + return /Blocked fetch/.test((e as Error).message); } } -// Allowed: the main domain and any subdomain must NOT be guard-blocked. for (const url of ['https://apify.com/', 'https://api.apify.com/v2/browser-info']) { check(`allow ${url}`, !(await guardBlocks(url)), 'not blocked by guard'); } -// Blocked: other public hosts, subdomain look-alikes, userinfo/host tricks, and -// the cloud metadata IP must all be guard-blocked. +// Block public-host look-alikes, URL tricks, and metadata IPs. const blockedTargets = [ - 'https://example.com/', // unrelated public host - 'https://evilapify.com/', // suffix without the dot — must not match .apify.com - 'https://apify.com.evil.com/', // real host is evil.com - 'https://apify.com@evil.com/', // userinfo trick — real host is evil.com - 'http://169.254.169.254/', // cloud link-local metadata + 'https://example.com/', + 'https://evilapify.com/', + 'https://apify.com.evil.com/', + 'https://apify.com@evil.com/', + 'http://169.254.169.254/', ]; for (const url of blockedTargets) { check(`block ${url}`, await guardBlocks(url), 'blocked by guard'); } -// Non-fetch egress primitives must be neutralized: WebSocket and EventSource -// are web-standard globals that connect directly (not through the fetch guard), -// so a script could otherwise open a wss:// / SSE channel to any public host and -// exfiltrate around the *.apify.com allowlist (apify/ai-team#216 finding A). +// WebSocket and EventSource must not bypass the fetch guard. function blocksConstruct(name: string, url: string): boolean { const Ctor = (globalThis as Record)[name]; - if (typeof Ctor !== 'function') return true; // absent → not an egress path + if (typeof Ctor !== 'function') return true; try { new (Ctor as new (u: string) => unknown)(url); - return false; // constructed → egress opened + return false; } catch (e) { - return /Blocked/.test((e as Error).message); // our guard rejection vs. any other error + return /Blocked/.test((e as Error).message); } } check('WebSocket blocked', blocksConstruct('WebSocket', 'wss://echo.websocket.org'), 'no wss egress'); check('EventSource blocked', blocksConstruct('EventSource', 'https://example.com/sse'), 'no SSE egress'); -// The apify binding must still work (fetch to *.apify.com). let bindingWorks = false; try { const found = await apify.store({ search: 'hello world', limit: 1 }); diff --git a/tests/unit/guard.test.ts b/tests/unit/guard.test.ts index 4fa1a2e..d94c5a2 100644 --- a/tests/unit/guard.test.ts +++ b/tests/unit/guard.test.ts @@ -1,14 +1,5 @@ -// Token-free, CI-runnable unit tests for worker/guard.ts's allowlist logic — no workerd, -// no `apify push`/`apify call`, no live Actor run. Fills the gap flagged in PR #1 review -// (2026-07-21): CI only ran `pnpm run typecheck`; every behavioral test required a live -// token, so `isAllowedHost`/`validateUrl`'s allowlist paths and `guardedFetch`'s redirect -// re-validation (the entire reason that function exists) had no test of any kind. -// -// guard.ts overrides `globalThis.fetch` as a side effect of being imported, and captures -// whatever `globalThis.fetch` was *at that moment* as its own internal `realFetch` (used by -// guardedFetch to perform the actual, pre-validated request). So: stub `globalThis.fetch` -// with a controllable mock BEFORE importing guard.ts, then call the exported `guardedFetch` -// directly — it runs against the mock, no network I/O, fully deterministic. +// Offline unit tests for URL validation, redirects, and builtin captures. +// Stub fetch before importing guard.ts because import installs the guard. import { describe, expect, it, vi, beforeAll } from 'vitest'; let guard: typeof import('../../worker/guard.js'); @@ -33,21 +24,17 @@ describe('isAllowedHost', () => { ['apify.com', true], ['api.apify.com', true], ['deeply.nested.apify.com', true], - ['APIFY.COM', true], // case-insensitive - ['apify.com.', true], // trailing FQDN dot stripped - ['evilapify.com', false], // suffix without the separating dot - ['apify.com.evil.com', false], // real host is evil.com + ['APIFY.COM', true], + ['apify.com.', true], + ['evilapify.com', false], + ['apify.com.evil.com', false], ['notapify.com', false], ['example.com', false], ])('%s -> %s', (hostname, expected) => { expect(guard.isAllowedHost(hostname)).toBe(expected); }); - // Regression test for a real bypass found in review: isAllowedHost used to call - // `hostname.toLowerCase()`/`.endsWith()` directly, resolving through the live, - // ordinary-script-writable `String.prototype` — no module-scope escape needed. See - // guard.ts's capture-block comment for the fix (capture the actual method functions, - // call them via .call() instead of value.method()). + // Regression test for String.prototype poisoning. it('still rejects a disallowed host after String.prototype.endsWith is poisoned', () => { const original = String.prototype.endsWith; // eslint-disable-next-line no-extend-native -- deliberately simulating the attack this test guards against @@ -93,12 +80,7 @@ describe('validateUrl', () => { expect(guard.validateUrl(new Request('https://apify.com/x')).hostname).toBe('apify.com'); }); - // Regression test for a real bypass found in review: validateUrl used to call the bare - // `new URL(...)`, which resolves whatever `globalThis.URL` currently is. A script that - // replaces it with a lying implementation (real .href, faked .hostname) could make - // validateUrl believe a disallowed host was apify.com, with no module-scope-escape trick - // needed at all — see guard.ts's capture-block comment for the fix (capture the real URL - // constructor before usercode.js can ever run). + // Regression test for URL constructor poisoning. it('still rejects a disallowed host after globalThis.URL is replaced with a lying constructor', () => { const OriginalURL = globalThis.URL; class LyingURL extends OriginalURL { @@ -160,7 +142,7 @@ describe('guardedFetch', () => { expect(response.status).toBe(200); expect(mockFetch).toHaveBeenCalledTimes(1); const [, init] = mockFetch.mock.calls[0]; - expect(init.redirect).toBe('manual'); // never lets the underlying fetch auto-follow + expect(init.redirect).toBe('manual'); }); it('follows a redirect to another allowed host', async () => { @@ -178,7 +160,7 @@ describe('guardedFetch', () => { mockFetch.mockClear(); mockFetch.mockResolvedValueOnce(redirectResponse('https://evil.com/steal', 302)); await expect(guard.guardedFetch('https://apify.com/start', undefined)).rejects.toThrow(/only apify\.com/); - expect(mockFetch).toHaveBeenCalledTimes(1); // never followed the malicious hop + expect(mockFetch).toHaveBeenCalledTimes(1); }); it('resolves a relative redirect Location against the current URL', async () => { @@ -202,12 +184,7 @@ describe('guardedFetch', () => { }); it('gives up after exactly MAX_REDIRECT_HOPS redirects to allowed hosts', async () => { - // MAX_REDIRECT_HOPS is module-private (not exported — see guard.ts's own comment on - // why nothing beyond the pure allowlist helpers is), so this pins the boundary by its - // observable effect instead: guard.ts's `hop > MAX_REDIRECT_HOPS` check means calls at - // hop 0..5 each make a real fetch (6 calls, MAX_REDIRECT_HOPS=5 + the initial request) - // before hop 6 throws without calling fetch again. A change to MAX_REDIRECT_HOPS's - // value, or an off-by-one in the `>` check, changes this exact count. + // Pins the redirect limit through observable request count. mockFetch.mockClear(); for (let i = 0; i < 10; i++) mockFetch.mockResolvedValueOnce(redirectResponse('https://apify.com/loop', 302)); await expect(guard.guardedFetch('https://apify.com/start', undefined)).rejects.toThrow(/exceeded 5 redirects/); @@ -224,12 +201,7 @@ describe('guardedFetch', () => { describe('module exports', () => { it('never exports a raw/unrestricted fetch capability', () => { - // Regression guard for PR #1's finding: guard.js must never export anything that - // hands the caller an unwrapped fetch function or a way to bypass the allowlist. - // Every export must be one of these known-safe, pure helpers — realObjectFreeze/ - // setHas/numberIsFinite/mathMin are safe by the same reasoning: each is still just - // the ordinary builtin operation, the concept itself carries no capability. See - // guard.ts's capture-block comment for why they need to be exported at all. + // Guard against exporting unrestricted fetch capabilities. const knownSafeExports = new Set([ 'isAllowedHost', 'validateUrl', 'nextRedirectInit', 'guardedFetch', 'realObjectFreeze', 'setHas', 'numberIsFinite', 'mathMin', 'realNumber', @@ -242,17 +214,11 @@ describe('module exports', () => { }); }); -// Regression tests for guard.ts's capture-block: capturing a builtin's *reference* only -// protects against `globalThis.X = somethingElse`. It does NOT protect a shared PROTOTYPE -// method or static function (`X.prototype.method = ...`, `Number.isFinite = ...`), which -// stays reachable through the still-live global name even if some OTHER code holds a -// captured constructor reference — a captured URL constructor's `.prototype` IS the same -// mutable object as the global `URL.prototype`. Each capture below needs its own resistance -// test; sharing one wouldn't prove the others are covered. +// Each captured builtin gets a poisoning regression test. describe('captured builtins resist prototype/static-method poisoning', () => { it('realObjectFreeze still freezes after the global Object.freeze is replaced with a no-op', () => { const original = Object.freeze; - Object.freeze = ((o: T) => o) as typeof Object.freeze; // simulates a hijacked global, not a real no-op call site + Object.freeze = ((o: T) => o) as typeof Object.freeze; try { const obj = guard.realObjectFreeze({ x: 1 }); expect(Object.isFrozen(obj)).toBe(true); @@ -283,7 +249,7 @@ describe('captured builtins resist prototype/static-method poisoning', () => { it('mathMin still returns the real minimum after the global Math.min is poisoned', () => { const original = Math.min; - Math.min = (a) => a; // always "returns the first argument", the wrong answer when a > b + Math.min = (a) => a; try { expect(guard.mathMin(5, 2)).toBe(2); } finally { @@ -293,8 +259,7 @@ describe('captured builtins resist prototype/static-method poisoning', () => { it('realNumber still coerces correctly after the global Number is poisoned', () => { const original = globalThis.Number; - // @ts-expect-error -- deliberately substituting an incompatible value to prove - // guard.ts's captured reference doesn't go through it. + // @ts-expect-error -- simulate a poisoned global. globalThis.Number = () => 999; try { expect(guard.realNumber('42')).toBe(42); @@ -305,7 +270,7 @@ describe('captured builtins resist prototype/static-method poisoning', () => { it('encodeUriComponent still escapes after the global encodeURIComponent is poisoned to a no-op', () => { const original = globalThis.encodeURIComponent; - globalThis.encodeURIComponent = (x) => String(x); // strips all escaping, e.g. lets '/'/'..' through + globalThis.encodeURIComponent = (x) => String(x); try { expect(guard.encodeUriComponent('../secret')).toBe('..%2Fsecret'); } finally { @@ -340,11 +305,7 @@ describe('captured builtins resist prototype/static-method poisoning', () => { Object.defineProperty(URL.prototype, 'hostname', { get: () => 'apify.com', configurable: true }); Object.defineProperty(URL.prototype, 'protocol', { get: () => 'https:', configurable: true }); try { - // Regression test for a real bypass found in review: validateUrl used to read - // `url.hostname`/`url.protocol` directly, resolving through URL.prototype's own - // accessors — poisonable the same way a prototype method is, no module-scope - // escape needed. See guard.ts's capture-block comment for the fix (capture the - // getter FUNCTION, invoke via .call(), never `value.property`). + // Regression test for URL accessor poisoning. expect(() => guard.validateUrl('http://example.com/')).toThrow(/only apify\.com/); } finally { Object.defineProperty(URL.prototype, 'hostname', originalHostname); diff --git a/tsconfig.json b/tsconfig.json index bf97d8d..fb3c19a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,10 +6,6 @@ "lib": ["es2022", "dom"], "strict": true }, - // tests/integration/*.ts is genuine Node (spawns workerd, uses fs/child_process) and - // typechecks separately under tsconfig.integration.json instead: this program's other - // files model the workerd/no-nodejs_compat environment (see tests/globals.d.ts, which - // declares `process`/`require` as absent) — @types/node's real ambient `process`/`require` - // globals would collide with that if both lived in one program. + // Integration tests use Node APIs and compile separately. "include": ["worker/*.ts", "tests/*.ts", "tests/unit/*.ts"] } diff --git a/vitest.config.ts b/vitest.config.ts index 8fbf17c..54b1f70 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,9 +1,6 @@ import { defineConfig } from 'vitest/config'; -// Only the .ts sources under tests/unit/ — without this, `pnpm build`'s tsc output leaves a -// compiled tests/unit/*.js next to each *.ts (gitignored build byproduct, same as tests/*.js -// for the probe fixtures), and vitest's own default file discovery picks up both, silently -// running every unit test twice under two different module instances. +// Exclude compiled test artifacts from Vitest discovery. export default defineConfig({ test: { include: ['tests/unit/**/*.test.ts'], diff --git a/vitest.integration.config.ts b/vitest.integration.config.ts index 957c46e..687cc1a 100644 --- a/vitest.integration.config.ts +++ b/vitest.integration.config.ts @@ -1,9 +1,6 @@ import { defineConfig } from 'vitest/config'; -// Separate from vitest.config.ts (the fast, pure-function unit suite): these tests spawn a -// real workerd process per case, so they're slower and need worker/runner.js + worker/guard.js -// already compiled (`pnpm build` first — see package.json's test:integration script and -// .github/workflows/typecheck.yml's integration job). +// Integration tests need compiled worker files and have longer timeouts. export default defineConfig({ test: { include: ['tests/integration/**/*.test.ts'], diff --git a/worker/config.capnp b/worker/config.capnp index 9814b82..f8df6cd 100644 --- a/worker/config.capnp +++ b/worker/config.capnp @@ -3,21 +3,12 @@ using Workerd = import "/workerd/workerd.capnp"; const config :Workerd.Config = ( services = [ (name = "main", worker = .codeRuntime), - # Ambient outbound for the sandboxed script's own fetch() calls (guard.js allowlists - # the hostname on top of this; this is a second, independent layer — restricted to - # "public" so that even a JS-level guard bug can't reach private/link-local addresses - # (e.g. cloud metadata) via a stolen or unwrapped fetch reference). + # User fetches: guard.js allowlists hosts; workerd blocks private addresses. (name = "internet", network = (allow = ["public"], tlsOptions = (trustBrowserCas = true))), - # This worker's own calls to the platform-internal Apify API (bound as env.INTERNAL_API - # in runner.ts, not exposed to the ambient/guarded fetch above). api.apify.com may - # resolve to a private address inside the platform network, hence the broader allowlist - # here — this service is never reachable from usercode.js: it's only ever passed as - # part of `env`, which workerd hands solely to the genuinely-dispatched - # `fetch(request, env)` call, never to any module-scope code. See runner.ts/guard.ts. + # Internal API binding; user code never receives this service. (name = "internalApi", network = (allow = ["public", "private", "local"], tlsOptions = (trustBrowserCas = true))), ], - # __PORT__ is substituted by entrypoint.sh from its own $PORT before workerd - # starts — single source of truth, see entrypoint.sh. + # entrypoint.sh substitutes __PORT__ before startup. sockets = [ ( name = "http", @@ -28,8 +19,7 @@ const config :Workerd.Config = ( ], ); -# runner.js is the entrypoint module; usercode.js is generated at container -# start by entrypoint.sh (the Actor input wrapped as `export async function run`). +# runner.js is the entrypoint; entrypoint.sh generates usercode.js. const codeRuntime :Workerd.Worker = ( modules = [ (name = "runner.js", esModule = embed "runner.js"), @@ -41,24 +31,16 @@ const codeRuntime :Workerd.Worker = ( (name = "DEFAULT_DATASET_ID", fromEnvironment = "ACTOR_DEFAULT_DATASET_ID"), (name = "DEFAULT_DATASET_ID_LEGACY", fromEnvironment = "APIFY_DEFAULT_DATASET_ID"), (name = "API_BASE_URL", fromEnvironment = "APIFY_API_BASE_URL"), - # This run's own meta.origin (e.g. "MCP" when apify-mcp-server started it), - # forwarded to sub-runs this script starts — see PARENT_ORIGIN in runner.ts. + # Forward this run's verified origin to sub-runs. (name = "PARENT_ORIGIN", fromEnvironment = "APIFY_META_ORIGIN"), - # Execution-level safeguards on Actor runs this script starts (all optional — - # see runner.ts's Limits and docs/API.md's "Execution limits"). Sourced from the - # Actor's own input fields, exported as env vars by entrypoint.sh. + # Optional execution limits from Actor input. (name = "MAX_ACTOR_RUNS", fromEnvironment = "CODE_RUNTIME_MAX_ACTOR_RUNS"), (name = "MAX_TOTAL_CHARGE_USD", fromEnvironment = "CODE_RUNTIME_MAX_TOTAL_CHARGE_USD"), (name = "DEFAULT_TIMEOUT_SECS", fromEnvironment = "CODE_RUNTIME_DEFAULT_TIMEOUT_SECS"), - # Unrestricted fetch to the platform-internal API, scoped to its own outbound network - # service above — see this file's "internalApi" service and runner.ts/guard.ts. + # Internal API fetch, scoped to internalApi. (name = "INTERNAL_API", service = "internalApi"), ], globalOutbound = "internet", compatibilityDate = "2026-01-15", - # No nodejs_compat: user code runs with web-standard APIs only. This is a - # security boundary, not a convenience toggle — the flag would expose node:net - # (a raw-socket egress path that bypasses guard.js's *.apify.com fetch allowlist) - # and process.env (which holds the run's APIFY_TOKEN). runner.js and guard.js use - # only web-standard APIs (fetch/URL/Response/Uint8Array), so they need nothing here. + # No nodejs_compat: blocks Node egress and process.env token access. ); diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh index fb0938a..c4714ee 100755 --- a/worker/entrypoint.sh +++ b/worker/entrypoint.sh @@ -1,9 +1,5 @@ #!/bin/sh -# Normal-mode (non-standby) Apify Actor entrypoint. Single-tenant per run: -# 1. read the Actor input and wrap its `code` into the runnable usercode.js module -# 2. boot workerd on loopback (it embeds runner.js + usercode.js) -# 3. trigger /run once, then exit -# workerd hosts the sandboxed worker and is reached only over loopback. +# Read input, start workerd, trigger one run, then exit. set -eu PORT=8787 @@ -21,8 +17,7 @@ if [ -z "${APIFY_TOKEN:-}" ] || [ -z "$STORE_ID" ] || [ -z "$DATASET_ID" ]; then exit 1 fi -# Fetch the Actor input and wrap its `code` into the runnable module. The `code` -# is inserted as code between the wrapper lines (not as a string) — no escaping. +# Insert input code directly into the generated module. INPUT_URL="${API_BASE}/v2/key-value-stores/${STORE_ID}/records/${INPUT_KEY}" input_status=$(curl -sS -o /tmp/input.json -w '%{http_code}' \ -H "Authorization: Bearer ${APIFY_TOKEN}" "$INPUT_URL") @@ -37,27 +32,18 @@ fi printf '\n}\n' } > /app/worker/usercode.js -# Execution-level safeguards (all optional Actor input fields — see runner.ts's Limits and -# docs/API.md's "Execution limits"). `// empty` yields an empty string (not "0"/"null") when -# the field is absent; runner.ts's parsePositiveNumberEnv treats a blank value as "no limit -# configured". +# Export optional execution limits; blank means unlimited. export CODE_RUNTIME_MAX_ACTOR_RUNS="$(jq -r '.maxActorRuns // empty' < /tmp/input.json)" export CODE_RUNTIME_MAX_TOTAL_CHARGE_USD="$(jq -r '.maxTotalChargeUsd // empty' < /tmp/input.json)" export CODE_RUNTIME_DEFAULT_TIMEOUT_SECS="$(jq -r '.defaultTimeoutSecs // empty' < /tmp/input.json)" -# config.capnp hardcodes __PORT__ as a placeholder so the port has one source ($PORT above). sed -i "s/__PORT__/${PORT}/" /app/worker/config.capnp /usr/local/bin/workerd serve --experimental /app/worker/config.capnp 2>"$WORKERD_STDERR" & workerd_pid=$! trap 'kill "$workerd_pid" 2>/dev/null || true' EXIT -# push_compile_failure reports a usercode.js syntax error as a normal, SUCCEEDED script -# result (same contract as a script that throws at runtime) instead of failing the whole -# Actor run. usercode.js wraps the user's `code` inside `export async function run(...) { -# ... }` with nothing else at module scope, so nothing in it can fail to *parse* except -# that inserted code — a startup crash naming usercode.js is therefore always a syntax -# error in the user's script, never our own code. See detection below. +# Report user-code compile errors as diagnostic output. push_compile_failure() { crash_log=$(cat "$WORKERD_STDERR" 2>/dev/null || true) echo "[code-runtime] usercode.js failed to compile: $crash_log" >&2 @@ -94,7 +80,5 @@ until curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; do sleep 0.1 done -# Trigger the single run. The worker runs the program and pushes { stdout, stderr, -# exitCode, statusMessage } to the default dataset. A non-2xx response fails the Actor -# run (curl -f). +# Trigger one run; non-2xx fails the Actor run. curl -fsS -X POST "http://127.0.0.1:${PORT}/run" diff --git a/worker/guard.ts b/worker/guard.ts index 4d17e6e..8482964 100644 --- a/worker/guard.ts +++ b/worker/guard.ts @@ -1,52 +1,10 @@ -// Restrict the user program's outbound network to apify.com and its subdomains. -// Imported before usercode.js so the overrides are in place even for code that -// runs at module-evaluation time. -// -// Egress surface (workerd, no nodejs_compat): the only JS-reachable outbound -// primitives are fetch, WebSocket, and EventSource. Raw sockets (node:net, -// cloudflare:sockets connect()) need module imports, which are already blocked. -// fetch is allowlisted below; WebSocket and EventSource are removed outright -// because runner.js and the apify binding never use them — leaving them would -// be a non-fetch egress path around the allowlist (apify/ai-team#216 finding A, -// via WebSocket). If a future need arises, wrap them like fetch instead. -// -// This module used to also hand runner.js an unrestricted "real fetch" for its own -// internal API calls via a pair of exports. That capability-through-export design was -// broken: anything in usercode.js's module scope can `import('./guard.js')` too (ES -// modules have no notion of a "trusted" importer), so user code could call the same -// exports runner.js did and steal the capability before runner.js's own use of it. The fix -// moves runner.js's internal API access off of any module export entirely and onto -// workerd's own env binding (`INTERNAL_API` in config.capnp, wired to a separate outbound -// network service — see there). `env` is a parameter workerd hands only to the -// genuinely-dispatched `fetch(request, env)` call; nothing at module-evaluation time -// (including an escaped top-level statement in usercode.js) ever receives a reference to -// it, so there is nothing here for user code to import or steal. +// Guard user fetches to apify.com and its subdomains. +// This module loads before usercode.js, including module-scope code. +// Internal API access uses env.INTERNAL_API, never an exported capability. const realFetch = globalThis.fetch.bind(globalThis); -// Every name below is captured HERE, at guard.js's own module-evaluation time — which -// always finishes before usercode.js's module body ever runs (see runner.ts's import-order -// comment) — because an ordinary script, no module-scope escape needed, can reassign or -// monkey-patch any JS builtin this file's (or runner.ts's) security decisions depend on. -// Three attack shapes, all closed the same way (capture the real thing before a script -// gets the chance to touch it): -// - Reassigning the global itself (`globalThis.URL = FakeClass`, -// `globalThis.encodeURIComponent = x => x`) — defeated by capturing a direct reference. -// - Poisoning a shared PROTOTYPE method or static function (`String.prototype.endsWith = -// () => true`, `Set.prototype.has = () => true`, `Number.isFinite = () => true`) — a -// captured *constructor* reference does NOT protect this (`RealURL.prototype` IS -// `URL.prototype`, the same mutable object the still-live global name reaches). Fixed -// by capturing the METHOD/FUNCTION itself and invoking it directly -// (`stringEndsWith.call(host, suffix)`), never through the poisonable `value.method()`. -// - Poisoning a shared PROTOTYPE ACCESSOR/getter (`Object.defineProperty(URL.prototype, -// 'hostname', { get: () => 'apify.com' })`) — same fix, one level removed: capture the -// getter FUNCTION and invoke it via `.call(instance)` instead of reading -// `instance.property`. -// This block is the whole audit surface: anything guard.ts or runner.ts uses to make a -// security/allowlist/ownership/budget decision belongs here, not called bare. Multiple real -// bypasses of exactly this shape were found in review before this was made systematic. When -// adding a new capture here, follow the plain descriptive name already used below (not the -// older `real`+Name scheme on the first two, kept as-is to avoid unrelated call-site churn), -// and add a poisoning-regression test in tests/unit/guard.test.ts mirroring the existing ones. +// Capture security-sensitive builtins before usercode.js can replace globals, +// prototype methods, static functions, or accessors. Add a poisoning test with each capture. const RealURL = globalThis.URL; export const realObjectFreeze: typeof Object.freeze = Object.freeze.bind(Object); const stringToLowerCase = String.prototype.toLowerCase; @@ -69,23 +27,13 @@ const responseStatusGetter = Object.getOwnPropertyDescriptor(Response.prototype, export const responseOk = (response: Response): boolean => responseOkGetter.call(response); export const responseStatus = (response: Response): number => responseStatusGetter.call(response); -// Exported so runner.ts's own internal-API URL building (buildUrl in runner.ts) uses the -// same captured, un-hijackable constructor this file's own allowlist relies on — a second, -// independent `new URL(...)` call site is just as reachable/poisonable as this file's own, -// and runner.ts's version builds the URL for the unrestricted, token-bearing internal API -// call, making it the higher-severity of the two if missed. +// runner.ts also uses this captured constructor for token-bearing API URLs. export { RealURL }; -// Match apify.com exactly or any subdomain. The leading dot in the suffix is -// what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` -// (ends with `.evil.com`) both fail. Uses the captured string-method references above (see -// this file's capture block), not `hostname.toLowerCase()`/`.endsWith()` directly — those -// resolve through the live, poisonable `String.prototype` at call time. +// The leading dot rejects look-alikes such as evilapify.com. export function isAllowedHost(hostname: string): boolean { const lowercased: string = stringToLowerCase.call(hostname); - // Strip a trailing FQDN dot (`apify.com.` -> `apify.com`) via slice, not `.replace(/\.$/, '')` - // — same captured-primitive reasoning as everything else in this block, one fewer method - // to capture. + // Normalize a trailing FQDN dot without using another mutable method. const host: string = stringEndsWith.call(lowercased, '.') ? stringSlice.call(lowercased, 0, -1) : lowercased; return host === 'apify.com' || stringEndsWith.call(host, '.apify.com'); } @@ -93,26 +41,20 @@ export function isAllowedHost(hostname: string): boolean { function requestUrl(input: RequestInfo | URL): string { if (typeof input === 'string') return input; if (input instanceof RealURL) return input.href; - if (input && typeof input.url === 'string') return input.url; // Request + if (input && typeof input.url === 'string') return input.url; return String(input); } -// Parses and validates one URL against the allowlist. Returns the parsed URL -// (callers use it to resolve a relative redirect Location) or throws. +// Parse and validate one URL; callers use the result for relative redirects. export function validateUrl(input: RequestInfo | URL): URL { let url: URL; try { - // Parse to the real host — defeats userinfo (`apify.com@evil.com`), - // path/query/fragment (`evil.com/apify.com`) and similar tricks. Uses RealURL - // (see the capture block above), not the bare global, so a hijacked globalThis.URL - // can't lie here. + // RealURL defeats userinfo and path tricks, and cannot be hijacked by user code. url = new RealURL(requestUrl(input)); } catch { throw new Error('Blocked fetch: only absolute http(s) URLs to apify.com are allowed'); } - // Read via the captured getters (see the capture block above), not `url.protocol`/ - // `url.hostname` directly — those resolve through URL.prototype's own accessors, which - // are poisonable the same way a prototype method is (RealURL.prototype IS URL.prototype). + // Captured getters prevent URL.prototype poisoning. const protocol = urlProtocol(url); if (protocol !== 'https:' && protocol !== 'http:') { throw new Error(`Blocked fetch: protocol "${protocol}" is not allowed`); @@ -124,12 +66,8 @@ export function validateUrl(input: RequestInfo | URL): URL { return url; } -// fetch() follows redirects internally by default, invisibly to a wrapper that -// only checks the initial URL — an allowlisted host could 302 to anywhere. -// Follow redirects ourselves, one hop at a time, and re-validate each Location -// against the allowlist before following it. Status-to-method mapping matches -// the WHATWG fetch spec: 303 always downgrades to GET; 301/302 downgrade to -// GET only when the original method was POST; 307/308 preserve method + body. +// Follow redirects manually so every Location passes the allowlist. +// Preserve WHATWG method/body behavior for 301/302/303/307/308. const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); const MAX_REDIRECT_HOPS = 5; @@ -140,9 +78,7 @@ export function nextRedirectInit(init: RequestInit | undefined, status: number): return { ...init, method: 'GET', body: undefined }; } -// `hop` is an internal recursion counter, not part of the public contract — kept unexported -// so a caller (including escaped usercode.js, which can import and call any export of this -// module) can't pass a pre-inflated or negative value to defeat MAX_REDIRECT_HOPS. +// Keep hop private so callers cannot bypass the redirect limit. async function guardedFetchHop(input: RequestInfo | URL, init: RequestInit | undefined, hop: number): Promise { if (hop > MAX_REDIRECT_HOPS) { throw new Error(`Blocked fetch: exceeded ${MAX_REDIRECT_HOPS} redirects`); @@ -152,8 +88,8 @@ async function guardedFetchHop(input: RequestInfo | URL, init: RequestInit | und const status = responseStatus(response); if (!setHas(REDIRECT_STATUSES, status)) return response; const location = response.headers.get('location'); - if (!location) return response; // redirect status with no Location: nothing to follow - const nextUrl = new RealURL(location, url); // resolves a relative Location against the current URL + if (!location) return response; + const nextUrl = new RealURL(location, url); return guardedFetchHop(nextUrl.href, nextRedirectInit(init, status), hop + 1); } @@ -161,10 +97,7 @@ export function guardedFetch(input: RequestInfo | URL, init: RequestInit | undef return guardedFetchHop(input, init, 0); } -// writable:false + configurable:false, matching blockGlobal() below — a plain -// assignment could be overwritten or deleted by the sandboxed script to -// recover the ambient (real, unrestricted) fetch reference some engines -// expose under a different name; locking it closes that off. +// Lock fetch so user code cannot restore an unrestricted reference. Object.defineProperty(globalThis, 'fetch', { value: (input: RequestInfo | URL, init?: RequestInit) => guardedFetch(input, init), writable: false, @@ -172,10 +105,7 @@ Object.defineProperty(globalThis, 'fetch', { enumerable: true, }); -// Remove the non-fetch egress primitives. These are web-standard globals present -// even without nodejs_compat, and they connect directly (not through the fetch -// guard), so a script could otherwise open a wss:// or SSE connection to any -// public host and exfiltrate data around the *.apify.com allowlist. +// Remove direct WebSocket and EventSource egress around the fetch guard. function blockGlobal(name: string): void { const blocked = function () { throw new Error(`Blocked ${name}: only fetch() to apify.com and its subdomains is allowed`); diff --git a/worker/runner.ts b/worker/runner.ts index 239da89..1128601 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -1,47 +1,7 @@ -// Single worker for the per-run code-runtime Actor. It runs the user's program -// (imported from the generated `usercode.js` module) with the `apify` REST -// binding and a captured `console`, then pushes `{ stdout, stderr, exitCode, -// statusMessage }` to the run's default dataset. The container entrypoint -// generates `usercode.js`, boots workerd, and triggers `/run` once. -// -// Single-tenant: one run = one container = one program = one token. No Worker -// Loader / per-request isolate is needed — the program runs in this worker, -// which is itself the sandbox (no filesystem, restricted outbound network). -// guard.js overrides globalThis.fetch to allow only apify.com. -// -// This worker's own (internal) API calls need an unrestricted, un-allowlisted -// fetch — the internal API is a private IP, not *.apify.com. That capability -// is bound as `env.INTERNAL_API` (config.capnp), a workerd service binding -// wired to a separate outbound network, not a shared/exported fetch reference. -// `env` is only ever handed to the genuinely-dispatched `fetch(request, env)` -// call below by workerd's own runtime — nothing at module-evaluation time -// (including attacker-controlled top-level code that escapes usercode.js's -// wrapper, see entrypoint.sh) ever receives a reference to it, so there is -// nothing for user code to import or steal. See guard.ts for why an -// export-based handoff (this worker's previous design) could not be made -// sound: usercode.js shares guard.js's module graph, so any function guard.js -// exported was equally callable by escaped user code. -// -// guard.js MUST be the first import in this file, whether or not a name is bound from it. -// Import declarations are hoisted and dependencies evaluate in the order first -// encountered; guard.js has to install its fetch/WebSocket/EventSource overrides (and -// capture RealURL/realObjectFreeze — see guard.ts) before usercode.js's module body runs, -// including any attacker-controlled top-level statement that escapes usercode.js's -// wrapper (see entrypoint.sh). Reversing this order (or dropping the import) silently -// makes the entire allowlist dead code — `globalThis.fetch` stays the real, unrestricted -// fetch for every script this Actor runs. tests/sandbox-isolation.ts is the regression -// test for this: it fails loudly (`allow https://example.com/` check reports NOT blocked) -// if this import is ever missing or reordered. -// -// Everything imported below is one of guard.js's own pre-captured builtin references (see -// the capture block at the top of guard.ts for the full reasoning): this file's own -// top-level code, and every function it defines, run AFTER usercode.js's module body -// (import order), so a script can shadow/poison any of these globals — or their prototypes/ -// accessors — before this file ever uses them, unless it uses guard.js's captured -// equivalents instead of calling them bare. This is exhaustive: every builtin this file's -// own security decisions (redirect/ownership/budget checks) or its internal-API request -// construction (URL building, path-segment escaping, request/response bodies) depends on is -// imported from here, never called bare. +// Run usercode.js in workerd, expose the Apify binding, and push output. +// guard.js must load first to install egress guards before user code runs. +// Internal API access uses env.INTERNAL_API; user code never receives env. +// Security-sensitive builtins come from guard.js captures. import { realObjectFreeze, setHas, numberIsFinite, mathMin, realNumber, encodeUriComponent, jsonStringify, responseOk, RealURL, @@ -52,16 +12,10 @@ type Fetcher = { fetch(input: RequestInfo | URL, init?: RequestInit): Promise { items: T[]; count: number; @@ -143,9 +96,7 @@ interface DatasetItemsPage extends ItemsPage { desc: boolean; } -// Awaiting this value resolves to one page (ItemsPage); `for await`-ing it auto-paginates -// through every item, one at a time. Same dual nature as apify-client's own PaginatedIterator -// (one call, one name, two ways to consume it) — see makePaginatedList for the mechanism. +// Await for one page; use for-await to consume all pages. type PaginatedItems> = Promise & AsyncIterable; interface DatasetSchemaOptions { @@ -196,23 +147,13 @@ interface Env { DEFAULT_DATASET_ID?: string; DEFAULT_DATASET_ID_LEGACY?: string; API_BASE_URL?: string; - // APIFY_META_ORIGIN, forwarded from the platform's own env var of the same - // name (bound as PARENT_ORIGIN in config.capnp). Reflects this run's own - // meta.origin, set by apify-core from the X-Apify-Request-Origin request - // header the caller sent when creating THIS run — 'MCP' when apify-mcp-server - // started it. Platform-injected, not user-settable: unlike an Actor input - // field, a script running inside this Actor cannot spoof it. + // Platform-verified origin; user input cannot spoof it. PARENT_ORIGIN?: string; - // Execution-level safeguards on Actor runs a script starts via - // actor.start/call/callAndGetItems — see makeApifyBinding's `Limits` and - // docs/API.md's "Execution limits" section. All optional; unset means - // "no limit beyond the Apify API's own defaults". + // Optional limits for runs started by user code. MAX_ACTOR_RUNS?: string; MAX_TOTAL_CHARGE_USD?: string; DEFAULT_TIMEOUT_SECS?: string; - // Unrestricted (non-allowlisted) fetch for this worker's own calls to the - // platform-internal Apify API. Bound to a separate outbound network - // service in config.capnp — see this file's header comment and guard.ts. + // Separate network binding for internal API calls. INTERNAL_API: Fetcher; } @@ -223,8 +164,6 @@ interface OutputItem { statusMessage: string; } -// --------------------------------------------------------------------------- - function stringify(x: unknown): string { if (typeof x === 'string') return x; try { return JSON.stringify(x); } catch { return String(x); } @@ -238,21 +177,7 @@ function errorDetail(err: unknown): string { return err instanceof Error && err.stack ? err.stack : errorMessage(err); } -// Wraps a page-fetcher into a value that's both a Promise (awaits to the first page) and an -// AsyncIterable (walks every page, yielding one item at a time) — the same dual nature as -// apify-client's own PaginatedIterator, implemented via the same trick it uses: attach -// Symbol.asyncIterator to a live Promise object (a Promise is a plain object at runtime, so -// this is legal, no class or wrapper needed). -// -// One shared implementation, unlike apify-client itself, which independently re-implements -// this exact trick three times (dataset items, key-value-store keys, request-queue requests) -// with three subtly different cursor/offset conventions — see docs/API.md's Conventions -// section for that comparison. Every paginated method in this binding goes through this one -// function instead. -// -// Continuation stops the same way the old dataset.iterate() did: a page shorter than the -// limit it was asked for is the natural end-of-data signal (no dataset `total` is trusted — -// see the listItems comment below for why). +// Return one page as a Promise and all pages through async iteration. function makePaginatedList>( fetchPage: (offset: number, limit: number | undefined) => Promise, offset: number, @@ -276,21 +201,12 @@ function makePaginatedList>( }) as PaginatedItems; } -// 'MCP' matches apify-core's META_ORIGINS.MCP / apify-mcp-server's own -// X-Apify-Request-Origin header value — reusing the platform's existing -// convention rather than inventing a new one. +// Matches apify-core's existing MCP request-origin value. const MCP_ORIGIN = 'MCP'; const REQUEST_ORIGIN_HEADER = 'X-Apify-Request-Origin'; -// Execution-level safeguards on Actor runs a script starts (actor.start/call/ -// callAndGetItems, which all funnel through createRun() below). Independent of -// any single run's own timeoutSecs/waitForFinishSecs/maxTotalChargeUsd, which -// only bound THAT run — nothing previously bounded how many runs one script -// could start, or their combined cost. See docs/API.md's "Execution limits". +// Limits apply across runs started by one script. interface Limits { - // Always constructed with all three keys present (the fetch handler below never omits - // one) — `| undefined` documents "may be undefined" without implying a caller can leave - // the key out entirely, per this codebase's own `?` vs `| undefined` convention. maxActorRuns: number | undefined; maxTotalChargeUsd: number | undefined; defaultTimeoutSecs: number | undefined; @@ -303,20 +219,13 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: internalFetch: Fetcher['fetch']; limits: Limits; }) { - // Every request this Actor makes identifies itself; requests made while THIS - // run's own origin is MCP additionally forward that origin so runs started by - // apify.actor.start/call/callAndGetItems() below get meta.origin: 'MCP' too, - // instead of the platform's default meta.origin: 'ACTOR' for actor-to-actor - // calls. Gated on parentOrigin (verified server-side, see the Env.PARENT_ORIGIN - // comment) rather than any Actor input, so a script can't forge an origin this - // run wasn't actually started with. + // Forward MCP origin only when platform metadata verified it. const baseHeaders: Record = { Authorization: `Bearer ${token}`, 'User-Agent': 'apify-code-runtime', ...(parentOrigin === MCP_ORIGIN ? { [REQUEST_ORIGIN_HEADER]: MCP_ORIGIN } : {}), }; - // Build a URL with optional query params; null/undefined values are dropped. const buildUrl = (path: string, searchParams?: SearchParams): URL => { const url = new RealURL(`${apiV2}${path}`); if (searchParams) { @@ -327,8 +236,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: return url; }; - // Single-source HTTP wrapper. Throws on non-2xx with the response body in the message. - // `body`: string / Uint8Array passed through; objects are JSON.stringify'd. + // Shared HTTP wrapper; errors include response bodies. const apiCall = async (method: string, path: string, options: ApiCallOptions = {}): Promise => { const { searchParams, body, contentType } = options; const headers: Record = { ...baseHeaders }; @@ -350,58 +258,29 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: return response; }; - // The Apify API's JSON envelope (`{ data: ... }`) carries whatever shape the endpoint - // returns; there's no schema to check it against here, so this stays honestly `any` - // rather than asserting a shape we haven't verified. + // API envelopes are untyped because endpoint shapes vary. const apiJson = async (method: string, path: string, options?: ApiCallOptions): Promise => (await apiCall(method, path, options)).json(); const apiData = async (method: string, path: string, options?: ApiCallOptions): Promise => (await apiJson(method, path, options)).data; - // Run IDs this script itself started, via actor.call() / actor.start() (and transitively - // actor.callAndGetItems(), which shares createRun() below). run.abort() below is scoped to - // this set — a script can only abort runs it started, not any account-wide runId it's - // handed or guesses. Also used by abortTrackedRuns() (called from the top-level exception - // handler below) to clean up runs still going when the script itself crashed. + // Scope run.abort() and crash cleanup to runs started by this script. const startedRunIds = new Set(); - // Gates nonTerminalRunIds membership: createRun() (below) adds a run's id here unless its - // status is already one of these; waitForFinish() removes it once the status becomes one - // of these. A status in this set means this script no longer needs to track (and - // therefore abortTrackedRuns(), below, no longer needs to abort) that run. Deliberately - // broader than the API's own terminal-status set (docs/API.md's - // SUCCEEDED/FAILED/ABORTED/TIMED-OUT) by one: ABORTING means a run is already mid-abort - // (e.g. this script already called run.abort() on it), so re-aborting it would just be a - // redundant API call, not a real cleanup action. + // ABORTING needs no second abort; unknown statuses stay tracked conservatively. const DONE_TRACKING_STATUSES = new Set(['SUCCEEDED', 'FAILED', 'ABORTED', 'ABORTING', 'TIMED-OUT']); const nonTerminalRunIds = new Set(); - // Conservative execution-level cost cap: each run's OWN maxTotalChargeUsd is a ceiling, - // not a bill, so this tracks committed ceilings (not realized spend) against - // limits.maxTotalChargeUsd and never lets a script authorize more combined ceiling than - // that budget, even though actual spend will usually be lower. + // Track committed run ceilings, not realized spend. let committedChargeUsd = 0; - // Separate from startedRunIds.size: reserved synchronously (see createRun below) so two - // concurrent createRun() calls — e.g. docs/API.md's own "Bounded parallel fan-out" recipe, - // `Promise.all(batch.map(() => apify.actor.start(...)))` — can't both read the - // pre-reservation count before either's POST resolves and both pass the cap check. + // Reserve synchronously so concurrent starts cannot bypass maxActorRuns. let reservedRunCount = 0; - // POST /acts/:id/runs, shared by actor.call() (start+wait, waitForFinishSecs defaults to - // DEFAULT_WAIT_FOR_FINISH_SECS, capped at 60s per the Apify API — for longer runs use - // start() + apify.run.waitForFinish()) and actor.start() (async kickoff, no wait). Returns - // the run record so the caller can read defaultDatasetId / defaultKeyValueStoreId. - // Intentionally does NOT use /run-sync, which returns the OUTPUT KVS record (a pattern - // only some Actors follow) rather than the structured run record. + // Start one Actor run; actor.call() and actor.start() share this path. const createRun = async ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }: StartOptions): Promise => { if (limits.maxActorRuns !== undefined && reservedRunCount >= limits.maxActorRuns) { throw new Error(`Blocked actor run: this script already started/is starting ${reservedRunCount} Actor run(s), the configured limit is ${limits.maxActorRuns}`); } - // Reject a malformed script-supplied maxTotalChargeUsd before it ever reaches - // committedChargeUsd's arithmetic below: NaN in particular is silently absorbing — - // `NaN - anything` and `anything - NaN` both stay NaN, so a single bad call would - // permanently corrupt the running total and (since `NaN <= 0` is false) defeat the - // whole execution-level budget check for the rest of the script, with no way to - // recover it via the catch block's rollback (subtracting NaN from NaN is still NaN). + // Reject non-finite or non-positive caps before budget arithmetic. if (maxTotalChargeUsd !== undefined && !(numberIsFinite(maxTotalChargeUsd) && maxTotalChargeUsd > 0)) { throw new Error(`Invalid maxTotalChargeUsd: ${maxTotalChargeUsd} (must be a finite number greater than 0)`); } @@ -411,13 +290,10 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: if (remaining <= 0) { throw new Error(`Blocked actor run: execution spending budget of $${limits.maxTotalChargeUsd} is exhausted`); } - // A run without its own cap could spend the whole remaining budget; a run with - // its own cap higher than what's left gets clamped down to what's left. + // Uncapped runs consume the remainder; larger caps are clamped. effectiveMaxCharge = effectiveMaxCharge === undefined ? remaining : mathMin(effectiveMaxCharge, remaining); } - // Reserve BEFORE the network round-trip below (nothing here awaits yet, so this runs - // to completion in one synchronous tick relative to any other createRun() call — see - // the comment on reservedRunCount above for why that matters). + // Reserve before the network request to close the concurrency race. reservedRunCount += 1; if (effectiveMaxCharge !== undefined) committedChargeUsd += effectiveMaxCharge; try { @@ -435,9 +311,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: if (!setHas(DONE_TRACKING_STATUSES, runRecord.status)) nonTerminalRunIds.add(runRecord.id); return runRecord; } catch (err) { - // The reservation never became a real run — release it, so a failed attempt - // (bad actorId, network error, ...) doesn't permanently eat into the script's - // run-count/budget allowance. + // Failed requests release their reservations. reservedRunCount -= 1; if (effectiveMaxCharge !== undefined) committedChargeUsd -= effectiveMaxCharge; throw err; @@ -448,22 +322,11 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: get: ({ actorId }: ActorIdOptions): Promise => apiData('GET', `/acts/${encodeUriComponent(actorId)}`), - // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether - // waitForFinish is set. Records the created run's ID in startedRunIds so run.abort() - // can be scoped to runs this script itself started (see the run.abort definition below). call: (opts: StartOptions): Promise => createRun({ waitForFinishSecs: DEFAULT_WAIT_FOR_FINISH_SECS, ...opts }), - // Async kickoff. Returns immediately with a run record in READY/RUNNING state. start: (opts: StartOptions): Promise => createRun(opts), - // Runs an Actor (same as call(), waitForFinishSecs defaults to DEFAULT_WAIT_FOR_FINISH_SECS) - // and returns its dataset items in one call. Calls createRun() directly rather than - // through `actor.call()` — same underlying request, no self-reference to `actor` needed. - // - // If the run is still RUNNING when the wait elapses, this reads whatever the dataset - // holds at that moment — items may be empty or a partial subset of the eventual total. - // Check `run.status` (returned alongside `items`); a non-terminal status means the - // items are a snapshot, not the final result — see docs/API.md. + // Start, wait up to 60 seconds, and read a dataset snapshot. callAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { const runRecord = await createRun({ actorId, input, waitForFinishSecs: DEFAULT_WAIT_FOR_FINISH_SECS, ...runOpts }); const { items } = await dataset.listItems({ @@ -477,8 +340,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: get: ({ runId }: RunIdOptions): Promise => apiData('GET', `/actor-runs/${encodeUriComponent(runId)}`), - // Block until the run terminates or `waitForFinishSecs` elapses (whichever comes first). - // The Apify API caps this at 60s per request; longer waits require a polling loop. + // Wait for termination or the API's 60-second cap. waitForFinish: async ({ runId, waitForFinishSecs = DEFAULT_WAIT_FOR_FINISH_SECS }: WaitOptions): Promise => { const runRecord: RunRecord = await apiData('GET', `/actor-runs/${encodeUriComponent(runId)}`, { searchParams: { waitForFinish: waitForFinishSecs }, @@ -487,9 +349,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: return runRecord; }, - // Scoped to runs this script itself started (see startedRunIds above) — without this, - // any runId a script is handed (e.g. read from a dataset item, or guessed) could abort - // an unrelated, account-wide run. + // Only runs started by this script may be aborted. abort: ({ runId }: RunIdOptions): Promise => { if (!setHas(startedRunIds, runId)) { throw new Error(`Blocked run.abort: "${runId}" was not started by this script`); @@ -498,8 +358,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: return apiData('POST', `/actor-runs/${encodeUriComponent(runId)}/abort`); }, - // Returns the full run log as text. `limit` tails the last N characters; the Apify API - // does not paginate logs, so this is a client-side slice (the full body is fetched). + // Return the full log, optionally limited to its last N characters. getLog: async ({ runId, limit }: GetLogOptions): Promise => { const response = await apiCall('GET', `/logs/${encodeUriComponent(runId)}`); const text = await response.text(); @@ -508,14 +367,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: }; const dataset = { - // `await` resolves to one page: { items, count, offset, limit, desc }. `for await` - // auto-paginates through the entire dataset, one item at a time (replaces the old, - // separate dataset.iterate() method — see makePaginatedList). offset/limit/desc echo - // back what the API actually applied (read from its x-apify-pagination-* response - // headers, not just the request), except `total`: the Apify API's - // `x-apify-pagination-total` header is unreliable for freshly-created datasets - // (eventually consistent), so it's never surfaced — `count` (this page's actual item - // count) is what you want instead. Use `inferFields` if you need an approximate total. + // Await for one page; for-await iterates all pages. listItems: ({ datasetId, fields, omit, limit, offset = 0, clean, desc }: DatasetListOptions): PaginatedItems => { const fetchPage = async (pageOffset: number, pageLimit?: number): Promise => { const response = await apiCall('GET', `/datasets/${encodeUriComponent(datasetId)}/items`, { @@ -540,9 +392,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: return makePaginatedList(fetchPage, offset, limit); }, - // Apify has no dedicated schema endpoint; we infer one from a small sample of items. - // Named inferFields (not getSchema) to avoid colliding with the Actor's own *declared* - // dataset schema (a different concept, described in this Actor's own actor.json). + // Infer fields from a sample because the API has no schema endpoint. inferFields: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { const meta = await apiData('GET', `/datasets/${encodeUriComponent(datasetId)}`); const { items } = await dataset.listItems({ datasetId, limit: sample }); @@ -574,9 +424,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: }; const keyValueStore = { - // Returns the value directly (parsed when JSON, string when text/*, Uint8Array otherwise). - // Returns null when the key does not exist (404), not an error — this matches the common - // "lookup or default" pattern in code. + // Parse JSON/text values; return null for a missing key. get: async ({ storeId, key }: KeyValueStoreGetOptions): Promise => { const response = await internalFetch(buildUrl(`/key-value-stores/${encodeUriComponent(storeId)}/records/${encodeUriComponent(key)}`), { headers: baseHeaders, @@ -589,8 +437,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: return new Uint8Array(await response.arrayBuffer()); }, - // `value`: object → application/json; string → text/plain; Uint8Array/ArrayBuffer → - // application/octet-stream (or whatever the caller passed via `contentType`). + // Objects, strings, and binary values use matching content types. set: async ({ storeId, key, value, contentType }: KeyValueStoreSetOptions): Promise => { let body: BodyInit; let resolvedContentType = contentType; @@ -619,12 +466,7 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: apiData('POST', '/key-value-stores', { searchParams: { name } }), }; - // GET /v2/store — Apify Store search (a top-level resource in the Apify API, - // tagged `Store`, distinct from `Actors` — hence a top-level binding rather - // than an `actor.*` method). Same dual nature as dataset.listItems: `await` for one - // page, `for await` to walk every match. offset/limit echo back the request (the - // endpoint's own JSON body doesn't carry pagination metadata beyond `items`, unlike - // dataset's header-based pagination — see makePaginatedList). + // Search the top-level Store resource with the shared paginator. const store = ({ search, limit, offset = 0, category }: StoreSearchOptions): PaginatedItems> => { const fetchPage = async (pageOffset: number, pageLimit: number | undefined): Promise> => { const page = await apiData('GET', '/store', { searchParams: { search, limit: pageLimit, offset: pageOffset, category } }); @@ -634,22 +476,14 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: return makePaginatedList(fetchPage, offset, limit); }; - // Best-effort cleanup for runs this script started but left non-terminal (e.g. the - // script itself threw with a run still in progress). Not part of the frozen `apify` - // binding handed to user code — called directly by the top-level exception handler - // below. One bad abort must not stop the others (the run tracking loop's whole point is - // to catch stragglers after an error): a run that already finished on the platform - // between our last check and now is an *expected* abort failure (the API rejects - // aborting a finished run), not a bug, so failures are reported back for logging, never - // thrown. + // Best-effort cleanup; one failed abort must not stop the others. const abortTrackedRuns = async (): Promise<{ runId: string; error: string }[]> => { const runIds = [...nonTerminalRunIds]; const results = await Promise.allSettled(runIds.map((runId) => run.abort({ runId }))); return results.flatMap((result, i) => result.status === 'rejected' ? [{ runId: runIds[i], error: errorMessage(result.reason) }] : []); }; - // Freeze every namespace (and the wrapper) so the script can't reassign a method to - // corrupt its own behavior or, for `console` below, its own output capture. + // Freeze the binding so user code cannot replace its methods. const binding = realObjectFreeze({ actor: realObjectFreeze(actor), store, @@ -660,12 +494,9 @@ function makeApifyBinding({ token, apiV2, parentOrigin, internalFetch, limits }: return { binding, abortTrackedRuns }; } -// The shape handed to user code as the `apify` binding. Exported (type-only — -// erased at compile time) so tests/*.ts can type-check probes against the same -// surface real usercode.js runs against, without importing runner.ts at runtime. +// Type-only binding surface used by tests. export type ApifyBinding = ReturnType['binding']; -// Push the captured streams as a single item to the run's default dataset. async function pushOutput({ apiV2, token, internalFetch, env, item }: { apiV2: string; token: string; @@ -683,22 +514,14 @@ async function pushOutput({ apiV2, token, internalFetch, env, item }: { if (!responseOk(response)) throw new Error(`Failed to push dataset item: ${response.status} ${await response.text()}`); } -// Parses an optional positive-number env var (as set by entrypoint.sh from Actor input). -// Absent or blank means "field omitted" -> no limit, matching .actor/actor.json's own -// description for each field. The platform validates the input schema's types/minimums -// before this code ever runs, so non-numeric/non-positive shouldn't reach here in -// practice — treating it as "no limit" rather than crashing is a deliberate fallback for -// that already-unlikely case, not a substitute for the schema validation. +// Blank or invalid values mean no configured limit. function parsePositiveNumberEnv(value: string | undefined): number | undefined { if (!value) return undefined; const parsed = realNumber(value); return numberIsFinite(parsed) && parsed > 0 ? parsed : undefined; } -// Frozen so escaped usercode.js module-scope code (which shares this module's namespace via -// `import('./runner.js')`, the same reachability every export in this file has — see guard.ts's -// header comment) can't reassign `.fetch` to a wrapper that captures the real `request`/`env` -// (APIFY_TOKEN, INTERNAL_API) the next time workerd genuinely dispatches to this worker. +// Freeze fetch so escaped module code cannot replace it. export default realObjectFreeze({ async fetch(request: Request, env: Env): Promise { const url = new RealURL(request.url); @@ -707,7 +530,6 @@ export default realObjectFreeze({ const token = env.APIFY_TOKEN; if (!token) throw new Error('APIFY_TOKEN missing from Actor run environment.'); - // APIFY_API_BASE_URL is the platform-internal API (may have a trailing slash). const apiV2 = `${(env.API_BASE_URL || 'https://api.apify.com').replace(/\/+$/, '')}/v2`; const internalFetch: Fetcher['fetch'] = (input, init) => env.INTERNAL_API.fetch(input, init); @@ -719,7 +541,7 @@ export default realObjectFreeze({ const stdout: string[] = []; const stderr: string[] = []; - // Frozen so the script can't reassign e.g. console.log to corrupt its own capture. + // Freeze the captured console methods. const captureConsole: ConsoleLike = realObjectFreeze({ log: (...args: unknown[]) => stdout.push(args.map(stringify).join(' ')), error: (...args: unknown[]) => stderr.push(args.map(stringify).join(' ')), @@ -729,18 +551,7 @@ export default realObjectFreeze({ const { binding, abortTrackedRuns } = makeApifyBinding({ token, apiV2, parentOrigin: env.PARENT_ORIGIN, internalFetch, limits }); - // A thrown program is a user-level failure: capture it in stderr and still - // push the output, so the run SUCCEEDS with diagnostics. Infra failures - // (missing env, dataset push) throw and fail the run. - // - // exitCode is the user script's effective status, distinct from the Actor run's - // status: 0 when the script returns normally, 1 when it throws. The run itself - // still SUCCEEDS on a throw, so callers detect a failed script via this field - // rather than heuristics on stderr (console.error is a legitimate log channel). - // statusMessage carries the same signal in prose, for callers that don't want to - // branch on exitCode. A script that fails to *compile* never reaches this handler at - // all (workerd fails the whole run before any request arrives) — entrypoint.sh - // handles that case directly, see its statusMessage "Failed to compile: ...". + // User errors become diagnostics; infrastructure errors fail the run. let exitCode = 0; let statusMessage = 'Script completed'; try { @@ -749,9 +560,7 @@ export default realObjectFreeze({ stderr.push(errorDetail(err)); exitCode = 1; statusMessage = `Script threw: ${errorMessage(err)}`; - // Best-effort: a script that started Actor runs and then crashed shouldn't leave - // them running unattended. Failures here don't change exitCode/statusMessage — - // the script's own failure is the primary signal; cleanup is secondary. + // Cleanup failures are diagnostics, not a second script failure. const abortFailures = await abortTrackedRuns(); for (const { runId, error } of abortFailures) { stderr.push(`Cleanup: failed to abort run ${runId}: ${error}`); diff --git a/worker/usercode.d.ts b/worker/usercode.d.ts index f36a870..64f2efa 100644 --- a/worker/usercode.d.ts +++ b/worker/usercode.d.ts @@ -1,7 +1,2 @@ -// usercode.js is generated at container startup by entrypoint.sh (gitignored, never -// checked in) — it wraps the run's `code` input in `export async function run(apify, -// console) { ...code... }`. This declaration lets `tsc` resolve runner.ts's import -// without the generated file present. The user's code is untyped JS text spliced -// into the function body, so `apify`/`console` stay `unknown` here on purpose — -// runner.ts's own ApifyBinding/ConsoleLike types describe what's actually passed in. +// Generated usercode.js wraps input code in run(apify, console). export function run(apify: unknown, consoleLike: unknown): Promise; From dccbcd61ef536dc8c78ed613625a541fa3fad044 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 4 Aug 2026 14:28:55 +0200 Subject: [PATCH 45/46] fix: use supported input schema keyword --- .actor/actor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actor/actor.json b/.actor/actor.json index b8e3691..3c78a0c 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -32,7 +32,7 @@ "title": "Max total charge (USD)", "type": "number", "description": "Execution-wide spending budget across every Actor run this script starts — distinct from a single call's own maxTotalChargeUsd, which only caps that one run. Each run's own cap is clamped so the combined total never exceeds this budget; starting a run once it's exhausted throws inside the script. Omit for no limit.", - "exclusiveMinimum": 0 + "minimum": 0 }, "defaultTimeoutSecs": { "title": "Default Actor run timeout (seconds)", From 46deff8855900b545f946d191e3bddfdc8f8efc5 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 4 Aug 2026 14:56:16 +0200 Subject: [PATCH 46/46] ci: run remote smoke tests on master --- .github/workflows/remote-smoke.yml | 41 ++++++++++++++++++++++++++++++ test.sh | 20 ++++++++++----- 2 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/remote-smoke.yml diff --git a/.github/workflows/remote-smoke.yml b/.github/workflows/remote-smoke.yml new file mode 100644 index 0000000..d65548e --- /dev/null +++ b/.github/workflows/remote-smoke.yml @@ -0,0 +1,41 @@ +name: Remote smoke tests + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: remote-smoke-${{ github.ref }} + cancel-in-progress: false + +jobs: + remote-smoke: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out source + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up pnpm + uses: pnpm/action-setup@f40ffcd9367d9f12939873eb1018b921a783ffaa # v4 + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Set up Apify CLI + uses: apify/setup-apify-cli-action@8b19e6a52312e8948b3b10cdb06bd3fa39e8db4a + with: + token: ${{ secrets.APIFY_TOKEN }} + + - name: Run remote smoke tests + run: ./test.sh diff --git a/test.sh b/test.sh index 7562eef..99bde2c 100755 --- a/test.sh +++ b/test.sh @@ -7,24 +7,32 @@ cd "$(dirname "$0")" # Build probes from tests/*.ts. PROBES="tests/binding-smoke.js tests/sandbox-isolation.js" +APIFY_CMD="${APIFY_CMD:-apify}" -command -v apify >/dev/null 2>&1 || { echo "apify CLI not found" >&2; exit 1; } -command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 1; } +if [ "$APIFY_CMD" = "apify" ] && ! command -v apify >/dev/null 2>&1; then + echo "apify CLI not found" >&2 + exit 1 +fi +command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 1; } + +run_apify() { + $APIFY_CMD "$@" +} echo "==> pnpm build" -pnpm build +CI=true pnpm build input_json="$(mktemp)" trap 'rm -f "$input_json"' EXIT echo "==> apify push" -apify push +run_apify push --force failed=0 for probe in $PROBES; do echo "==> apify call: ${probe}" jq -n --arg code "$(cat "$probe")" '{ code: $code }' > "$input_json" - output="$(apify call -f "$input_json" -o 2>&1)" + output="$(run_apify call -f "$input_json" -o 2>&1)" echo "$output" if printf '%s' "$output" | grep -q 'ALL_TESTS_PASSED'; then echo "==> ${probe} passed" @@ -40,7 +48,7 @@ echo "==> all probes passed" # Regression probe for module-scope capability theft. echo "==> apify call: tests/fixtures/realfetch-escape.js (regression: guard.js capability theft)" jq -n --arg code "$(cat tests/fixtures/realfetch-escape.js)" '{ code: $code }' > "$input_json" -if apify call -f "$input_json" -o; then +if run_apify call -f "$input_json" -o; then echo "==> realfetch-escape passed (run succeeded — module-scope steal attempt found nothing to steal)" else echo "==> realfetch-escape FAILED (run crashed — capability-theft regression, see guard.ts/runner.ts/config.capnp's INTERNAL_API binding)" >&2