From 9b08a312821d2457ce07827a482b7637322948f6 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 7 Aug 2026 21:27:34 -0700 Subject: [PATCH 001/103] Added a Temporal durable-session layer over opencode serve. Phase 1 of a drop-in durability layer: an opencode session becomes a durable Temporal workflow that owns the conversation and prompt queue and drives each turn against the shipping opencode server over HTTP. The turn runs server-side (prompt_async), so it survives a worker crash; recovery re-attaches and is idempotent on the user-message count. A crash demo kills the worker mid-turn and the session still completes: the runTurn activity re-drives on a fresh worker (attempt 2) with a single prompt sent. opencode's loop, tools, model, storage, and API are untouched. Phase 2 (a durable SessionExecution on the v2 engine) is scoped in the package README. --- bun.lock | 331 ++++++++++++++++++++++++++-- packages/temporal/README.md | 80 +++++++ packages/temporal/package.json | 21 ++ packages/temporal/src/activities.ts | 54 +++++ packages/temporal/src/crash-demo.ts | 89 ++++++++ packages/temporal/src/demo.ts | 40 ++++ packages/temporal/src/opencode.ts | 66 ++++++ packages/temporal/src/worker.ts | 28 +++ packages/temporal/src/workflows.ts | 61 +++++ 9 files changed, 757 insertions(+), 13 deletions(-) create mode 100644 packages/temporal/README.md create mode 100644 packages/temporal/package.json create mode 100644 packages/temporal/src/activities.ts create mode 100644 packages/temporal/src/crash-demo.ts create mode 100644 packages/temporal/src/demo.ts create mode 100644 packages/temporal/src/opencode.ts create mode 100644 packages/temporal/src/worker.ts create mode 100644 packages/temporal/src/workflows.ts diff --git a/bun.lock b/bun.lock index d2a4a7745d70..2d9dc9962c25 100644 --- a/bun.lock +++ b/bun.lock @@ -942,6 +942,19 @@ "vite": "catalog:", }, }, + "packages/temporal": { + "name": "@opencode-ai/temporal", + "version": "0.0.0", + "dependencies": { + "@temporalio/activity": "^1.11.0", + "@temporalio/client": "^1.11.0", + "@temporalio/worker": "^1.11.0", + "@temporalio/workflow": "^1.11.0", + }, + "devDependencies": { + "tsx": "^4.19.0", + }, + }, "packages/tui": { "name": "@opencode-ai/tui", "version": "1.18.18", @@ -1663,6 +1676,10 @@ "@graphql-typed-document-node/core": ["@graphql-typed-document-node/core@3.2.0", "", { "peerDependencies": { "graphql": "^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" } }, "sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ=="], + "@grpc/grpc-js": ["@grpc/grpc-js@1.14.4", "", { "dependencies": { "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ=="], + + "@grpc/proto-loader": ["@grpc/proto-loader@0.8.1", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.5.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg=="], + "@happy-dom/global-registrator": ["@happy-dom/global-registrator@20.0.11", "", { "dependencies": { "@types/node": "^20.0.0", "happy-dom": "^20.0.11" } }, "sha512-GqNqiShBT/lzkHTMC/slKBrvN0DsD4Di8ssBk4aDaVgEn+2WMzE6DXxq701ndSXj7/0cJ8mNT71pM7Bnrr6JRw=="], "@hey-api/codegen-core": ["@hey-api/codegen-core@0.5.5", "", { "dependencies": { "@hey-api/types": "0.1.2", "ansi-colors": "4.1.3", "c12": "3.3.3", "color-support": "1.1.3" }, "peerDependencies": { "typescript": ">=5.5.3" } }, "sha512-f2ZHucnA2wBGAY8ipB4wn/mrEYW+WUxU2huJmUvfDO6AE2vfILSHeF3wCO39Pz4wUYPoAWZByaauftLrOfC12Q=="], @@ -1749,10 +1766,40 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + "@js-sdsl/ordered-map": ["@js-sdsl/ordered-map@4.4.2", "", {}, "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw=="], + "@js-temporal/polyfill": ["@js-temporal/polyfill@0.5.1", "", { "dependencies": { "jsbi": "^4.3.0" } }, "sha512-hloP58zRVCRSpgDxmqCWJNlizAlUgJFqG2ypq79DCvyv9tHjRYMDOcPFjzfl/A1/YxDvRCZz8wvZvmapQnKwFQ=="], "@jsdevtools/ono": ["@jsdevtools/ono@7.1.3", "", {}, "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg=="], + "@jsonjoy.com/base64": ["@jsonjoy.com/base64@1.1.2", "", { "peerDependencies": { "tslib": "2" } }, "sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA=="], + + "@jsonjoy.com/buffers": ["@jsonjoy.com/buffers@17.67.0", "", { "peerDependencies": { "tslib": "2" } }, "sha512-tfExRpYxBvi32vPs9ZHaTjSP4fHAfzSmcahOfNxtvGHcyJel+aibkPlGeBB+7AoC6hL7lXIE++8okecBxx7lcw=="], + + "@jsonjoy.com/codegen": ["@jsonjoy.com/codegen@1.0.0", "", { "peerDependencies": { "tslib": "2" } }, "sha512-E8Oy+08cmCf0EK/NMxpaJZmOxPqM+6iSe2S4nlSBrPZOORoDJILxtbSUEDKQyTamm/BVAhIGllOBNU79/dwf0g=="], + + "@jsonjoy.com/fs-core": ["@jsonjoy.com/fs-core@4.67.0", "", { "dependencies": { "@jsonjoy.com/fs-node-builtins": "4.67.0", "@jsonjoy.com/fs-node-utils": "4.67.0", "thingies": "^2.5.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-+QOYAGujzm86pKcX4N0JQ1YcLEjypr/I+wmQRxwI8W7K0QXKSi8vQVC2oKQGjcfbHq02JvCQijypfvyhcTz7uw=="], + + "@jsonjoy.com/fs-fsa": ["@jsonjoy.com/fs-fsa@4.67.0", "", { "dependencies": { "@jsonjoy.com/fs-core": "4.67.0", "@jsonjoy.com/fs-node-builtins": "4.67.0", "@jsonjoy.com/fs-node-utils": "4.67.0", "thingies": "^2.5.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-2jCnH5ofKXb+6vcl8dQArO1Gb4FT7vLbMGVnNim0ekXkY78DPVXZvJ8DQp9WLbqP/G/gxiPVz/DOMoOCD4BqqQ=="], + + "@jsonjoy.com/fs-node": ["@jsonjoy.com/fs-node@4.67.0", "", { "dependencies": { "@jsonjoy.com/fs-core": "4.67.0", "@jsonjoy.com/fs-node-builtins": "4.67.0", "@jsonjoy.com/fs-node-utils": "4.67.0", "@jsonjoy.com/fs-print": "4.67.0", "@jsonjoy.com/fs-snapshot": "4.67.0", "glob-to-regex.js": "^1.0.0", "thingies": "^2.5.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-EZ/mSrxRYphDbyll1VuDW0mvj/USoe2M5sxT2nYqyYyvdxsIsijhJOiygBHAaj86Eqd/Kb9ukkwXjBisRTk1tg=="], + + "@jsonjoy.com/fs-node-builtins": ["@jsonjoy.com/fs-node-builtins@4.67.0", "", { "peerDependencies": { "tslib": "2" } }, "sha512-os7Cft1EudH0xZs5Kh5/qHI72jk8DMQ1561elyHkHd9c9xaa4wOfK1iMh4JB9y+kXtpXjEnT4cjHqqc7A3X3lA=="], + + "@jsonjoy.com/fs-node-to-fsa": ["@jsonjoy.com/fs-node-to-fsa@4.67.0", "", { "dependencies": { "@jsonjoy.com/fs-fsa": "4.67.0", "@jsonjoy.com/fs-node-builtins": "4.67.0", "@jsonjoy.com/fs-node-utils": "4.67.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-5e6WTnLhw0Q5mPEACOsA7h2BA++N1FCSmhXRX5gone8Le4fsqcpgpukqW5hWneLfzIr5AOEliu0PyokdYx3Bwg=="], + + "@jsonjoy.com/fs-node-utils": ["@jsonjoy.com/fs-node-utils@4.67.0", "", { "dependencies": { "@jsonjoy.com/fs-node-builtins": "4.67.0", "glob-to-regex.js": "^1.0.1" }, "peerDependencies": { "tslib": "2" } }, "sha512-ZcCPh4jvUqYxAgu4lLe+6eQbijxEkZIrCq0Jhh669o3v3zrCn8N4YAFod3zllZtSHhwb+YbER29LUW/SdNrP7Q=="], + + "@jsonjoy.com/fs-print": ["@jsonjoy.com/fs-print@4.67.0", "", { "dependencies": { "@jsonjoy.com/fs-node-utils": "4.67.0", "tree-dump": "^1.1.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-xBhay3ayVlFeScafZy+7jyH0+I6MLomaL+2nn/KWirgjwh58w8+u44j7oSvE8EQ2NLF63B1eVc65awmbs/39Iw=="], + + "@jsonjoy.com/fs-snapshot": ["@jsonjoy.com/fs-snapshot@4.67.0", "", { "dependencies": { "@jsonjoy.com/buffers": "^17.65.0", "@jsonjoy.com/fs-node-utils": "4.67.0", "@jsonjoy.com/json-pack": "^17.65.0", "@jsonjoy.com/util": "^17.65.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-wn5c6Qx0iVX1dV74l5WOCIPH9lz3xU6A0QG45n0c53athmmr7Z+XTg0YOndME88g/5LjcqwiSSYD9aSr9+goYg=="], + + "@jsonjoy.com/json-pack": ["@jsonjoy.com/json-pack@1.21.0", "", { "dependencies": { "@jsonjoy.com/base64": "^1.1.2", "@jsonjoy.com/buffers": "^1.2.0", "@jsonjoy.com/codegen": "^1.0.0", "@jsonjoy.com/json-pointer": "^1.0.2", "@jsonjoy.com/util": "^1.9.0", "hyperdyperid": "^1.2.0", "thingies": "^2.5.0", "tree-dump": "^1.1.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-+AKG+R2cfZMShzrF2uQw34v3zbeDYUqnQ+jg7ORic3BGtfw9p/+N6RJbq/kkV8JmYZaINknaEQ2m0/f693ZPpg=="], + + "@jsonjoy.com/json-pointer": ["@jsonjoy.com/json-pointer@1.0.2", "", { "dependencies": { "@jsonjoy.com/codegen": "^1.0.0", "@jsonjoy.com/util": "^1.9.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-Fsn6wM2zlDzY1U+v4Nc8bo3bVqgfNTGcn6dMgs6FjrEnt4ZCe60o6ByKRjOGlI2gow0aE/Q41QOigdTqkyK5fg=="], + + "@jsonjoy.com/util": ["@jsonjoy.com/util@1.9.0", "", { "dependencies": { "@jsonjoy.com/buffers": "^1.0.0", "@jsonjoy.com/codegen": "^1.0.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-pLuQo+VPRnN8hfPqUTLTHk126wuYdXVxE6aDmjSeV4NCAgyxWbiOIeNJVtID3h1Vzpoi9m4jXezf73I6LgabgQ=="], + "@jsx-email/all": ["@jsx-email/all@2.2.3", "", { "dependencies": { "@jsx-email/body": "1.0.2", "@jsx-email/button": "1.0.4", "@jsx-email/column": "1.0.3", "@jsx-email/container": "1.0.2", "@jsx-email/font": "1.0.3", "@jsx-email/head": "1.0.2", "@jsx-email/heading": "1.0.2", "@jsx-email/hr": "1.0.2", "@jsx-email/html": "1.0.2", "@jsx-email/img": "1.0.2", "@jsx-email/link": "1.0.2", "@jsx-email/markdown": "2.0.4", "@jsx-email/preview": "1.0.2", "@jsx-email/render": "1.1.1", "@jsx-email/row": "1.0.2", "@jsx-email/section": "1.0.2", "@jsx-email/tailwind": "2.4.4", "@jsx-email/text": "1.0.2" }, "peerDependencies": { "react": "^18.2.0" } }, "sha512-OBvLe/hVSQc0LlMSTJnkjFoqs3bmxcC4zpy/5pT5agPCSKMvAKQjzmsc2xJ2wO73jSpRV1K/g38GmvdCfrhSoQ=="], "@jsx-email/body": ["@jsx-email/body@1.0.2", "", { "peerDependencies": { "react": "^18.2.0" } }, "sha512-NjR2tgLH4XGfGkm+O8kcVwi9MBqZsXZCLlmk3HlMux3/n/+a5zB+yhJqXWZBJl2i+6cSF+E2O6hK11ekyK9WWQ=="], @@ -2007,6 +2054,8 @@ "@opencode-ai/storybook": ["@opencode-ai/storybook@workspace:packages/storybook"], + "@opencode-ai/temporal": ["@opencode-ai/temporal@workspace:packages/temporal"], + "@opencode-ai/tui": ["@opencode-ai/tui@workspace:packages/tui"], "@opencode-ai/ui": ["@opencode-ai/ui@workspace:packages/ui"], @@ -2739,8 +2788,38 @@ "@stripe/stripe-js": ["@stripe/stripe-js@8.6.1", "", {}, "sha512-UJ05U2062XDgydbUcETH1AoRQLNhigQ2KmDn1BG8sC3xfzu6JKg95Qt6YozdzFpxl1Npii/02m2LEWFt1RYjVA=="], + "@swc/core": ["@swc/core@1.15.47", "", { "dependencies": { "@swc/counter": "^0.1.3", "@swc/types": "^0.1.27" }, "optionalDependencies": { "@swc/core-darwin-arm64": "1.15.47", "@swc/core-darwin-x64": "1.15.47", "@swc/core-linux-arm-gnueabihf": "1.15.47", "@swc/core-linux-arm64-gnu": "1.15.47", "@swc/core-linux-arm64-musl": "1.15.47", "@swc/core-linux-ppc64-gnu": "1.15.47", "@swc/core-linux-s390x-gnu": "1.15.47", "@swc/core-linux-x64-gnu": "1.15.47", "@swc/core-linux-x64-musl": "1.15.47", "@swc/core-win32-arm64-msvc": "1.15.47", "@swc/core-win32-ia32-msvc": "1.15.47", "@swc/core-win32-x64-msvc": "1.15.47" }, "peerDependencies": { "@swc/helpers": ">=0.5.17" }, "optionalPeers": ["@swc/helpers"] }, "sha512-FbsO5JcfOjfH38W/rohBRBweJeERsAuIP4f377lmkmxTcq9exjtx4SkRuZY5CdfhR2CBVwDIJegBpJDffwNsOg=="], + + "@swc/core-darwin-arm64": ["@swc/core-darwin-arm64@1.15.47", "", { "os": "darwin", "cpu": "arm64" }, "sha512-GsoMtan3ojGGMGFbl31mmRu5ctZ56re8grGE8mO/OHJ8O+JRkzod02fe7X6ZQ8JvamA3imkEkx/h3u+vsOgPgA=="], + + "@swc/core-darwin-x64": ["@swc/core-darwin-x64@1.15.47", "", { "os": "darwin", "cpu": "x64" }, "sha512-leTi7Rx3KF4zcC637iqWgk9SoV8VXAD8ppQYXsep63px5A/UftOcxLN1pmr8Z1si/YvX90ompP/rHgpYkgwXWg=="], + + "@swc/core-linux-arm-gnueabihf": ["@swc/core-linux-arm-gnueabihf@1.15.47", "", { "os": "linux", "cpu": "arm" }, "sha512-hBqHuoWKKIsKmDBn9qVeWqj5GWZhtlcczVaqQmNRXsDfq+voR5CxKRfamA367QjJXtceYuliLFfEL8QsskRM2g=="], + + "@swc/core-linux-arm64-gnu": ["@swc/core-linux-arm64-gnu@1.15.47", "", { "os": "linux", "cpu": "arm64" }, "sha512-TBxvRz+B4K205TWHHZxWVxkC2RFNP/Mz3PNcECBos5PsKwxjg3QSJzdoebr0VCf0Bfh8HOPldKxAP/8XkFe9gA=="], + + "@swc/core-linux-arm64-musl": ["@swc/core-linux-arm64-musl@1.15.47", "", { "os": "linux", "cpu": "arm64" }, "sha512-3Yu3Uq/VgytqsPjTMbkPU1ExADytbdWbruJYhA584E9jrpE2Ki+R6VVPoZCeAVk1Cb7QxcRTgblw6bSa6a/R+w=="], + + "@swc/core-linux-ppc64-gnu": ["@swc/core-linux-ppc64-gnu@1.15.47", "", { "os": "linux", "cpu": "ppc64" }, "sha512-wfdMi5IaOaNtmh2/6geRoxIdNfqylUZFdtzTKS655y1axWfIWyx7As74vv0wVdjeCIZ3WmCI9odDd4rUttXOSQ=="], + + "@swc/core-linux-s390x-gnu": ["@swc/core-linux-s390x-gnu@1.15.47", "", { "os": "linux", "cpu": "s390x" }, "sha512-3hHYBY0yx8Ez7GMRrkhXHQzMdR5IZA6Wq5Ee4svlgwvSECLpnAJ9+0AimEGUFDvuLwE7nV/2+PYe8+Nm4rvNcQ=="], + + "@swc/core-linux-x64-gnu": ["@swc/core-linux-x64-gnu@1.15.47", "", { "os": "linux", "cpu": "x64" }, "sha512-TjfhjgP/jGCfFHYC3JQPhJA1HwErbIJ9JfREDc1KNkvY6P0LodCgKVIlQ5deeTbkG7ih3bF5PHJLuLpaZjdRyQ=="], + + "@swc/core-linux-x64-musl": ["@swc/core-linux-x64-musl@1.15.47", "", { "os": "linux", "cpu": "x64" }, "sha512-CQpS8Ge/avfjZd0UEwG/sds83Uu32deQXcV1Jo3jD0mmvQQqtYAjpsDZXugmheeAwmt+YIuoVtVHro8LMYHqsQ=="], + + "@swc/core-win32-arm64-msvc": ["@swc/core-win32-arm64-msvc@1.15.47", "", { "os": "win32", "cpu": "arm64" }, "sha512-0W8IKHsUTYiT7G2RqtOoVWk+89yzZikIiDUb/sCK6BmQDBhN91hQSfyUtW12jhEWLzYgcfmisfsZrmZE+84U1A=="], + + "@swc/core-win32-ia32-msvc": ["@swc/core-win32-ia32-msvc@1.15.47", "", { "os": "win32", "cpu": "ia32" }, "sha512-ZIp49d2Z4/ka2jO9otOg4hDvTdPmp86kVOgS2M5FCPI7eKKZ1W0boxWn+8XeZrfERtFGW0AlMRm4JhlJa7l3NA=="], + + "@swc/core-win32-x64-msvc": ["@swc/core-win32-x64-msvc@1.15.47", "", { "os": "win32", "cpu": "x64" }, "sha512-2h8Iek95vnixkBRCo+H8p09+Q5ll2NgSMFrWTy0iKt7+/t+8/T5mBpiT6c0ZxSS7wcWjwZ9sGZkK70tTSYHdDw=="], + + "@swc/counter": ["@swc/counter@0.1.3", "", {}, "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="], + "@swc/helpers": ["@swc/helpers@0.5.23", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw=="], + "@swc/types": ["@swc/types@0.1.28", "", { "dependencies": { "@swc/counter": "^0.1.3" } }, "sha512-V6Mnml8v09QALx6K0elJ7o9K/MkVDtW3t6L+7Ou/JcWtb3xwId2AH4FeOceySd2JaO87IMw4+6vSZxLm34LPbw=="], + "@szmarczak/http-timer": ["@szmarczak/http-timer@4.0.6", "", { "dependencies": { "defer-to-connect": "^2.0.0" } }, "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w=="], "@tailwindcss/node": ["@tailwindcss/node@4.1.11", "", { "dependencies": { "@ampproject/remapping": "^2.3.0", "enhanced-resolve": "^5.18.1", "jiti": "^2.4.2", "lightningcss": "1.30.1", "magic-string": "^0.30.17", "source-map-js": "^1.2.1", "tailwindcss": "4.1.11" } }, "sha512-yzhzuGRmv5QyU9qLNg4GTlYI6STedBWRE7NjxP45CsFYYq9taI0zJXZBMqIC/c8fViNLhmrbpSFS57EoxUmD6Q=="], @@ -2787,6 +2866,22 @@ "@tanstack/virtual-core": ["@tanstack/virtual-core@3.17.3", "", {}, "sha512-8Np/TFELpI0ySuJoVmjvOrQYXH/8sTX0Biv9szhFhY39xOdAAY+smrMxjxOum/ux3eM8MUJQsEJ0/R0UpvC8dw=="], + "@temporalio/activity": ["@temporalio/activity@1.21.1", "", { "dependencies": { "@temporalio/client": "1.21.1", "@temporalio/common": "1.21.1" } }, "sha512-UjA1d4ugL3pRXElwnggtVAMe3lppUPzYpBPmpDLTXfOtxEXPTK1x+8OC2jrZY7qmvpHkOdoo86S+UYFzJkMk/A=="], + + "@temporalio/client": ["@temporalio/client@1.21.1", "", { "dependencies": { "@grpc/grpc-js": "^1.12.4", "@temporalio/common": "1.21.1", "@temporalio/proto": "1.21.1", "abort-controller": "^3.0.0", "long": "^5.2.3", "nexus-rpc": "^0.0.2", "uuid": "^11.1.0" } }, "sha512-rdZAh20wzI5i/SyS46Nv9mE6t2KkS9DTrB57HEMLsq6eqm58Nc6la0WQKJGsUNkmN98KKirByidZ5D/ik3kiQw=="], + + "@temporalio/common": ["@temporalio/common@1.21.1", "", { "dependencies": { "@temporalio/proto": "1.21.1", "long": "^5.2.3", "ms": "3.0.0-canary.1", "nexus-rpc": "^0.0.2", "proto3-json-serializer": "^2.0.0" } }, "sha512-8Pis59xYLrGu6GfkkWvrYWkSPEvo8lBBILsR6gBHGbymNlEc0ynylRXqPmRjwuLfYS7jHzxqjjO7cvXJQ/z2Fg=="], + + "@temporalio/core-bridge": ["@temporalio/core-bridge@1.21.1", "", { "dependencies": { "@grpc/grpc-js": "^1.12.4", "@temporalio/common": "1.21.1" } }, "sha512-gCy/6TFhcFAjFPRN1DeHSwAnXU380Jn/y6yG2dkSV+rYK0JRl66SE2NvdcAjzhoPO/twHEaHklbppojH0cUpUw=="], + + "@temporalio/nexus": ["@temporalio/nexus@1.21.1", "", { "dependencies": { "@temporalio/client": "1.21.1", "@temporalio/common": "1.21.1", "@temporalio/proto": "1.21.1", "long": "^5.2.3", "nexus-rpc": "^0.0.2" } }, "sha512-CIAoTt/WpSE0bn1mE9q5O6hU76q97W349e0FSrUuwQAYDXBy7jzFM6ZzYmm7S5Uk2tC1xrRCAjUGg1lMnHQppw=="], + + "@temporalio/proto": ["@temporalio/proto@1.21.1", "", { "dependencies": { "long": "^5.2.3", "protobufjs": "^7.6.4" } }, "sha512-eSHGrZ6CxbtjrAzxiMgKrWeDiBlWk6/JkIqsB1hrkPB6TQXC67Az8v0BL0Fj3ur8ktQZbaacON/xCDjTsvJyGw=="], + + "@temporalio/worker": ["@temporalio/worker@1.21.1", "", { "dependencies": { "@grpc/grpc-js": "^1.12.4", "@swc/core": "^1.3.102", "@temporalio/activity": "1.21.1", "@temporalio/client": "1.21.1", "@temporalio/common": "1.21.1", "@temporalio/core-bridge": "1.21.1", "@temporalio/nexus": "1.21.1", "@temporalio/proto": "1.21.1", "@temporalio/workflow": "1.21.1", "heap-js": "^2.6.0", "memfs": "^4.6.0", "nexus-rpc": "^0.0.2", "protobufjs": "^7.6.4", "rxjs": "^7.8.1", "source-map": "^0.7.4", "source-map-loader": "^5.0.0", "supports-color": "^8.1.1", "swc-loader": "^0.2.3", "unionfs": "^4.5.1", "webpack": "^5.108.4" } }, "sha512-ccXus6+w317tL+NsJXEYWpHFxcy0VDPfstmBzej0yZrkzWNvAkaWVGQbguKoLToDM8+DMaqklx1dX4gJnknR1g=="], + + "@temporalio/workflow": ["@temporalio/workflow@1.21.1", "", { "dependencies": { "@temporalio/common": "1.21.1", "@temporalio/proto": "1.21.1", "nexus-rpc": "^0.0.2" } }, "sha512-Tsoe9RnB0mL75DGVo3wJJrgTl+QnHYUysPjqRkzQdzgeKravN8RxE0HCfS3cYRZej3eEVwoDftgbo7/KR0NXWQ=="], + "@testing-library/dom": ["@testing-library/dom@10.4.1", "", { "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", "@types/aria-query": "^5.0.1", "aria-query": "5.3.0", "dom-accessibility-api": "^0.5.9", "lz-string": "^1.5.0", "picocolors": "1.1.1", "pretty-format": "^27.0.2" } }, "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg=="], "@testing-library/jest-dom": ["@testing-library/jest-dom@6.9.1", "", { "dependencies": { "@adobe/css-tools": "^4.4.0", "aria-query": "^5.0.0", "css.escape": "^1.5.1", "dom-accessibility-api": "^0.6.3", "picocolors": "^1.1.1", "redent": "^3.0.0" } }, "sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA=="], @@ -3035,12 +3130,46 @@ "@vscode/l10n": ["@vscode/l10n@0.0.18", "", {}, "sha512-KYSIHVmslkaCDyw013pphY+d7x1qV8IZupYfeIfzNA+nsaWHbn5uPuQRvdRFsa9zFzGeudPuoGoZ1Op4jrJXIQ=="], + "@webassemblyjs/ast": ["@webassemblyjs/ast@1.14.1", "", { "dependencies": { "@webassemblyjs/helper-numbers": "1.13.2", "@webassemblyjs/helper-wasm-bytecode": "1.13.2" } }, "sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ=="], + + "@webassemblyjs/floating-point-hex-parser": ["@webassemblyjs/floating-point-hex-parser@1.13.2", "", {}, "sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA=="], + + "@webassemblyjs/helper-api-error": ["@webassemblyjs/helper-api-error@1.13.2", "", {}, "sha512-U56GMYxy4ZQCbDZd6JuvvNV/WFildOjsaWD3Tzzvmw/mas3cXzRJPMjP83JqEsgSbyrmaGjBfDtV7KDXV9UzFQ=="], + + "@webassemblyjs/helper-buffer": ["@webassemblyjs/helper-buffer@1.14.1", "", {}, "sha512-jyH7wtcHiKssDtFPRB+iQdxlDf96m0E39yb0k5uJVhFGleZFoNw1c4aeIcVUPPbXUVJ94wwnMOAqUHyzoEPVMA=="], + + "@webassemblyjs/helper-numbers": ["@webassemblyjs/helper-numbers@1.13.2", "", { "dependencies": { "@webassemblyjs/floating-point-hex-parser": "1.13.2", "@webassemblyjs/helper-api-error": "1.13.2", "@xtuc/long": "4.2.2" } }, "sha512-FE8aCmS5Q6eQYcV3gI35O4J789wlQA+7JrqTTpJqn5emA4U2hvwJmvFRC0HODS+3Ye6WioDklgd6scJ3+PLnEA=="], + + "@webassemblyjs/helper-wasm-bytecode": ["@webassemblyjs/helper-wasm-bytecode@1.13.2", "", {}, "sha512-3QbLKy93F0EAIXLh0ogEVR6rOubA9AoZ+WRYhNbFyuB70j3dRdwH9g+qXhLAO0kiYGlg3TxDV+I4rQTr/YNXkA=="], + + "@webassemblyjs/helper-wasm-section": ["@webassemblyjs/helper-wasm-section@1.14.1", "", { "dependencies": { "@webassemblyjs/ast": "1.14.1", "@webassemblyjs/helper-buffer": "1.14.1", "@webassemblyjs/helper-wasm-bytecode": "1.13.2", "@webassemblyjs/wasm-gen": "1.14.1" } }, "sha512-ds5mXEqTJ6oxRoqjhWDU83OgzAYjwsCV8Lo/N+oRsNDmx/ZDpqalmrtgOMkHwxsG0iI//3BwWAErYRHtgn0dZw=="], + + "@webassemblyjs/ieee754": ["@webassemblyjs/ieee754@1.13.2", "", { "dependencies": { "@xtuc/ieee754": "^1.2.0" } }, "sha512-4LtOzh58S/5lX4ITKxnAK2USuNEvpdVV9AlgGQb8rJDHaLeHciwG4zlGr0j/SNWlr7x3vO1lDEsuePvtcDNCkw=="], + + "@webassemblyjs/leb128": ["@webassemblyjs/leb128@1.13.2", "", { "dependencies": { "@xtuc/long": "4.2.2" } }, "sha512-Lde1oNoIdzVzdkNEAWZ1dZ5orIbff80YPdHx20mrHwHrVNNTjNr8E3xz9BdpcGqRQbAEa+fkrCb+fRFTl/6sQw=="], + + "@webassemblyjs/utf8": ["@webassemblyjs/utf8@1.13.2", "", {}, "sha512-3NQWGjKTASY1xV5m7Hr0iPeXD9+RDobLll3T9d2AO+g3my8xy5peVyjSag4I50mR1bBSN/Ct12lo+R9tJk0NZQ=="], + + "@webassemblyjs/wasm-edit": ["@webassemblyjs/wasm-edit@1.14.1", "", { "dependencies": { "@webassemblyjs/ast": "1.14.1", "@webassemblyjs/helper-buffer": "1.14.1", "@webassemblyjs/helper-wasm-bytecode": "1.13.2", "@webassemblyjs/helper-wasm-section": "1.14.1", "@webassemblyjs/wasm-gen": "1.14.1", "@webassemblyjs/wasm-opt": "1.14.1", "@webassemblyjs/wasm-parser": "1.14.1", "@webassemblyjs/wast-printer": "1.14.1" } }, "sha512-RNJUIQH/J8iA/1NzlE4N7KtyZNHi3w7at7hDjvRNm5rcUXa00z1vRz3glZoULfJ5mpvYhLybmVcwcjGrC1pRrQ=="], + + "@webassemblyjs/wasm-gen": ["@webassemblyjs/wasm-gen@1.14.1", "", { "dependencies": { "@webassemblyjs/ast": "1.14.1", "@webassemblyjs/helper-wasm-bytecode": "1.13.2", "@webassemblyjs/ieee754": "1.13.2", "@webassemblyjs/leb128": "1.13.2", "@webassemblyjs/utf8": "1.13.2" } }, "sha512-AmomSIjP8ZbfGQhumkNvgC33AY7qtMCXnN6bL2u2Js4gVCg8fp735aEiMSBbDR7UQIj90n4wKAFUSEd0QN2Ukg=="], + + "@webassemblyjs/wasm-opt": ["@webassemblyjs/wasm-opt@1.14.1", "", { "dependencies": { "@webassemblyjs/ast": "1.14.1", "@webassemblyjs/helper-buffer": "1.14.1", "@webassemblyjs/wasm-gen": "1.14.1", "@webassemblyjs/wasm-parser": "1.14.1" } }, "sha512-PTcKLUNvBqnY2U6E5bdOQcSM+oVP/PmrDY9NzowJjislEjwP/C4an2303MCVS2Mg9d3AJpIGdUFIQQWbPds0Sw=="], + + "@webassemblyjs/wasm-parser": ["@webassemblyjs/wasm-parser@1.14.1", "", { "dependencies": { "@webassemblyjs/ast": "1.14.1", "@webassemblyjs/helper-api-error": "1.13.2", "@webassemblyjs/helper-wasm-bytecode": "1.13.2", "@webassemblyjs/ieee754": "1.13.2", "@webassemblyjs/leb128": "1.13.2", "@webassemblyjs/utf8": "1.13.2" } }, "sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ=="], + + "@webassemblyjs/wast-printer": ["@webassemblyjs/wast-printer@1.14.1", "", { "dependencies": { "@webassemblyjs/ast": "1.14.1", "@xtuc/long": "4.2.2" } }, "sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw=="], + "@webcontainer/env": ["@webcontainer/env@1.1.1", "", {}, "sha512-6aN99yL695Hi9SuIk1oC88l9o0gmxL1nGWWQ/kNy81HigJ0FoaoTXpytCj6ItzgyCEwA9kF1wixsTuv5cjsgng=="], "@webgpu/types": ["@webgpu/types@0.1.54", "", {}, "sha512-81oaalC8LFrXjhsczomEQ0u3jG+TqE6V9QHLA8GNZq/Rnot0KDugu3LhSYSlie8tSdooAN1Hov05asrUUp9qgg=="], "@xmldom/xmldom": ["@xmldom/xmldom@0.8.13", "", {}, "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw=="], + "@xtuc/ieee754": ["@xtuc/ieee754@1.2.0", "", {}, "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA=="], + + "@xtuc/long": ["@xtuc/long@4.2.2", "", {}, "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ=="], + "@zip.js/zip.js": ["@zip.js/zip.js@2.7.62", "", {}, "sha512-OaLvZ8j4gCkLn048ypkZu29KX30r8/OfFF2w4Jo5WXFr+J04J+lzJ5TKZBVgFXhlvSkqNFQdfnY1Q8TMTCyBVA=="], "abbrev": ["abbrev@4.0.0", "", {}, "sha512-a1wflyaL0tHtJSmLSOVybYhy22vRih4eduhhrkcjgrWGnRfrZtovJ2FRjxuTtkkj47O/baf0R86QU5OuYpz8fA=="], @@ -3071,7 +3200,7 @@ "ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="], - "ajv-keywords": ["ajv-keywords@3.5.2", "", { "peerDependencies": { "ajv": "^6.9.1" } }, "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ=="], + "ajv-keywords": ["ajv-keywords@5.1.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3" }, "peerDependencies": { "ajv": "^8.8.2" } }, "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw=="], "ansi-align": ["ansi-align@3.0.1", "", { "dependencies": { "string-width": "^4.1.0" } }, "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w=="], @@ -3319,6 +3448,8 @@ "chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="], + "chrome-trace-event": ["chrome-trace-event@1.0.4", "", {}, "sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ=="], + "chromium-pickle-js": ["chromium-pickle-js@0.2.0", "", {}, "sha512-1R5Fho+jBq0DDydt+/vHWj5KJNJCKdARKOCwZUen84I5BreWoLqRLANH1U87eJy1tiASPtMnGqJJq0ZsLoRPOw=="], "ci-info": ["ci-info@4.4.0", "", {}, "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg=="], @@ -3667,8 +3798,14 @@ "escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], + "eslint-scope": ["eslint-scope@5.1.1", "", { "dependencies": { "esrecurse": "^4.3.0", "estraverse": "^4.1.1" } }, "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw=="], + "esprima": ["esprima@4.0.1", "", { "bin": { "esparse": "./bin/esparse.js", "esvalidate": "./bin/esvalidate.js" } }, "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="], + "esrecurse": ["esrecurse@4.3.0", "", { "dependencies": { "estraverse": "^5.2.0" } }, "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag=="], + + "estraverse": ["estraverse@4.3.0", "", {}, "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw=="], + "estree-util-attach-comments": ["estree-util-attach-comments@3.0.0", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw=="], "estree-util-build-jsx": ["estree-util-build-jsx@3.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "estree-walker": "^3.0.0" } }, "sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ=="], @@ -3807,6 +3944,8 @@ "fs-minipass": ["fs-minipass@3.0.3", "", { "dependencies": { "minipass": "^7.0.3" } }, "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw=="], + "fs-monkey": ["fs-monkey@1.1.0", "", {}, "sha512-QMUezzXWII9EV5aTFXW1UBVUO77wYPpjqIF8/AviUCThNeSYZykpoTixUeaNNBwmCev0AMDWMAni+f8Hxb1IFw=="], + "fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="], "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], @@ -3859,6 +3998,8 @@ "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], + "glob-to-regex.js": ["glob-to-regex.js@1.2.0", "", { "peerDependencies": { "tslib": "2" } }, "sha512-QMwlOQKU/IzqMUOAZWubUOT8Qft+Y0KQWnX9nK3ch0CJg0tTp4TvGZsTfudYKv2NzoQSyPcnA6TYeIQ3jGichQ=="], + "glob-to-regexp": ["glob-to-regexp@0.4.1", "", {}, "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw=="], "global-agent": ["global-agent@3.0.0", "", { "dependencies": { "boolean": "^3.0.1", "es6-error": "^4.1.1", "matcher": "^3.0.0", "roarr": "^2.15.3", "semver": "^7.3.2", "serialize-error": "^7.0.1" } }, "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q=="], @@ -3947,6 +4088,8 @@ "he": ["he@1.2.0", "", { "bin": { "he": "bin/he" } }, "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw=="], + "heap-js": ["heap-js@2.7.1", "", {}, "sha512-EQfezRg0NCZGNlhlDR3Evrw1FVL2G3LhU7EgPoxufQKruNBSYA8MiRPHeWbU+36o+Fhel0wMwM+sLEiBAlNLJA=="], + "heap-snapshot-toolkit": ["heap-snapshot-toolkit@1.1.3", "", {}, "sha512-joThu2rEsDu8/l4arupRDI1qP4CZXNG+J6Wr348vnbLGSiBkwRdqZ6aOHl5BzEiC+Dc8OTbMlmWjD0lbXD5K2Q=="], "hey-listen": ["hey-listen@1.0.8", "", {}, "sha512-COpmrF2NOg4TBWUJ5UVyaCU2A88wEMkUPK4hNqyCkqHbxT92BbvfjoSozkAIIm6XhicGlJHhFdullInrdhwU8Q=="], @@ -3989,6 +4132,8 @@ "husky": ["husky@9.1.7", "", { "bin": { "husky": "bin.js" } }, "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA=="], + "hyperdyperid": ["hyperdyperid@1.2.0", "", {}, "sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A=="], + "i18n-iso-countries": ["i18n-iso-countries@7.14.0", "", { "dependencies": { "diacritics": "1.3.0" } }, "sha512-nXHJZYtNrfsi1UQbyRqm3Gou431elgLjKl//CYlnBGt5aTWdRPH1PiS2T/p/n8Q8LnqYqzQJik3Q7mkwvLokeg=="], "i18next": ["i18next@23.16.8", "", { "dependencies": { "@babel/runtime": "^7.23.2" } }, "sha512-06r/TitrM88Mg5FdUXAKL96dJMzgqLE5dv3ryBAra4KCwD9mJ4ndOTS95ZuymIGoE+2hzfdaMak2X11/es7ZWg=="], @@ -4147,6 +4292,8 @@ "jake": ["jake@10.9.4", "", { "dependencies": { "async": "^3.2.6", "filelist": "^1.0.4", "picocolors": "^1.1.1" }, "bin": { "jake": "bin/cli.js" } }, "sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA=="], + "jest-worker": ["jest-worker@27.5.1", "", { "dependencies": { "@types/node": "*", "merge-stream": "^2.0.0", "supports-color": "^8.0.0" } }, "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg=="], + "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], "jose": ["jose@6.0.11", "", {}, "sha512-QxG7EaliDARm1O1S8BGakqncGT9s25bKL1WSf6/oa17Tkqwi8D2ZNglqCF+DsYF88/rV66Q/Q2mFAy697E1DUg=="], @@ -4261,6 +4408,8 @@ "lodash": ["lodash@4.18.1", "", {}, "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q=="], + "lodash.camelcase": ["lodash.camelcase@4.3.0", "", {}, "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="], + "lodash.escaperegexp": ["lodash.escaperegexp@4.1.2", "", {}, "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw=="], "lodash.includes": ["lodash.includes@4.3.0", "", {}, "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w=="], @@ -4365,6 +4514,8 @@ "media-typer": ["media-typer@0.3.0", "", {}, "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ=="], + "memfs": ["memfs@4.67.0", "", { "dependencies": { "@jsonjoy.com/fs-core": "4.67.0", "@jsonjoy.com/fs-fsa": "4.67.0", "@jsonjoy.com/fs-node": "4.67.0", "@jsonjoy.com/fs-node-builtins": "4.67.0", "@jsonjoy.com/fs-node-to-fsa": "4.67.0", "@jsonjoy.com/fs-node-utils": "4.67.0", "@jsonjoy.com/fs-print": "4.67.0", "@jsonjoy.com/fs-snapshot": "4.67.0", "@jsonjoy.com/json-pack": "^1.11.0", "@jsonjoy.com/util": "^1.9.0", "glob-to-regex.js": "^1.0.1", "thingies": "^2.5.0", "tree-dump": "^1.0.3", "tslib": "^2.0.0" } }, "sha512-yuwPWDAs2kfwpQFuFNQI2OkiJ4ZqkGvSFq2jbgC9pFPfgh1N1lPxJaBj5rHp9R1wfJlSzUQkFnSw6WYGPwBbRg=="], + "merge-anything": ["merge-anything@5.1.7", "", { "dependencies": { "is-what": "^4.1.8" } }, "sha512-eRtbOb1N5iyH0tkQDAoQ4Ipsp/5qSR79Dzrz8hEPxRX10RWWR/iQXdoKmBSRCThY1Fh5EhISDtpSc93fpxUniQ=="], "merge-descriptors": ["merge-descriptors@1.0.3", "", {}, "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ=="], @@ -4469,6 +4620,8 @@ "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], + "minimizer-webpack-plugin": ["minimizer-webpack-plugin@5.6.1", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "jest-worker": "^27.4.5", "schema-utils": "^4.3.0", "terser": "^5.31.1" }, "peerDependencies": { "@minify-html/node": "*", "@swc/core": "*", "@swc/css": "*", "@swc/html": "*", "clean-css": "*", "cssnano": "*", "csso": "*", "esbuild": "*", "html-minifier-terser": "*", "lightningcss": "*", "postcss": "*", "uglify-js": "*", "webpack": "^5.1.0" }, "optionalPeers": ["@minify-html/node", "@swc/core", "@swc/css", "@swc/html", "clean-css", "cssnano", "csso", "esbuild", "html-minifier-terser", "lightningcss", "postcss", "uglify-js"] }, "sha512-DoeAZz8Q1C1znwsUzej1fdoi4jCf7/+Em27ouLqfK/+3m8G+D7yDhUwrc3CNhjSzGUN1kn7Iv4sWmjflQHenpw=="], + "minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="], "minipass-collect": ["minipass-collect@2.0.1", "", { "dependencies": { "minipass": "^7.0.3" } }, "sha512-D7V8PO9oaz7PWGLbCACuI1qEOsq7UKfLotx/C0Aet43fCUB/wfQ7DYeq2oR/svFJGYDHPr38SHATeaj/ZoKHKw=="], @@ -4521,8 +4674,12 @@ "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], + "neo-async": ["neo-async@2.6.2", "", {}, "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="], + "neotraverse": ["neotraverse@0.6.18", "", {}, "sha512-Z4SmBUweYa09+o6pG+eASabEpP6QkQ70yHj351pQoEXIs8uHbaU2DWVmzBANKgflPa47A50PtB2+NgRpQvr7vA=="], + "nexus-rpc": ["nexus-rpc@0.0.2", "", {}, "sha512-IWjIExdVYlmwXuzHdY/Q3lXCv1gbqoAXPazQhy2w4Xgtgha3H0OOujEESVPQcFUFMWm+pAk2gKnb57g8S41JZg=="], + "nf3": ["nf3@0.1.12", "", {}, "sha512-qbMXT7RTGh74MYWPeqTIED8nDW70NXOULVHpdWcdZ7IVHVnAsMV9fNugSNnvooipDc1FMOzpis7T9nXJEbJhvQ=="], "nitro": ["nitro@3.0.1-alpha.1", "", { "dependencies": { "consola": "^3.4.2", "crossws": "^0.4.1", "db0": "^0.3.4", "h3": "2.0.1-rc.5", "jiti": "^2.6.1", "nf3": "^0.1.10", "ofetch": "^2.0.0-alpha.3", "ohash": "^2.0.11", "oxc-minify": "^0.96.0", "oxc-transform": "^0.96.0", "srvx": "^0.9.5", "undici": "^7.16.0", "unenv": "^2.0.0-rc.24", "unstorage": "^2.0.0-alpha.4" }, "peerDependencies": { "rolldown": "*", "rollup": "^4", "vite": "^7", "xml2js": "^0.6.2" }, "optionalPeers": ["rolldown", "rollup", "vite", "xml2js"], "bin": { "nitro": "dist/cli/index.mjs" } }, "sha512-U4AxIsXxdkxzkFrK0XAw0e5Qbojk8jQ50MjjRBtBakC4HurTtQoiZvF+lSe382jhuQZCfAyywGWOFa9QzXLFaw=="], @@ -4821,7 +4978,9 @@ "proto-list": ["proto-list@1.2.4", "", {}, "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA=="], - "protobufjs": ["protobufjs@7.6.2", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-N9EiLovGEQOJSPF26Ij7qUGvahfEnq0eeYZ02aigIedkmz1qZSwjnP9SBITHJuF/6MYbIW4HDN8zdYjsjqJKXQ=="], + "proto3-json-serializer": ["proto3-json-serializer@2.0.2", "", { "dependencies": { "protobufjs": "^7.2.5" } }, "sha512-SAzp/O4Yh02jGdRc+uIrGoe87dkN/XtwxfZ4ZyafJHymd79ozp5VG5nyZ7ygqPM5+cpLDjjGnYFUkngonyDPOQ=="], + + "protobufjs": ["protobufjs@7.6.5", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw=="], "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], @@ -5007,6 +5166,8 @@ "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], + "rxjs": ["rxjs@7.8.2", "", { "dependencies": { "tslib": "^2.1.0" } }, "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA=="], + "s-js": ["s-js@0.4.9", "", {}, "sha512-RtpOm+cM6O0sHg6IA70wH+UC3FZcND+rccBZpBAHzlUgNO2Bm5BN+FnM8+OBxzXdwpKWFwX11JGF0MFRkhSoIQ=="], "safe-array-concat": ["safe-array-concat@1.1.4", "", { "dependencies": { "call-bind": "^1.0.9", "call-bound": "^1.0.4", "get-intrinsic": "^1.3.0", "has-symbols": "^1.1.0", "isarray": "^2.0.5" } }, "sha512-wtZlHyOje6OZTGqAoaDKxFkgRtkF9CnHAVnCHKfuj200wAgL+bSJhdsCD2l0Qx/2ekEXjPWcyKkfGb5CPboslg=="], @@ -5029,6 +5190,8 @@ "scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="], + "schema-utils": ["schema-utils@4.3.3", "", { "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", "ajv-formats": "^2.1.1", "ajv-keywords": "^5.1.0" } }, "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA=="], + "section-matter": ["section-matter@1.0.0", "", { "dependencies": { "extend-shallow": "^2.0.1", "kind-of": "^6.0.0" } }, "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA=="], "secure-json-parse": ["secure-json-parse@4.1.0", "", {}, "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA=="], @@ -5137,10 +5300,12 @@ "sort-keys-length": ["sort-keys-length@1.0.1", "", { "dependencies": { "sort-keys": "^1.0.0" } }, "sha512-GRbEOUqCxemTAk/b32F2xa8wDTs+Z1QHOkbhJDQTvv/6G3ZkbJ+frYWsTcc7cBB3Fu4wy4XlLCuNtJuMn7Gsvw=="], - "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + "source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + "source-map-loader": ["source-map-loader@5.0.0", "", { "dependencies": { "iconv-lite": "^0.6.3", "source-map-js": "^1.0.2" }, "peerDependencies": { "webpack": "^5.72.1" } }, "sha512-k2Dur7CbSLcAH73sBcIkV5xjPV4SzqO1NJ7+XaQl8if3VODDUj3FNchNGpqgJSKbvUfJuhVdv8K2Eu8/TNl2eA=="], + "source-map-support": ["source-map-support@0.5.21", "", { "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w=="], "space-separated-tokens": ["space-separated-tokens@2.0.2", "", {}, "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q=="], @@ -5245,12 +5410,14 @@ "superstruct": ["superstruct@1.0.4", "", {}, "sha512-7JpaAoX2NGyoFlI9NBh66BQXGONc+uE+MRS5i2iOBKuS4e+ccgMDjATgZldkah+33DakBxDHiss9kvUcGAO8UQ=="], - "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "supports-color": ["supports-color@8.1.1", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q=="], "supports-preserve-symlinks-flag": ["supports-preserve-symlinks-flag@1.0.0", "", {}, "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w=="], "sury": ["sury@11.0.0-alpha.4", "", { "peerDependencies": { "rescript": "12.x" }, "optionalPeers": ["rescript"] }, "sha512-oeG/GJWZvQCKtGPpLbu0yCZudfr5LxycDo5kh7SJmKHDPCsEPJssIZL2Eb4Tl7g9aPEvIDuRrkS+L0pybsMEMA=="], + "swc-loader": ["swc-loader@0.2.7", "", { "dependencies": { "@swc/counter": "^0.1.3" }, "peerDependencies": { "@swc/core": "^1.2.147", "webpack": ">=2" } }, "sha512-nwYWw3Fh9ame3Rtm7StS9SBLpHRRnYcK7bnpF3UKZmesAK0gw2/ADvlURFAINmPvKtDLzp+GBiP9yLoEjg6S9w=="], + "system-architecture": ["system-architecture@0.1.0", "", {}, "sha512-ulAk51I9UVUyJgxlv9M6lFot2WP3e7t8Kz9+IS6D4rVba1tR9kON+Ey69f+1R4Q8cd45Lod6a4IcJIxnzGc/zA=="], "tagged-tag": ["tagged-tag@1.0.0", "", {}, "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng=="], @@ -5279,6 +5446,8 @@ "thenify-all": ["thenify-all@1.6.0", "", { "dependencies": { "thenify": ">= 3.1.0 < 4" } }, "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA=="], + "thingies": ["thingies@2.6.1", "", { "peerDependencies": { "tslib": "^2" } }, "sha512-cV/CMGTK3M4MlnJ/0At6ismOw/A0EEniDNScajjz/Br3c1sqE72YD01rGpPTKwd27wAxI5Pr+6+0w8yofzFRYw=="], + "thread-stream": ["thread-stream@4.2.0", "", { "dependencies": { "real-require": "^1.0.0" } }, "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ=="], "thunky": ["thunky@1.1.0", "", {}, "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA=="], @@ -5323,6 +5492,8 @@ "traverse": ["traverse@0.3.9", "", {}, "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="], + "tree-dump": ["tree-dump@1.1.0", "", { "peerDependencies": { "tslib": "2" } }, "sha512-rMuvhU4MCDbcbnleZTFezWsaZXRFemSqAM+7jPnzUl1fo9w3YEKOxAeui0fz3OI4EU4hf23iyA7uQRVko+UaBA=="], + "tree-sitter-bash": ["tree-sitter-bash@0.25.0", "", { "dependencies": { "node-addon-api": "^8.2.1", "node-gyp-build": "^4.8.2" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-gZtlj9+qFS81qKxpLfD6H0UssQ3QBc/F0nKkPsiFDyfQF2YBqYvglFJUzchrPpVhZe9kLZTrJ9n2J6lmka69Vg=="], "tree-sitter-powershell": ["tree-sitter-powershell@0.25.10", "", { "dependencies": { "node-addon-api": "^7.1.0", "node-gyp-build": "^4.8.0" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-bEt8QoySpGFnU3aa8WedQyNMaN6aTwy/WUbvIVt0JSKF+BbJoSHNHu+wCbhj7xLMsfB0AuffmiJm+B8gzva8Lg=="], @@ -5347,6 +5518,8 @@ "tsscmp": ["tsscmp@1.0.6", "", {}, "sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA=="], + "tsx": ["tsx@4.23.6", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-D/YYGUDqKlLvXhM5fBBbiENaGICxLfU4viHnZEkgmgplnDFa+Kczy34VV7AmLJgdzisv0I/J3zitfC26JH3GXg=="], + "tuf-js": ["tuf-js@4.1.0", "", { "dependencies": { "@tufjs/models": "4.1.0", "debug": "^4.4.3", "make-fetch-happen": "^15.0.1" } }, "sha512-50QV99kCKH5P/Vs4E2Gzp7BopNV+KzTXqWeaxrfu5IQJBOULRsTIS9seSsOVT8ZnGXzCyx55nYWAi4qJzpZKEQ=="], "tunnel": ["tunnel@0.0.6", "", {}, "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="], @@ -5403,6 +5576,8 @@ "unifont": ["unifont@0.5.2", "", { "dependencies": { "css-tree": "^3.0.0", "ofetch": "^1.4.1", "ohash": "^2.0.0" } }, "sha512-LzR4WUqzH9ILFvjLAUU7dK3Lnou/qd5kD+IakBtBK4S15/+x2y9VX+DcWQv6s551R6W+vzwgVS6tFg3XggGBgg=="], + "unionfs": ["unionfs@4.6.0", "", { "dependencies": { "fs-monkey": "^1.0.0" } }, "sha512-fJAy3gTHjFi5S3TP5EGdjs/OUMFFvI/ady3T8qVuZfkv8Qi8prV/Q8BuFEgODJslhZTT2z2qdD2lGdee9qjEnA=="], + "unist-util-find-after": ["unist-util-find-after@5.0.0", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0" } }, "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ=="], "unist-util-is": ["unist-util-is@6.0.1", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g=="], @@ -5523,6 +5698,8 @@ "walk-up-path": ["walk-up-path@4.0.0", "", {}, "sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A=="], + "watchpack": ["watchpack@2.5.2", "", { "dependencies": { "graceful-fs": "^4.1.2" } }, "sha512-6i/00NBjP4yGPs+caKSyRfpTF/8Torsu0MOW3mMzIbhgISFder8i7xbqgHlLMwJrdiN8ndBV3UA1/AfzPSr+jg=="], + "web-namespaces": ["web-namespaces@2.0.1", "", {}, "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ=="], "web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="], @@ -5533,6 +5710,8 @@ "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], + "webpack": ["webpack@5.109.2", "", { "dependencies": { "@types/estree": "^1.0.8", "@types/json-schema": "^7.0.15", "@webassemblyjs/ast": "^1.14.1", "@webassemblyjs/wasm-edit": "^1.14.1", "@webassemblyjs/wasm-parser": "^1.14.1", "acorn": "^8.16.0", "browserslist": "^4.28.1", "chrome-trace-event": "^1.0.2", "enhanced-resolve": "^5.24.4", "es-module-lexer": "^2.1.0", "eslint-scope": "5.1.1", "events": "^3.2.0", "graceful-fs": "^4.2.11", "mime-db": "^1.54.0", "minimizer-webpack-plugin": "^5.6.1", "neo-async": "^2.6.2", "schema-utils": "^4.3.3", "tapable": "^2.3.0", "watchpack": "^2.5.2", "webpack-sources": "^3.5.1" }, "peerDependencies": { "webpack-cli": "*" }, "optionalPeers": ["webpack-cli"], "bin": { "webpack": "bin/webpack.js" } }, "sha512-U9/cvLzxObKNEZ9+TtdqrHM5/9z3lgl2c+c4BzbqGxFQvQvBAq87yql5A8pQ+rrMbS496MZJeF5enVBndIy2hw=="], + "webpack-sources": ["webpack-sources@3.5.0", "", {}, "sha512-HPuy+uuoTCaaoEoI1LQ3JN9+vrPBvEesnnX1jADHy728cHSMlq4wUc4afYqahq2B1mhQVZxCXOkNTnXltr+2vQ=="], "webpack-virtual-modules": ["webpack-virtual-modules@0.5.0", "", {}, "sha512-kyDivFZ7ZM0BVOUteVbDFhlRt7Ah/CSPwJdi8hBpkK7QLumUqdLtVfm/PX/hkcnrvr0i77fO5+TjZ94Pe+C9iw=="], @@ -5737,8 +5916,6 @@ "@astrojs/mdx/acorn": ["acorn@8.16.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="], - "@astrojs/mdx/source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], - "@astrojs/sitemap/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], "@astrojs/solid-js/vite": ["vite@6.4.2", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ=="], @@ -5899,6 +6076,8 @@ "@develar/schema-utils/ajv": ["ajv@6.15.0", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw=="], + "@develar/schema-utils/ajv-keywords": ["ajv-keywords@3.5.2", "", { "peerDependencies": { "ajv": "^6.9.1" } }, "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ=="], + "@dot/log/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], "@electron/asar/commander": ["commander@5.1.0", "", {}, "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg=="], @@ -5927,6 +6106,8 @@ "@fastify/proxy-addr/ipaddr.js": ["ipaddr.js@2.4.0", "", {}, "sha512-9VGk3HGanVE6JoZXHiCpnGy5X0jYDnN4EA4lntFPj+1vIWlFhIylq2CrrCOJH9EAhc5CYhq18F2Av2tgoAPsYQ=="], + "@grpc/proto-loader/yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], + "@hey-api/json-schema-ref-parser/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], "@hey-api/openapi-ts/open": ["open@11.0.0", "", { "dependencies": { "default-browser": "^5.4.0", "define-lazy-prop": "^3.0.0", "is-in-ssh": "^1.0.0", "is-inside-container": "^1.0.0", "powershell-utils": "^0.1.0", "wsl-utils": "^0.3.0" } }, "sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw=="], @@ -5935,6 +6116,14 @@ "@img/sharp-wasm32/@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], + "@jsonjoy.com/fs-snapshot/@jsonjoy.com/json-pack": ["@jsonjoy.com/json-pack@17.67.0", "", { "dependencies": { "@jsonjoy.com/base64": "17.67.0", "@jsonjoy.com/buffers": "17.67.0", "@jsonjoy.com/codegen": "17.67.0", "@jsonjoy.com/json-pointer": "17.67.0", "@jsonjoy.com/util": "17.67.0", "hyperdyperid": "^1.2.0", "thingies": "^2.5.0", "tree-dump": "^1.1.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-t0ejURcGaZsn1ClbJ/3kFqSOjlryd92eQY465IYrezsXmPcfHPE/av4twRSxf6WE+TkZgLY+71vCZbiIiFKA/w=="], + + "@jsonjoy.com/fs-snapshot/@jsonjoy.com/util": ["@jsonjoy.com/util@17.67.0", "", { "dependencies": { "@jsonjoy.com/buffers": "17.67.0", "@jsonjoy.com/codegen": "17.67.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-6+8xBaz1rLSohlGh68D1pdw3AwDi9xydm8QNlAFkvnavCJYSze+pxoW2VKP8p308jtlMRLs5NTHfPlZLd4w7ew=="], + + "@jsonjoy.com/json-pack/@jsonjoy.com/buffers": ["@jsonjoy.com/buffers@1.2.1", "", { "peerDependencies": { "tslib": "2" } }, "sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA=="], + + "@jsonjoy.com/util/@jsonjoy.com/buffers": ["@jsonjoy.com/buffers@1.2.1", "", { "peerDependencies": { "tslib": "2" } }, "sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA=="], + "@jsx-email/cli/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], "@jsx-email/cli/esbuild": ["esbuild@0.19.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.19.12", "@esbuild/android-arm": "0.19.12", "@esbuild/android-arm64": "0.19.12", "@esbuild/android-x64": "0.19.12", "@esbuild/darwin-arm64": "0.19.12", "@esbuild/darwin-x64": "0.19.12", "@esbuild/freebsd-arm64": "0.19.12", "@esbuild/freebsd-x64": "0.19.12", "@esbuild/linux-arm": "0.19.12", "@esbuild/linux-arm64": "0.19.12", "@esbuild/linux-ia32": "0.19.12", "@esbuild/linux-loong64": "0.19.12", "@esbuild/linux-mips64el": "0.19.12", "@esbuild/linux-ppc64": "0.19.12", "@esbuild/linux-riscv64": "0.19.12", "@esbuild/linux-s390x": "0.19.12", "@esbuild/linux-x64": "0.19.12", "@esbuild/netbsd-x64": "0.19.12", "@esbuild/openbsd-x64": "0.19.12", "@esbuild/sunos-x64": "0.19.12", "@esbuild/win32-arm64": "0.19.12", "@esbuild/win32-ia32": "0.19.12", "@esbuild/win32-x64": "0.19.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-aARqgq8roFBj054KvQr5f1sFu0D65G+miZRCuJyJ0G13Zwx7vRar5Zhn2tkQNzIXcBrNVsv/8stehpj+GAjgbg=="], @@ -5951,8 +6140,6 @@ "@mdx-js/mdx/acorn": ["acorn@8.16.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="], - "@mdx-js/mdx/source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], - "@modelcontextprotocol/sdk/express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="], "@modelcontextprotocol/sdk/hono": ["hono@4.12.23", "", {}, "sha512-eIaZ9qDgu7XV0pxOCrg7/WhnQ6Ivm22UcxhXx/A3dcbqbbYgBEkc6e/J/s7j2tS96zoB0S9VBdLwQNCWwUo4LA=="], @@ -6055,6 +6242,8 @@ "@opencode-ai/web/@shikijs/transformers": ["@shikijs/transformers@3.20.0", "", { "dependencies": { "@shikijs/core": "3.20.0", "@shikijs/types": "3.20.0" } }, "sha512-PrHHMRr3Q5W1qB/42kJW6laqFyWdhrPF2hNR9qjOm1xcSiAO3hAHo7HaVyHE6pMyevmy3i51O8kuGGXC78uK3g=="], + "@opentelemetry/otlp-transformer/protobufjs": ["protobufjs@7.6.2", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-N9EiLovGEQOJSPF26Ij7qUGvahfEnq0eeYZ02aigIedkmz1qZSwjnP9SBITHJuF/6MYbIW4HDN8zdYjsjqJKXQ=="], + "@opentui/core/diff": ["diff@9.0.0", "", {}, "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw=="], "@opentui/core/marked": ["marked@17.0.1", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-boeBdiS0ghpWcSwoNm/jJBwdpFaMnZWRzjA6SkUMYb40SVaN1x7mmfGKp0jvexGcx+7y2La5zRZsYFZI6Qpypg=="], @@ -6149,6 +6338,10 @@ "@tanstack/server-functions-plugin/@babel/code-frame": ["@babel/code-frame@7.27.1", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg=="], + "@temporalio/client/uuid": ["uuid@11.1.1", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ=="], + + "@temporalio/common/ms": ["ms@3.0.0-canary.1", "", {}, "sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g=="], + "@testing-library/dom/aria-query": ["aria-query@5.3.0", "", { "dependencies": { "dequal": "^2.0.3" } }, "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A=="], "@testing-library/dom/dom-accessibility-api": ["dom-accessibility-api@0.5.16", "", {}, "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg=="], @@ -6191,8 +6384,6 @@ "ai-gateway-provider/@openrouter/ai-sdk-provider": ["@openrouter/ai-sdk-provider@2.10.0", "", { "peerDependencies": { "ai": "^6.0.0", "zod": "^3.25.0 || ^4.0.0" } }, "sha512-FMsAEjLUt5pWuRE2LDC/LCvVrFjLlrEzUITH5+5SZtfq7KZ2wrOHjQVxzz92sju8S9ltpzW87CLW8/b0oBXVCw=="], - "ajv-keywords/ajv": ["ajv@6.15.0", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw=="], - "ansi-align/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], "anymatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], @@ -6251,6 +6442,8 @@ "c12/dotenv": ["dotenv@17.4.2", "", {}, "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw=="], + "clean-css/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + "clone-response/mimic-response": ["mimic-response@1.0.1", "", {}, "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ=="], "compress-commons/is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], @@ -6315,7 +6508,7 @@ "esbuild-plugin-copy/chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], - "estree-util-to-js/source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], + "esrecurse/estraverse": ["estraverse@5.3.0", "", {}, "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA=="], "execa/get-stream": ["get-stream@8.0.1", "", {}, "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA=="], @@ -6357,6 +6550,8 @@ "iconv-corefoundation/node-addon-api": ["node-addon-api@1.7.2", "", {}, "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg=="], + "istanbul-lib-report/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "istanbul-reports/html-escaper": ["html-escaper@2.0.2", "", {}, "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg=="], "js-beautify/glob": ["glob@10.5.0", "", { "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg=="], @@ -6465,12 +6660,16 @@ "readdir-glob/minimatch": ["minimatch@5.1.9", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw=="], + "recast/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + "rimraf/glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="], "roarr/sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="], "router/path-to-regexp": ["path-to-regexp@8.4.2", "", {}, "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA=="], + "schema-utils/ajv-formats": ["ajv-formats@2.1.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA=="], + "send/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], "send/mime": ["mime@1.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="], @@ -6491,6 +6690,10 @@ "sort-keys/is-plain-obj": ["is-plain-obj@1.1.0", "", {}, "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg=="], + "source-map-loader/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], + + "source-map-support/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + "sst/aws4fetch": ["aws4fetch@1.0.18", "", {}, "sha512-3Cf+YaUl07p24MoQ46rFwulAmiyCwH2+1zw1ZyPAX5OtJ34Hh185DwB8y/qRLb6cYYYtSFJ9pthyLc0MD4e8sQ=="], "sst/jose": ["jose@5.2.3", "", {}, "sha512-KUXdbctm1uHVL8BYhnyHkgp3zDX5KW8ZhAKVFEfUbU2P8Alpzjb+48hHvjOdQIyPshoblhzsuqOwEEAbtHVirA=="], @@ -6519,6 +6722,8 @@ "tree-sitter-bash/node-addon-api": ["node-addon-api@8.8.0", "", {}, "sha512-c5Ko1fZJIJmzhFIkhRN76WTq+fC6tWnGy9CXA0fA+XygsWZmEwG8vmbkNqxMyoaa0Tin4djul49NzdVcJJcjeA=="], + "tsx/esbuild": ["esbuild@0.28.1", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.1", "@esbuild/android-arm": "0.28.1", "@esbuild/android-arm64": "0.28.1", "@esbuild/android-x64": "0.28.1", "@esbuild/darwin-arm64": "0.28.1", "@esbuild/darwin-x64": "0.28.1", "@esbuild/freebsd-arm64": "0.28.1", "@esbuild/freebsd-x64": "0.28.1", "@esbuild/linux-arm": "0.28.1", "@esbuild/linux-arm64": "0.28.1", "@esbuild/linux-ia32": "0.28.1", "@esbuild/linux-loong64": "0.28.1", "@esbuild/linux-mips64el": "0.28.1", "@esbuild/linux-ppc64": "0.28.1", "@esbuild/linux-riscv64": "0.28.1", "@esbuild/linux-s390x": "0.28.1", "@esbuild/linux-x64": "0.28.1", "@esbuild/netbsd-arm64": "0.28.1", "@esbuild/netbsd-x64": "0.28.1", "@esbuild/openbsd-arm64": "0.28.1", "@esbuild/openbsd-x64": "0.28.1", "@esbuild/openharmony-arm64": "0.28.1", "@esbuild/sunos-x64": "0.28.1", "@esbuild/win32-arm64": "0.28.1", "@esbuild/win32-ia32": "0.28.1", "@esbuild/win32-x64": "0.28.1" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw=="], + "tw-to-css/postcss": ["postcss@8.4.31", "", { "dependencies": { "nanoid": "^3.3.6", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } }, "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ=="], "tw-to-css/tailwindcss": ["tailwindcss@3.3.2", "", { "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", "chokidar": "^3.5.3", "didyoumean": "^1.2.2", "dlv": "^1.1.3", "fast-glob": "^3.2.12", "glob-parent": "^6.0.2", "is-glob": "^4.0.3", "jiti": "^1.18.2", "lilconfig": "^2.1.0", "micromatch": "^4.0.5", "normalize-path": "^3.0.0", "object-hash": "^3.0.0", "picocolors": "^1.0.0", "postcss": "^8.4.23", "postcss-import": "^15.1.0", "postcss-js": "^4.0.1", "postcss-load-config": "^4.0.1", "postcss-nested": "^6.0.1", "postcss-selector-parser": "^6.0.11", "postcss-value-parser": "^4.2.0", "resolve": "^1.22.2", "sucrase": "^3.32.0" }, "bin": { "tailwind": "lib/cli.js", "tailwindcss": "lib/cli.js" } }, "sha512-9jPkMiIBXvPc2KywkraqsUfbfj+dHDb+JPWtSJa9MLFdrPyazI7q6WX2sUrm7R9eVR7qqv3Pas7EvQFzxKnI6w=="], @@ -6561,6 +6766,14 @@ "vscode-languageserver-protocol/vscode-jsonrpc": ["vscode-jsonrpc@8.2.0", "", {}, "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA=="], + "webpack/acorn": ["acorn@8.16.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="], + + "webpack/enhanced-resolve": ["enhanced-resolve@5.24.5", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.3.3" } }, "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A=="], + + "webpack/es-module-lexer": ["es-module-lexer@2.1.0", "", {}, "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ=="], + + "webpack/webpack-sources": ["webpack-sources@3.5.1", "", {}, "sha512-jyuiGJdtvY434z5bUZrjz67v76/ePNvFZTp9Mdz29IlH4+GPsgyGjiv0fKI+M7BdkU6ADjulUcKAd3tUK3WlEw=="], + "wrangler/esbuild": ["esbuild@0.25.4", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.4", "@esbuild/android-arm": "0.25.4", "@esbuild/android-arm64": "0.25.4", "@esbuild/android-x64": "0.25.4", "@esbuild/darwin-arm64": "0.25.4", "@esbuild/darwin-x64": "0.25.4", "@esbuild/freebsd-arm64": "0.25.4", "@esbuild/freebsd-x64": "0.25.4", "@esbuild/linux-arm": "0.25.4", "@esbuild/linux-arm64": "0.25.4", "@esbuild/linux-ia32": "0.25.4", "@esbuild/linux-loong64": "0.25.4", "@esbuild/linux-mips64el": "0.25.4", "@esbuild/linux-ppc64": "0.25.4", "@esbuild/linux-riscv64": "0.25.4", "@esbuild/linux-s390x": "0.25.4", "@esbuild/linux-x64": "0.25.4", "@esbuild/netbsd-arm64": "0.25.4", "@esbuild/netbsd-x64": "0.25.4", "@esbuild/openbsd-arm64": "0.25.4", "@esbuild/openbsd-x64": "0.25.4", "@esbuild/sunos-x64": "0.25.4", "@esbuild/win32-arm64": "0.25.4", "@esbuild/win32-ia32": "0.25.4", "@esbuild/win32-x64": "0.25.4" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-8pgjLUcUjcgDg+2Q4NYXnPbo/vncAY4UmyaCm0jZevERqCHZIaWwdJHkf8XQtu4AxSKCdvrUbT0XUr1IdZzI8Q=="], "wrap-ansi/ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="], @@ -6717,8 +6930,12 @@ "@develar/schema-utils/ajv/json-schema-traverse": ["json-schema-traverse@0.4.1", "", {}, "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="], + "@dot/log/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "@electron/asar/minimatch/brace-expansion": ["brace-expansion@1.1.15", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-EwOCDEex4quD37XhqM3omwtMoJjr//isUZz1JopUNWms+4Z2ViyM/k1YIRePpoVNnQhENnxtFjLaxNHrT7xIUg=="], + "@electron/fuses/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "@electron/fuses/fs-extra/jsonfile": ["jsonfile@6.2.1", "", { "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q=="], "@electron/notarize/fs-extra/jsonfile": ["jsonfile@6.2.1", "", { "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q=="], @@ -6741,8 +6958,22 @@ "@expressive-code/plugin-shiki/shiki/@shikijs/types": ["@shikijs/types@3.23.0", "", { "dependencies": { "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-3JZ5HXOZfYjsYSk0yPwBrkupyYSLpAE26Qc0HLghhZNGTZg/SKxXIIgoxOpmmeQP0RRSDJTk1/vPfw9tbw+jSQ=="], + "@grpc/proto-loader/yargs/cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], + + "@grpc/proto-loader/yargs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + "@hey-api/json-schema-ref-parser/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], + "@jsonjoy.com/fs-snapshot/@jsonjoy.com/json-pack/@jsonjoy.com/base64": ["@jsonjoy.com/base64@17.67.0", "", { "peerDependencies": { "tslib": "2" } }, "sha512-5SEsJGsm15aP8TQGkDfJvz9axgPwAEm98S5DxOuYe8e1EbfajcDmgeXXzccEjh+mLnjqEKrkBdjHWS5vFNwDdw=="], + + "@jsonjoy.com/fs-snapshot/@jsonjoy.com/json-pack/@jsonjoy.com/codegen": ["@jsonjoy.com/codegen@17.67.0", "", { "peerDependencies": { "tslib": "2" } }, "sha512-idnkUplROpdBOV0HMcwhsCUS5TRUi9poagdGs70A6S4ux9+/aPuKbh8+UYRTLYQHtXvAdNfQWXDqZEx5k4Dj2Q=="], + + "@jsonjoy.com/fs-snapshot/@jsonjoy.com/json-pack/@jsonjoy.com/json-pointer": ["@jsonjoy.com/json-pointer@17.67.0", "", { "dependencies": { "@jsonjoy.com/util": "17.67.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-+iqOFInH+QZGmSuaybBUNdh7yvNrXvqR+h3wjXm0N/3JK1EyyFAeGJvqnmQL61d1ARLlk/wJdFKSL+LHJ1eaUA=="], + + "@jsonjoy.com/fs-snapshot/@jsonjoy.com/util/@jsonjoy.com/codegen": ["@jsonjoy.com/codegen@17.67.0", "", { "peerDependencies": { "tslib": "2" } }, "sha512-idnkUplROpdBOV0HMcwhsCUS5TRUi9poagdGs70A6S4ux9+/aPuKbh8+UYRTLYQHtXvAdNfQWXDqZEx5k4Dj2Q=="], + + "@jsx-email/cli/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "@jsx-email/cli/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.19.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA=="], "@jsx-email/cli/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.19.12", "", { "os": "android", "cpu": "arm" }, "sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w=="], @@ -7033,8 +7264,6 @@ "ai-gateway-provider/@ai-sdk/perplexity/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.45", "", { "dependencies": { "@ai-sdk/provider": "3.0.15", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8", "undici": "^5.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-7u5B/E2uZmU65SlJhhQGFHZwRCN0xOz4HHtFc4sEGV9PHbX3fGiEiZBpc/SABay1dGeJgK3VD60rvLGoWdWPXA=="], - "ajv-keywords/ajv/json-schema-traverse": ["json-schema-traverse@0.4.1", "", {}, "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="], - "ansi-align/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], "ansi-align/string-width/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], @@ -7087,6 +7316,8 @@ "body-parser/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + "builder-util/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "builder-util/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], "c12/chokidar/readdirp": ["readdirp@5.0.0", "", {}, "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ=="], @@ -7135,14 +7366,20 @@ "electron-builder-squirrel-windows/builder-util/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + "electron-builder/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "electron-builder/yargs/cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], "electron-builder/yargs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + "electron-publish/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "electron-updater/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], "electron-winstaller/fs-extra/universalify": ["universalify@0.1.2", "", {}, "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="], + "esbuild-plugin-copy/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "esbuild-plugin-copy/chokidar/readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], "express/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], @@ -7203,6 +7440,58 @@ "string-width-cjs/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "tsx/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.1", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ=="], + + "tsx/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.28.1", "", { "os": "android", "cpu": "arm" }, "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ=="], + + "tsx/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.1", "", { "os": "android", "cpu": "arm64" }, "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg=="], + + "tsx/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.28.1", "", { "os": "android", "cpu": "x64" }, "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng=="], + + "tsx/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q=="], + + "tsx/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ=="], + + "tsx/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw=="], + + "tsx/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ=="], + + "tsx/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.1", "", { "os": "linux", "cpu": "arm" }, "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ=="], + + "tsx/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g=="], + + "tsx/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.1", "", { "os": "linux", "cpu": "ia32" }, "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w=="], + + "tsx/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg=="], + + "tsx/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ=="], + + "tsx/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ=="], + + "tsx/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ=="], + + "tsx/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag=="], + + "tsx/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.1", "", { "os": "linux", "cpu": "x64" }, "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA=="], + + "tsx/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw=="], + + "tsx/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.1", "", { "os": "none", "cpu": "x64" }, "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg=="], + + "tsx/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.1", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q=="], + + "tsx/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.1", "", { "os": "openbsd", "cpu": "x64" }, "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw=="], + + "tsx/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg=="], + + "tsx/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.1", "", { "os": "sunos", "cpu": "x64" }, "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ=="], + + "tsx/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA=="], + + "tsx/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg=="], + + "tsx/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.1", "", { "os": "win32", "cpu": "x64" }, "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A=="], + "tw-to-css/tailwindcss/chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], "tw-to-css/tailwindcss/glob-parent": ["glob-parent@6.0.2", "", { "dependencies": { "is-glob": "^4.0.3" } }, "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A=="], @@ -7359,6 +7648,14 @@ "@electron/universal/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + "@grpc/proto-loader/yargs/cliui/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + + "@grpc/proto-loader/yargs/cliui/wrap-ansi": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], + + "@grpc/proto-loader/yargs/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], + + "@grpc/proto-loader/yargs/string-width/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + "@jsx-email/cli/tailwindcss/chokidar/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], "@jsx-email/cli/tailwindcss/chokidar/readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], @@ -7525,6 +7822,8 @@ "electron-builder-squirrel-windows/app-builder-lib/which/isexe": ["isexe@3.1.5", "", {}, "sha512-6B3tLtFqtQS4ekarvLVMZ+X+VlvQekbe4taUkf/rhVO3d/h0M2rfARm/pXLcPEsjjMsFgrFgSrhQIxcSVrBz8w=="], + "electron-builder-squirrel-windows/builder-util/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "electron-builder-squirrel-windows/builder-util/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], "electron-builder/yargs/cliui/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], @@ -7601,6 +7900,10 @@ "@aws-sdk/token-providers/@aws-sdk/core/@aws-sdk/xml-builder/fast-xml-parser/strnum": ["strnum@2.3.0", "", {}, "sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q=="], + "@grpc/proto-loader/yargs/cliui/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + + "@grpc/proto-loader/yargs/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "@jsx-email/cli/tailwindcss/chokidar/readdirp/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], "@sentry/bundler-plugin-core/glob/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], @@ -7619,6 +7922,8 @@ "electron-builder-squirrel-windows/app-builder-lib/@electron/get/fs-extra/universalify": ["universalify@0.1.2", "", {}, "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="], + "electron-builder-squirrel-windows/app-builder-lib/electron-publish/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "electron-builder/yargs/cliui/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "electron-builder/yargs/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], diff --git a/packages/temporal/README.md b/packages/temporal/README.md new file mode 100644 index 000000000000..d7417e391db9 --- /dev/null +++ b/packages/temporal/README.md @@ -0,0 +1,80 @@ +# @opencode-ai/temporal + +A Temporal durable-execution layer for opencode. It makes an opencode **session** a durable +Temporal workflow, so a coding session survives worker loss, can run detached or in the +background, and can be driven from anywhere by signal. It is a drop-in layer: opencode's loop, +tools, model, storage, and HTTP API are untouched. + +There are two phases. + +## Phase 1 (this directory): wrap the shipping `opencode serve` + +A workflow owns one session. It creates the session, then drains a queue of prompts, running each +turn as an activity that drives the shipping opencode server over its HTTP API. The conversation, +the prompt queue, and which turns have completed all live in workflow state, so the session is +durable and resumable. The turn itself runs inside opencode (`prompt_async` forks it server-side), +so it keeps running even while the Temporal worker is down; recovery re-attaches by reading the +recorded messages. + +- `src/opencode.ts` — a thin `fetch` client for the opencode HTTP API. +- `src/activities.ts` — `createSession`, `runTurn` (idempotent on the user-message count, so a + retry never double-sends), `abortTurn`. +- `src/workflows.ts` — `durableSession`: create session, drain the prompt queue, one turn per + activity; signals `submitPrompt` / `abortSession` / `closeSession`; query `getState`. +- `src/worker.ts` — the Temporal worker (runs on Node via `tsx`). +- `src/demo.ts` — drive a two-turn session end to end. +- `src/crash-demo.ts` — kill the worker mid-turn and show the session still completes. + +### Run it + +```bash +# 1. a Temporal dev server +temporal server start-dev --port 7237 + +# 2. opencode serve with a provider key (any provider opencode supports) +OPENAI_API_KEY=... bun run --cwd packages/opencode --conditions=browser src/index.ts serve --port 4599 + +# 3. the durable-session worker +TEMPORAL_ADDRESS=127.0.0.1:7237 OPENCODE_BASE_URL=http://127.0.0.1:4599 bun run --cwd packages/temporal worker + +# 4. drive a session +TEMPORAL_ADDRESS=127.0.0.1:7237 bun run --cwd packages/temporal demo +``` + +Env: `TEMPORAL_ADDRESS` (default `127.0.0.1:7237`), `TEMPORAL_TASK_QUEUE` (`opencode-durable`), +`OPENCODE_BASE_URL` (`http://127.0.0.1:4599`), `OPENCODE_PROVIDER` (`openai`), `OPENCODE_MODEL` +(`gpt-5-mini`). + +### What it proves + +`bun run --cwd packages/temporal crash-demo` starts a turn, `SIGKILL`s the worker mid-turn, starts +a fresh worker, and checks the outcome. A passing run shows: + +``` +reply: "RECOVERED" +runTurn attempts (max): 2 | started-event attempts: [ 1, 2 ] +user messages in session: 1 +CRASH-RECOVERY: PASS +``` + +That is: the turn completed after the crash, Temporal re-drove the activity on a new worker +(attempt 2), and idempotency kept it to a single prompt (no double-send). + +### Honest limits of Phase 1 + +- It makes the **orchestration** durable (the session, the queue, turn re-drive), not opencode's + in-process turn. If the opencode **server** dies mid-turn, the turn's host side effects can be + partial; re-driving re-attaches to whatever the server recorded. +- The worker talks to the opencode server over unauthenticated HTTP by default. Run them together + or set `OPENCODE_SERVER_PASSWORD` and pass the header. + +## Phase 2 (next): a durable `SessionExecution` on the v2 engine + +opencode's newer v2 engine (`packages/core` + `packages/server`) is already event-sourced per +session and exposes a substitutable `SessionExecution` service (`active` / `resume` / `wake` / +`interrupt`) whose local impl comments "Future remote placement belongs here." Phase 2 provides a +Temporal-backed `SessionExecution`: a workflow per session (`resume` = start-or-signal, `wake` = +signal, `interrupt` = cancel) with `SessionRunner.run` (one continuation from recorded history) as +the activity. Because turn state lives in the event log, the workflow stays thin and the recovery +is engine-level, not a re-attach. It is wired by changing one binding in +`packages/server/src/routes.ts`. diff --git a/packages/temporal/package.json b/packages/temporal/package.json new file mode 100644 index 000000000000..5f478bad923e --- /dev/null +++ b/packages/temporal/package.json @@ -0,0 +1,21 @@ +{ + "name": "@opencode-ai/temporal", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "A Temporal durable-execution layer that drives an opencode session as a durable workflow.", + "scripts": { + "worker": "tsx src/worker.ts", + "demo": "tsx src/demo.ts", + "crash-demo": "tsx src/crash-demo.ts" + }, + "dependencies": { + "@temporalio/activity": "^1.11.0", + "@temporalio/client": "^1.11.0", + "@temporalio/worker": "^1.11.0", + "@temporalio/workflow": "^1.11.0" + }, + "devDependencies": { + "tsx": "^4.19.0" + } +} diff --git a/packages/temporal/src/activities.ts b/packages/temporal/src/activities.ts new file mode 100644 index 000000000000..b1c36819dd88 --- /dev/null +++ b/packages/temporal/src/activities.ts @@ -0,0 +1,54 @@ +import { heartbeat } from "@temporalio/activity" +import * as oc from "./opencode" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +export async function createSession(title?: string): Promise { + return oc.createSession(title) +} + +/** + * Run one turn idempotently and return the assistant's text. + * + * `turnIndex` is the 0-based position of this turn's user message in the session. The workflow + * drives turns strictly in order, so the count of user messages already recorded is the + * idempotency key: post the prompt only when this turn's user message does not exist yet. A + * Temporal retry (e.g. after a worker crash) therefore never double-sends; it just resumes + * polling. Because the turn runs server-side (prompt_async), it keeps going while the worker is + * down, and recovery re-attaches by reading the recorded messages. + */ +export async function runTurn(input: { sessionID: string; turnIndex: number; text: string }): Promise { + const { sessionID, turnIndex, text } = input + + const userCount = (await oc.listMessages(sessionID)).filter((m) => m.info.role === "user").length + if (userCount <= turnIndex) { + await oc.promptAsync(sessionID, text) + } + + // A turn produces several assistant messages (one per step). It is done when the session is idle + // AND this turn's user message has at least one completed assistant message after it; the final + // answer is the last such message's text. + for (;;) { + heartbeat() + const msgs = await oc.listMessages(sessionID) + const users = msgs.filter((m) => m.info.role === "user") + if (users.length > turnIndex && (await oc.isIdle(sessionID))) { + const userCreated = users[turnIndex].info.time?.created ?? 0 + const replies = msgs.filter( + (m) => + m.info.role === "assistant" && + (m.info.time?.created ?? 0) >= userCreated && + m.info.time?.completed, + ) + if (replies.length > 0) { + const last = replies[replies.length - 1] + return oc.assistantText(last) || replies.map(oc.assistantText).filter(Boolean).join("\n") + } + } + await wait(1500) + } +} + +export async function abortTurn(sessionID: string): Promise { + await oc.abort(sessionID) +} diff --git a/packages/temporal/src/crash-demo.ts b/packages/temporal/src/crash-demo.ts new file mode 100644 index 000000000000..6581b3c20dc0 --- /dev/null +++ b/packages/temporal/src/crash-demo.ts @@ -0,0 +1,89 @@ +import { spawn, type ChildProcess } from "node:child_process" +import { Client, Connection } from "@temporalio/client" +import { durableSession, submitPrompt, closeSession, getState } from "./workflows" +import * as oc from "./opencode" + +// Self-contained crash-recovery proof. It owns the worker lifecycle: start a turn, KILL the worker +// mid-turn, restart it, and show the workflow still completes, the runTurn activity was re-driven +// (attempt > 1), and no duplicate prompt was sent (exactly one user message). The turn itself keeps +// running in the opencode server while the worker is down; recovery re-attaches to it. + +const ADDR = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" +const QUEUE = process.env.TEMPORAL_TASK_QUEUE ?? "opencode-durable-crash" +const OPENCODE = process.env.OPENCODE_BASE_URL ?? "http://127.0.0.1:4599" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +function startWorker(): ChildProcess { + return spawn("bun", ["run", "worker"], { + cwd: process.cwd(), + env: { ...process.env, TEMPORAL_ADDRESS: ADDR, TEMPORAL_TASK_QUEUE: QUEUE, OPENCODE_BASE_URL: OPENCODE }, + stdio: "ignore", + detached: true, // own process group, so we can kill the whole tree (bun -> tsx -> node) + }) +} + +function killWorker(w: ChildProcess): void { + try { + process.kill(-(w.pid as number), "SIGKILL") // negative pid = the group + } catch { + w.kill("SIGKILL") + } +} + +async function main() { + const connection = await Connection.connect({ address: ADDR }) + const client = new Client({ connection }) + + console.log("[1] starting worker A") + let worker = startWorker() + await wait(10_000) // let it bundle + connect + + const workflowId = `crash-${Date.now()}` + const handle = await client.workflow.start(durableSession, { + taskQueue: QUEUE, + workflowId, + args: [{ title: "crash demo" }], + }) + await handle.signal(submitPrompt, "Create a file crash.txt containing exactly RECOVERED, then read it back and reply with only its contents.") + await handle.signal(closeSession) + console.log(`[2] started workflow ${workflowId}, turn in progress`) + + await wait(4_000) // let the turn get going (prompt posted, polling) + console.log("[3] KILLING worker A mid-turn") + killWorker(worker) + + await wait(4_000) // worker down; the turn keeps running server-side + console.log("[4] starting worker B (recovery)") + worker = startWorker() + + console.log("[5] awaiting workflow completion ...") + await handle.result() + const state = await handle.query(getState) + const reply = state.turns[0]?.reply ?? "" + + // Evidence from history: how many attempts did runTurn take? + const events = (await handle.fetchHistory()).events ?? [] + const startedAttempts = events + .filter((e: any) => e.activityTaskStartedEventAttributes) + .map((e: any) => Number(e.activityTaskStartedEventAttributes.attempt ?? 1)) + const maxAttempt = startedAttempts.length ? Math.max(...startedAttempts) : 1 + + // Idempotency: exactly one user message for the one prompt (no double-send on retry). + const userMsgs = (await oc.listMessages(state.sessionID!)).filter((m) => m.info.role === "user").length + + console.log("\n=== RESULT ===") + console.log("reply:", JSON.stringify(reply)) + console.log("runTurn attempts (max):", maxAttempt, "| started-event attempts:", startedAttempts) + console.log("user messages in session:", userMsgs) + const ok = reply.includes("RECOVERED") && maxAttempt >= 2 && userMsgs === 1 + console.log("CRASH-RECOVERY:", ok ? "PASS" : "FAIL") + + killWorker(worker) + process.exit(ok ? 0 : 1) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/packages/temporal/src/demo.ts b/packages/temporal/src/demo.ts new file mode 100644 index 000000000000..e4d694184731 --- /dev/null +++ b/packages/temporal/src/demo.ts @@ -0,0 +1,40 @@ +import { Client, Connection } from "@temporalio/client" +import { durableSession, submitPrompt, closeSession, getState } from "./workflows" + +// Drives one durable session: start the workflow, queue prompts by signal, close it, wait, and +// print the recorded conversation. Nothing here holds the turn open; the workflow owns it. +async function main() { + const address = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" + const taskQueue = process.env.TEMPORAL_TASK_QUEUE ?? "opencode-durable" + + const connection = await Connection.connect({ address }) + const client = new Client({ connection }) + + const workflowId = process.env.WORKFLOW_ID ?? `oc-session-${Date.now()}` + const prompts = process.argv.slice(2) + if (prompts.length === 0) { + prompts.push( + "Create a file called note.txt containing exactly the token DURABLE_OK, then confirm.", + "Run `cat note.txt` and reply with only its contents.", + ) + } + + const handle = await client.workflow.start(durableSession, { + taskQueue, + workflowId, + args: [{ title: "durable demo" }], + }) + console.log(`started workflow ${workflowId}`) + + for (const p of prompts) await handle.signal(submitPrompt, p) + await handle.signal(closeSession) + + await handle.result() + const state = await handle.query(getState) + console.log(JSON.stringify(state, null, 2)) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/packages/temporal/src/opencode.ts b/packages/temporal/src/opencode.ts new file mode 100644 index 000000000000..991b1239fa29 --- /dev/null +++ b/packages/temporal/src/opencode.ts @@ -0,0 +1,66 @@ +// A thin HTTP client for the shipping `opencode serve` API. Used by activities (Node context), +// never from workflow code. The durability layer treats opencode as a black-box server it drives. + +const BASE = () => process.env.OPENCODE_BASE_URL ?? "http://127.0.0.1:4599" +const MODEL = () => ({ + providerID: process.env.OPENCODE_PROVIDER ?? "openai", + modelID: process.env.OPENCODE_MODEL ?? "gpt-5-mini", +}) + +export interface OcMessage { + info: { id: string; role: string; time?: { created?: number; completed?: number } } + parts: Array<{ type: string; text?: string }> +} + +async function api(path: string, init?: RequestInit): Promise { + const res = await fetch(BASE() + path, init) + if (!res.ok) { + const body = await res.text().catch(() => "") + throw new Error(`opencode ${init?.method ?? "GET"} ${path} -> ${res.status} ${body.slice(0, 300)}`) + } + return res +} + +export async function createSession(title?: string): Promise { + const res = await api("/session", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(title ? { title } : {}), + }) + return (await res.json()).id as string +} + +export async function listMessages(sessionID: string): Promise { + const res = await api(`/session/${sessionID}/message`) + return (await res.json()) as OcMessage[] +} + +// Fire-and-return: the server forks the turn and answers 204 immediately, so the turn keeps +// running even if the caller (our worker) dies. Recovery re-attaches by polling listMessages. +export async function promptAsync(sessionID: string, text: string): Promise { + await api(`/session/${sessionID}/prompt_async`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: MODEL(), parts: [{ type: "text", text }] }), + }) +} + +export async function abort(sessionID: string): Promise { + await api(`/session/${sessionID}/abort`, { method: "POST" }).catch(() => {}) +} + +// The session is busy for the whole turn (all steps) and idle when it is fully done. Verified: the +// status does not blip idle between steps, so this is a reliable end-of-turn signal. +export async function isIdle(sessionID: string): Promise { + const res = await api("/session/status") + const map = (await res.json()) as Record + return (map[sessionID]?.type ?? "idle") === "idle" +} + +export function assistantText(m: OcMessage): string { + return m.parts + .filter((p) => p.type === "text" && p.text) + .map((p) => p.text) + .join("") + .trim() +} diff --git a/packages/temporal/src/worker.ts b/packages/temporal/src/worker.ts new file mode 100644 index 000000000000..932b09e2ec22 --- /dev/null +++ b/packages/temporal/src/worker.ts @@ -0,0 +1,28 @@ +import { fileURLToPath } from "node:url" +import { NativeConnection, Worker } from "@temporalio/worker" +import * as activities from "./activities" + +async function main() { + const address = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" + const namespace = process.env.TEMPORAL_NAMESPACE ?? "default" + const taskQueue = process.env.TEMPORAL_TASK_QUEUE ?? "opencode-durable" + + const connection = await NativeConnection.connect({ address }) + const worker = await Worker.create({ + connection, + namespace, + taskQueue, + workflowsPath: fileURLToPath(new URL("./workflows.ts", import.meta.url)), + activities, + }) + console.log( + `opencode-temporal worker ready: queue=${taskQueue} temporal=${address} ` + + `opencode=${process.env.OPENCODE_BASE_URL ?? "http://127.0.0.1:4599"}`, + ) + await worker.run() +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/packages/temporal/src/workflows.ts b/packages/temporal/src/workflows.ts new file mode 100644 index 000000000000..0e4c414ea090 --- /dev/null +++ b/packages/temporal/src/workflows.ts @@ -0,0 +1,61 @@ +import { proxyActivities, defineSignal, defineQuery, setHandler, condition } from "@temporalio/workflow" +import type * as activities from "./activities" + +// Unlimited retries with heartbeat: this is what makes a turn survive a worker crash. The turn +// runs server-side, so a re-run just re-attaches (idempotent on the user-message count). +const { createSession, runTurn, abortTurn } = proxyActivities({ + startToCloseTimeout: "15 minutes", + // Short heartbeat so a dead worker's in-flight turn is detected and re-driven quickly. The turn + // keeps running server-side meanwhile, so the retry just re-attaches. + heartbeatTimeout: "8 seconds", +}) + +export const submitPrompt = defineSignal<[string]>("submitPrompt") +export const abortSession = defineSignal("abortSession") +export const closeSession = defineSignal("closeSession") +export const getState = defineQuery("getState") + +export interface DurableSessionInput { + title?: string +} + +export interface DurableSessionState { + sessionID?: string + turns: Array<{ text: string; reply: string }> + pending: number +} + +/** + * One durable opencode session. The conversation, the queue of prompts, and which turns have + * completed all live in workflow state, so the session survives worker loss, can run detached or + * in the background, and can be driven from anywhere by signal. The turns still execute in the + * opencode server; this workflow is the durable brain that drives and remembers them. + */ +export async function durableSession(input: DurableSessionInput = {}): Promise { + const queue: string[] = [] + const turns: Array<{ text: string; reply: string }> = [] + let closed = false + + const sessionID = await createSession(input.title) + + setHandler(submitPrompt, (text) => { + queue.push(text) + }) + setHandler(closeSession, () => { + closed = true + }) + setHandler(abortSession, () => { + // Best-effort: tell the server to abort the active turn; the running activity then returns the + // (aborted) assistant message and the loop moves on. + void abortTurn(sessionID) + }) + setHandler(getState, () => ({ sessionID, turns, pending: queue.length })) + + for (;;) { + await condition(() => queue.length > 0 || closed) + if (queue.length === 0 && closed) break + const text = queue.shift()! + const reply = await runTurn({ sessionID, turnIndex: turns.length, text }) + turns.push({ text, reply }) + } +} From 2c0ed2a1262bcc9ed5ba240723123fa6121b7551 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 7 Aug 2026 22:35:27 -0700 Subject: [PATCH 002/103] Added a Temporal-backed SessionExecution to the v2 engine. Phase 2: make each v2 session a durable Temporal workflow. SessionExecutionTemporal implements the substitutable SessionExecution service (wake -> signalWithStart, resume -> forced signalWithStart, interrupt -> cancel signal), with the local coordinator's drain (SessionRunner.run for the whole turn) moved into a runContinuation activity that runs against the durable event log. The Temporal client and an embedded worker are co-hosted inside the server process (both run under bun). It is opt-in via OPENCODE_SESSION_EXECUTION=temporal; the one-line swap is at routes.ts, and the loop, tools, model, storage, and HTTP API are untouched. Verified: with it enabled, prompting a v2 session drove a full turn to completion (step.ended) and Temporal recorded a completed per-session workflow (session-exec-). --- bun.lock | 4 + packages/core/package.json | 4 + .../session/execution/temporal-activities.ts | 36 +++++ .../session/execution/temporal-workflow.ts | 66 ++++++++++ .../core/src/session/execution/temporal.ts | 123 ++++++++++++++++++ packages/server/src/routes.ts | 9 +- 6 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 packages/core/src/session/execution/temporal-activities.ts create mode 100644 packages/core/src/session/execution/temporal-workflow.ts create mode 100644 packages/core/src/session/execution/temporal.ts diff --git a/bun.lock b/bun.lock index 2d9dc9962c25..52233e590cb5 100644 --- a/bun.lock +++ b/bun.lock @@ -332,6 +332,10 @@ "@opentelemetry/sdk-trace-base": "2.6.1", "@parcel/watcher": "2.5.1", "@silvia-odwyer/photon-node": "0.3.4", + "@temporalio/activity": "^1.21.0", + "@temporalio/client": "^1.21.0", + "@temporalio/worker": "^1.21.0", + "@temporalio/workflow": "^1.21.0", "ai-gateway-provider": "3.2.0", "bun-pty": "0.4.8", "cross-spawn": "catalog:", diff --git a/packages/core/package.json b/packages/core/package.json index ee24893c3ae5..afd13a2e95a4 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -61,6 +61,10 @@ "drizzle-kit": "catalog:" }, "dependencies": { + "@temporalio/activity": "^1.21.0", + "@temporalio/client": "^1.21.0", + "@temporalio/worker": "^1.21.0", + "@temporalio/workflow": "^1.21.0", "@ai-sdk/alibaba": "1.0.17", "@ai-sdk/amazon-bedrock": "4.0.112", "@ai-sdk/anthropic": "3.0.82", diff --git a/packages/core/src/session/execution/temporal-activities.ts b/packages/core/src/session/execution/temporal-activities.ts new file mode 100644 index 000000000000..3300b8520abb --- /dev/null +++ b/packages/core/src/session/execution/temporal-activities.ts @@ -0,0 +1,36 @@ +// The runContinuation activity: it runs one drain (SessionRunner.run for the whole turn) by calling +// the `drain` closure the layer captured over the app's Effect context. It heartbeats so a worker +// crash is detected quickly, and forwards Temporal cancellation as an AbortSignal so an interrupt +// turns into Effect fiber interruption inside the runner. + +import { heartbeat, Context } from "@temporalio/activity" + +export interface DrainInput { + sessionID: string + force: boolean +} + +export type Activities = { + runContinuation(input: DrainInput): Promise +} + +export function makeActivities( + drain: (input: DrainInput, signal: AbortSignal) => Promise, +): Activities { + return { + async runContinuation(input) { + const beat = setInterval(() => { + try { + heartbeat() + } catch { + // heartbeat outside an activity context is a no-op for our purposes + } + }, 3000) + try { + await drain(input, Context.current().cancellationSignal) + } finally { + clearInterval(beat) + } + }, + } +} diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts new file mode 100644 index 000000000000..aa3f30bd3091 --- /dev/null +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -0,0 +1,66 @@ +// The durable equivalent of SessionRunCoordinator, as a Temporal workflow (one per session). +// +// MUST stay pure: Temporal bundles this in an isolated sandbox, so no `effect`, no +// `@opencode-ai/core`, no Node builtins. It only ever sees the sessionID string. All real work +// (SessionRunner.run against the durable event log) happens in the runContinuation activity. +// +// Semantics mirror the coordinator (run-coordinator.ts): a wake (or force) drives exactly one +// drain, repeated wakes coalesce into at most one follow-up, and the workflow ends when a drain +// finishes with nothing pending. A later wake starts a fresh run via signalWithStart. + +import { + proxyActivities, + defineSignal, + setHandler, + condition, + CancellationScope, + isCancellation, +} from "@temporalio/workflow" +import type { Activities } from "./temporal-activities" + +const { runContinuation } = proxyActivities({ + startToCloseTimeout: "30 minutes", + // Short heartbeat so a dead worker's in-flight drain is re-driven quickly; the run re-reads the + // durable log, so a retry is a safe re-attach. + heartbeatTimeout: "10 seconds", + retry: { maximumAttempts: 100 }, +}) + +export const wake = defineSignal("wake") +export const force = defineSignal("force") +export const interrupt = defineSignal("interrupt") + +export async function sessionExecution(sessionID: string): Promise { + // Starting the workflow implies there is work to drain (it is started via signalWithStart). + let pendingWake = true + let forceNext = false + let stopping = false + + setHandler(wake, () => { + pendingWake = true + }) + setHandler(force, () => { + pendingWake = true + forceNext = true + }) + setHandler(interrupt, () => { + stopping = true + CancellationScope.current().cancel() + }) + + for (;;) { + await condition(() => pendingWake || stopping) + if (stopping) return + const f = forceNext + pendingWake = false + forceNext = false + try { + await runContinuation({ sessionID, force: f }) + } catch (e) { + if (isCancellation(e)) return + throw e + } + // Quiescent: no wake arrived while draining. End; a future wake starts a new run. + if (!pendingWake) return + } +} diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts new file mode 100644 index 000000000000..4ad4c326c971 --- /dev/null +++ b/packages/core/src/session/execution/temporal.ts @@ -0,0 +1,123 @@ +export * as SessionExecutionTemporal from "./temporal" + +import { fileURLToPath } from "node:url" +import { Effect, Layer } from "effect" +import { Client, Connection } from "@temporalio/client" +import { NativeConnection, Worker } from "@temporalio/worker" + +import { LocationServiceMap } from "../../location-service-map" +import { makeGlobalNode } from "../../effect/app-node" +import { SessionRunner } from "../runner" +import { SessionSchema } from "../schema" +import { SessionStore } from "../store" +import { SessionExecution } from "../execution" +import { makeActivities, type DrainInput } from "./temporal-activities" +import * as WF from "./temporal-workflow" + +const ADDRESS = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" +const NAMESPACE = process.env.TEMPORAL_NAMESPACE ?? "default" +const TASK_QUEUE = process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? "opencode-session-exec" +const workflowId = (id: string) => `session-exec-${id}` + +/** + * A Temporal-backed SessionExecution. It makes each session a durable workflow: + * - wake -> signalWithStart(wake) (start while idle, or coalesce into the running run) + * - resume -> signalWithStart(force) (force one drain even with no eligible input) + * - interrupt -> signal(interrupt) (cancels the workflow's scope -> aborts the drain) + * - active -> the set of sessions this process has started + * + * The drain itself (SessionRunner.run for the whole turn) is exactly the local coordinator's body, + * run inside a Temporal activity. Because turn state lives in the durable event log, a worker crash + * is recovered by re-running the activity: it re-reads recorded history and continues. + */ +const layer = Layer.effect( + SessionExecution.Service, + Effect.gen(function* () { + const store = yield* SessionStore.Service + const locations = yield* LocationServiceMap.Service + // The app context the local drain runs in: providing it, then the per-location layer, supplies + // SessionRunner and all of its dependencies. + const ctx = yield* Effect.context() + + const drain = (input: DrainInput, signal: AbortSignal): Promise => + Effect.runPromise( + Effect.gen(function* () { + const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) + if (!session) return + yield* SessionRunner.Service.use((runner) => + runner.run({ sessionID: session.id, force: input.force }), + ).pipe(Effect.provide(locations.get(session.location))) + }).pipe(Effect.provide(ctx), Effect.scoped), + { signal }, + ) + + // Worker connection (native) hosts the runContinuation activity + the workflow. + const nativeConn = yield* Effect.acquireRelease( + Effect.promise(() => NativeConnection.connect({ address: ADDRESS })), + (conn) => Effect.promise(() => conn.close().catch(() => {})), + ) + const worker = yield* Effect.promise(() => + Worker.create({ + connection: nativeConn, + namespace: NAMESPACE, + taskQueue: TASK_QUEUE, + workflowsPath: fileURLToPath(new URL("./temporal-workflow.ts", import.meta.url)), + activities: makeActivities(drain), + }), + ) + const runHandle = worker.run() + runHandle.catch(() => {}) + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + worker.shutdown() + await runHandle.catch(() => {}) + }), + ) + + // Client connection drives the per-session workflows. + const clientConn = yield* Effect.acquireRelease( + Effect.promise(() => Connection.connect({ address: ADDRESS })), + (conn) => Effect.promise(() => conn.close().catch(() => {})), + ) + const client = new Client({ connection: clientConn, namespace: NAMESPACE }) + const started = new Set() + + const drive = (id: SessionSchema.ID, forced: boolean) => + Effect.promise(async () => { + await client.workflow.signalWithStart(WF.sessionExecution, { + taskQueue: TASK_QUEUE, + workflowId: workflowId(id), + args: [id], + signal: forced ? WF.force : WF.wake, + signalArgs: [], + }) + started.add(id) + }) + + yield* Effect.logInfo("SessionExecutionTemporal ready").pipe( + Effect.annotateLogs({ address: ADDRESS, taskQueue: TASK_QUEUE }), + ) + + return SessionExecution.Service.of({ + active: Effect.sync(() => new Set(started) as ReadonlySet), + wake: (id) => drive(id, false).pipe(Effect.asVoid), + // resume must return Effect; we drive a forced run but do not surface the + // typed error here (a follow-up: carry it back via a Temporal update). never <: RunError. + resume: (id) => drive(id, true).pipe(Effect.asVoid), + interrupt: (id) => + Effect.promise(async () => { + await client.workflow + .getHandle(workflowId(id)) + .signal(WF.interrupt) + .catch(() => {}) + started.delete(id) + }), + }) + }), +) + +export const node = makeGlobalNode({ + service: SessionExecution.Service, + layer, + deps: [SessionStore.node, LocationServiceMap.node], +}) diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index cc1b1ae6a55d..0aacf03f406e 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -10,6 +10,7 @@ import { SessionV2 } from "@opencode-ai/core/session" import { SessionExecution } from "@opencode-ai/core/session/execution" import { LocationServiceMap } from "@opencode-ai/core/location-service-map" import { SessionExecutionLocal } from "@opencode-ai/core/session/execution/local" +import { SessionExecutionTemporal } from "@opencode-ai/core/session/execution/temporal" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { HttpRouter, HttpServer } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" @@ -49,7 +50,13 @@ export function createEmbeddedRoutes() { } function makeRoutes(auth: Layer.Layer) { - const serviceLayer = AppNodeBuilder.build(applicationServices, [[SessionExecution.node, SessionExecutionLocal.node]]) + // Opt in to the Temporal-backed durable execution with OPENCODE_SESSION_EXECUTION=temporal; + // otherwise the stock in-process coordinator is used. + const executionNode = + process.env.OPENCODE_SESSION_EXECUTION === "temporal" + ? SessionExecutionTemporal.node + : SessionExecutionLocal.node + const serviceLayer = AppNodeBuilder.build(applicationServices, [[SessionExecution.node, executionNode]]) return HttpApiBuilder.layer(Api, { openapiPath: "/openapi.json" }).pipe( Layer.provide(handlers), From 65644d6153f3e1e8a3e58b950f204676249012e1 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 7 Aug 2026 22:38:27 -0700 Subject: [PATCH 003/103] Verified v2 crash recovery and documented Phase 2. Added the engine-level crash-recovery harness (kill the whole server mid-turn, restart, the turn continues from the event log) and updated the README: Phase 2 is built and verified, with the run recipe, the code layout, and the known limits (resume does not yet return the typed RunError; active is process-local; startup needs Temporal reachable). --- packages/temporal/README.md | 55 +++++++++++++++---- packages/temporal/scripts/v2-crash-test.sh | 62 ++++++++++++++++++++++ 2 files changed, 107 insertions(+), 10 deletions(-) create mode 100755 packages/temporal/scripts/v2-crash-test.sh diff --git a/packages/temporal/README.md b/packages/temporal/README.md index d7417e391db9..02a396590cdc 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -68,13 +68,48 @@ That is: the turn completed after the crash, Temporal re-drove the activity on a - The worker talks to the opencode server over unauthenticated HTTP by default. Run them together or set `OPENCODE_SERVER_PASSWORD` and pass the header. -## Phase 2 (next): a durable `SessionExecution` on the v2 engine - -opencode's newer v2 engine (`packages/core` + `packages/server`) is already event-sourced per -session and exposes a substitutable `SessionExecution` service (`active` / `resume` / `wake` / -`interrupt`) whose local impl comments "Future remote placement belongs here." Phase 2 provides a -Temporal-backed `SessionExecution`: a workflow per session (`resume` = start-or-signal, `wake` = -signal, `interrupt` = cancel) with `SessionRunner.run` (one continuation from recorded history) as -the activity. Because turn state lives in the event log, the workflow stays thin and the recovery -is engine-level, not a re-attach. It is wired by changing one binding in -`packages/server/src/routes.ts`. +## Phase 2 (built): a durable `SessionExecution` on the v2 engine + +opencode's v2 engine (`packages/core` + `packages/server`) is already event-sourced per session and +exposes a substitutable `SessionExecution` service (`active` / `resume` / `wake` / `interrupt`) +whose local impl comments "Future remote placement belongs here." Phase 2 provides a Temporal-backed +`SessionExecution` in `packages/core/src/session/execution/`: + +- `temporal-workflow.ts` — the pure per-session workflow (the Temporal equivalent of + `SessionRunCoordinator`: `wake`/`force` drive one drain, wakes coalesce, quiescent runs end). +- `temporal-activities.ts` — the `runContinuation` activity (heartbeats; forwards cancellation). +- `temporal.ts` — the `SessionExecution` layer + node: `wake` → `signalWithStart`, `resume` → + forced `signalWithStart`, `interrupt` → cancel signal; the drain is the local coordinator's body + (`SessionRunner.run`) run in the activity against the durable event log. The Temporal client and + an embedded worker are co-hosted in the server process (both run under bun). + +Wiring is one binding in `packages/server/src/routes.ts`, opt-in via +`OPENCODE_SESSION_EXECUTION=temporal`. Because turn state lives in the event log, the workflow stays +thin and recovery is engine-level: a run re-reads recorded history and continues, it does not +re-attach. + +### Run it + +```bash +temporal server start-dev --port 7237 +OPENAI_API_KEY=... OPENCODE_SESSION_EXECUTION=temporal TEMPORAL_ADDRESS=127.0.0.1:7237 \ + bun run --cwd packages/cli src/index.ts serve --port 4601 +``` + +Create a session and prompt it against `POST /api/session` and `POST /api/session/:id/prompt`; each +session runs as a Temporal workflow `session-exec-`. + +### Verified + +- With Temporal execution on, prompting a v2 session drove a full turn to completion (`step.ended`, + `TEMPORAL_V2_OK`), recorded as a completed per-session workflow. +- Engine-level crash recovery (`scripts/v2-crash-test.sh`): killing the whole server (with its + embedded worker) mid-turn, then restarting, still completes the turn. Temporal re-drives + `runContinuation` (attempt 2), the run continues from the event log, and the workflow completes. + +### Known limits + +- `resume` drives a forced run but does not yet carry the typed `RunError` back to the caller (a + follow-up: a Temporal update). `active` reports sessions this process started (process-local, like + the local coordinator), not a durable-visibility query. Layer construction connects to Temporal at + server startup, so the server needs Temporal reachable when `OPENCODE_SESSION_EXECUTION=temporal`. diff --git a/packages/temporal/scripts/v2-crash-test.sh b/packages/temporal/scripts/v2-crash-test.sh new file mode 100755 index 000000000000..307ee2a87359 --- /dev/null +++ b/packages/temporal/scripts/v2-crash-test.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Engine-level crash recovery for the v2 Temporal SessionExecution (Phase 2). +# +# Kills the whole v2 server (which co-hosts the embedded Temporal worker) mid-turn, restarts it, +# and shows the turn still completes: Temporal re-drives the runContinuation activity, and +# SessionRunner.run re-reads the durable event log and continues from where it stopped. +# +# Prereqs: a Temporal dev server on :7237, an OpenAI key at $OPENCODE_KEY_FILE (default +# ~/.config/ai363/llm.key), and the v2 server run with OPENCODE_SESSION_EXECUTION=temporal. +set -u +REPO=$(cd "$(dirname "$0")/../../.." && pwd) +KEY_FILE=${OPENCODE_KEY_FILE:-$HOME/.config/ai363/llm.key} +PORT=${OPENCODE_PORT:-4601} +B=http://127.0.0.1:$PORT/api +AUTH=$(printf 'opencode:%s' "$(cat "$HOME/.local/state/opencode/password")" | base64) +H="Authorization: Basic $AUTH" + +boot() { + cd "$REPO" + nohup env OPENAI_API_KEY="$(cat "$KEY_FILE")" OPENCODE_SESSION_EXECUTION=temporal TEMPORAL_ADDRESS=127.0.0.1:7237 \ + bun run --cwd packages/cli src/index.ts serve --port "$PORT" --hostname 127.0.0.1 >/tmp/oc-v2-temporal.log 2>&1 & + until lsof -ti tcp:$PORT >/dev/null 2>&1 && grep -q "SessionExecutionTemporal ready" /tmp/oc-v2-temporal.log 2>/dev/null; do sleep 1; done +} +killserver() { lsof -ti tcp:$PORT 2>/dev/null | xargs -r kill -9 2>/dev/null; pkill -9 -f "packages/cli src/index.ts serve" 2>/dev/null; sleep 1; } + +echo "[1] ensure server up"; { grep -q "SessionExecutionTemporal ready" /tmp/oc-v2-temporal.log 2>/dev/null && lsof -ti tcp:$PORT >/dev/null 2>&1; } || boot + +echo "[2] create + prompt a multi-step task" +SID=$(curl -sS -m10 -X POST $B/session -H "$H" -H 'content-type: application/json' -d '{"model":{"providerID":"openai","id":"gpt-5-mini"}}' | python3 -c 'import sys,json;print((json.load(sys.stdin).get("data") or {}).get("id",""))') +echo " SID=$SID" +curl -sS -m10 -o /dev/null -w ' prompt HTTP %{http_code}\n' -X POST $B/session/$SID/prompt -H "$H" -H 'content-type: application/json' \ + -d '{"prompt":{"text":"Do these strictly in order using your tools, one per step: (1) write a file a.txt containing STEP_A; (2) read a.txt; (3) write a file result.txt containing exactly the token CRASH_RECOVERED; (4) read result.txt and reply with only its contents."}}' + +echo "[3] let a few steps record, then KILL the whole server mid-turn" +sleep 7 +echo " events before crash: $(curl -sS -m8 $B/session/$SID/history -H "$H" | python3 -c 'import sys,json;print(len(json.load(sys.stdin).get("data",[])))')" +killserver; echo " server killed" + +echo "[4] restart server (embedded worker re-registers; Temporal re-drives)" +sleep 3; boot; echo " server back up" + +echo "[5] await turn completion post-recovery" +DONE=no +for i in $(seq 1 40); do + sleep 3 + r=$(curl -sS -m8 $B/session/$SID/history -H "$H" | python3 -c ' +import sys,json +items=json.load(sys.stdin).get("data",[]) +types=[e.get("type","") for e in items] +print("ENDED" if any(t.endswith("step.ended") for t in types) else "pending", "seen="+str("CRASH_RECOVERED" in json.dumps(items)))' 2>/dev/null) + echo " poll $i: $r"; [[ "$r" == ENDED* ]] && { DONE=yes; break; } +done + +echo "[6] evidence from Temporal" +temporal workflow show --address 127.0.0.1:7237 --workflow-id "session-exec-$SID" --output json 2>/dev/null > /tmp/v2wf.json +python3 - <<'PY' +import json +ev=json.load(open("/tmp/v2wf.json")).get("events",[]) +attempts=[int(e["activityTaskStartedEventAttributes"].get("attempt",1)) for e in ev if e.get("activityTaskStartedEventAttributes")] +print(" runContinuation attempts:", attempts, "| max:", max(attempts) if attempts else 0) +PY +echo "RESULT: turn completed post-crash = $DONE" From a07c05bf82fe81e3b23d3923ddd70b817a917182 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 7 Aug 2026 22:39:47 -0700 Subject: [PATCH 004/103] Typed the SessionExecution active set as SessionID. --- packages/core/src/session/execution/temporal.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 4ad4c326c971..18d0bfd87f8c 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -80,7 +80,7 @@ const layer = Layer.effect( (conn) => Effect.promise(() => conn.close().catch(() => {})), ) const client = new Client({ connection: clientConn, namespace: NAMESPACE }) - const started = new Set() + const started = new Set() const drive = (id: SessionSchema.ID, forced: boolean) => Effect.promise(async () => { @@ -99,7 +99,7 @@ const layer = Layer.effect( ) return SessionExecution.Service.of({ - active: Effect.sync(() => new Set(started) as ReadonlySet), + active: Effect.sync(() => new Set(started)), wake: (id) => drive(id, false).pipe(Effect.asVoid), // resume must return Effect; we drive a forced run but do not surface the // typed error here (a follow-up: carry it back via a Temporal update). never <: RunError. From 41c0e368d03a49c903d7c895cae107e9efb4beb5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 8 Aug 2026 00:39:20 -0700 Subject: [PATCH 005/103] Made resume await the run and surface its error as a RunError. resume previously drove a forced run fire-and-forget and returned void, so a run failure was swallowed. It now drives the run through a Temporal Update-with-Start and awaits the result: a genuine run error is thrown non-retryable by the activity (so only crashes/timeouts still retry), rejects the update, and the layer maps it to a RunError (carried as ContextSnapshotDecodeError with the original text in details). The per-session workflow is now long-lived with an idle timeout so update-with-start can always reach it. Verified (scripts/resume-check.ts): resume resolves on a healthy session and rejects on a failing one. Full typecheck stays green (30/30). --- .../session/execution/temporal-workflow.ts | 62 ++++++++++----- .../core/src/session/execution/temporal.ts | 58 +++++++++++--- packages/temporal/README.md | 14 +++- packages/temporal/scripts/resume-check.ts | 76 +++++++++++++++++++ 4 files changed, 177 insertions(+), 33 deletions(-) create mode 100644 packages/temporal/scripts/resume-check.ts diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index aa3f30bd3091..ede719737656 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -4,13 +4,15 @@ // `@opencode-ai/core`, no Node builtins. It only ever sees the sessionID string. All real work // (SessionRunner.run against the durable event log) happens in the runContinuation activity. // -// Semantics mirror the coordinator (run-coordinator.ts): a wake (or force) drives exactly one -// drain, repeated wakes coalesce into at most one follow-up, and the workflow ends when a drain -// finishes with nothing pending. A later wake starts a fresh run via signalWithStart. +// Semantics mirror the coordinator (run-coordinator.ts): drains are serialized (one at a time), +// a `wake` drives a drain and is tolerant of errors, and `resume` (an Update) drives a forced +// drain and returns its result to the caller (throwing the run's error). The workflow stays alive +// to serve later wakes/resumes and terminates after an idle period. import { proxyActivities, defineSignal, + defineUpdate, setHandler, condition, CancellationScope, @@ -21,46 +23,68 @@ import type { Activities } from "./temporal-activities" const { runContinuation } = proxyActivities({ startToCloseTimeout: "30 minutes", // Short heartbeat so a dead worker's in-flight drain is re-driven quickly; the run re-reads the - // durable log, so a retry is a safe re-attach. + // durable log, so a retry is a safe re-attach. A genuine run error is thrown non-retryable by the + // activity, so only crashes/timeouts actually retry. heartbeatTimeout: "10 seconds", retry: { maximumAttempts: 100 }, }) export const wake = defineSignal("wake") -export const force = defineSignal("force") export const interrupt = defineSignal("interrupt") +export const resume = defineUpdate("resume") + +const IDLE_TIMEOUT = "5 minutes" export async function sessionExecution(sessionID: string): Promise { - // Starting the workflow implies there is work to drain (it is started via signalWithStart). - let pendingWake = true - let forceNext = false + let pendingWake = true // started via signalWithStart -> there is work to drain let stopping = false + let draining = false + let handlers = 0 + + // Serialize drains, like the coordinator (one owner fiber per session at a time). + const drainOnce = async (force: boolean) => { + await condition(() => !draining || stopping) + if (stopping) return + draining = true + try { + await runContinuation({ sessionID, force }) + } finally { + draining = false + } + } setHandler(wake, () => { pendingWake = true }) - setHandler(force, () => { - pendingWake = true - forceNext = true - }) setHandler(interrupt, () => { stopping = true CancellationScope.current().cancel() }) + // resume = coordinator.run: force one drain and surface its result (a run error rejects the + // Update, so the caller observes it). + setHandler(resume, async () => { + handlers++ + try { + await drainOnce(true) + } finally { + handlers-- + } + }) for (;;) { - await condition(() => pendingWake || stopping) + const gotWork = await condition(() => pendingWake || stopping, IDLE_TIMEOUT) if (stopping) return - const f = forceNext + if (!gotWork) { + // Idle: terminate only when nothing is in flight. A later wake/resume starts a fresh run. + if (!draining && handlers === 0) return + continue + } pendingWake = false - forceNext = false try { - await runContinuation({ sessionID, force: f }) + await drainOnce(false) } catch (e) { + // wake tolerates run errors (the coordinator logs and moves on); only cancellation stops us. if (isCancellation(e)) return - throw e } - // Quiescent: no wake arrived while draining. End; a future wake starts a new run. - if (!pendingWake) return } } diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 18d0bfd87f8c..393a875b46cf 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -1,8 +1,9 @@ export * as SessionExecutionTemporal from "./temporal" import { fileURLToPath } from "node:url" -import { Effect, Layer } from "effect" -import { Client, Connection } from "@temporalio/client" +import { Cause, Effect, Exit, Layer } from "effect" +import { Client, Connection, WithStartWorkflowOperation } from "@temporalio/client" +import { ApplicationFailure } from "@temporalio/activity" import { NativeConnection, Worker } from "@temporalio/worker" import { LocationServiceMap } from "../../location-service-map" @@ -11,6 +12,7 @@ import { SessionRunner } from "../runner" import { SessionSchema } from "../schema" import { SessionStore } from "../store" import { SessionExecution } from "../execution" +import { ContextSnapshotDecodeError } from "../error" import { makeActivities, type DrainInput } from "./temporal-activities" import * as WF from "./temporal-workflow" @@ -19,6 +21,15 @@ const NAMESPACE = process.env.TEMPORAL_NAMESPACE ?? "default" const TASK_QUEUE = process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? "opencode-session-exec" const workflowId = (id: string) => `session-exec-${id}` +// The v2 RunError union has no generic member, so a run failure surfaced across the durable +// boundary is carried as a ContextSnapshotDecodeError with the original text in `details`. Faithful +// per-member reconstruction (Schema round-trip of the exact tagged error) is a further follow-up. +const toRunError = (sessionID: SessionSchema.ID, e: unknown): SessionRunner.RunError => { + const anyE = e as { cause?: { message?: string }; message?: string } | undefined + const message = anyE?.cause?.message ?? anyE?.message ?? String(e) + return new ContextSnapshotDecodeError({ sessionID, details: `session run failed: ${message}` }) +} + /** * A Temporal-backed SessionExecution. It makes each session a durable workflow: * - wake -> signalWithStart(wake) (start while idle, or coalesce into the running run) @@ -39,8 +50,8 @@ const layer = Layer.effect( // SessionRunner and all of its dependencies. const ctx = yield* Effect.context() - const drain = (input: DrainInput, signal: AbortSignal): Promise => - Effect.runPromise( + const drain = async (input: DrainInput, signal: AbortSignal): Promise => { + const exit = await Effect.runPromiseExit( Effect.gen(function* () { const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) if (!session) return @@ -50,6 +61,17 @@ const layer = Layer.effect( }).pipe(Effect.provide(ctx), Effect.scoped), { signal }, ) + if (Exit.isSuccess(exit)) return + const cause = exit.cause + if (Cause.hasInterruptsOnly(cause)) throw new Error("session run interrupted") + // A genuine run error is thrown non-retryable so Temporal surfaces it (to resume) rather than + // retrying; only crashes / task timeouts (never thrown here) go through the retry policy. + throw ApplicationFailure.create({ + message: Cause.pretty(cause), + type: "SessionRunError", + nonRetryable: true, + }) + } // Worker connection (native) hosts the runContinuation activity + the workflow. const nativeConn = yield* Effect.acquireRelease( @@ -82,13 +104,13 @@ const layer = Layer.effect( const client = new Client({ connection: clientConn, namespace: NAMESPACE }) const started = new Set() - const drive = (id: SessionSchema.ID, forced: boolean) => + const drive = (id: SessionSchema.ID) => Effect.promise(async () => { await client.workflow.signalWithStart(WF.sessionExecution, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), args: [id], - signal: forced ? WF.force : WF.wake, + signal: WF.wake, signalArgs: [], }) started.add(id) @@ -100,10 +122,26 @@ const layer = Layer.effect( return SessionExecution.Service.of({ active: Effect.sync(() => new Set(started)), - wake: (id) => drive(id, false).pipe(Effect.asVoid), - // resume must return Effect; we drive a forced run but do not surface the - // typed error here (a follow-up: carry it back via a Temporal update). never <: RunError. - resume: (id) => drive(id, true).pipe(Effect.asVoid), + wake: (id) => drive(id).pipe(Effect.asVoid), + // resume = coordinator.run: drive a forced run via an Update-with-Start and AWAIT its result, + // so a run error is surfaced to the caller (as a RunError) instead of being swallowed. + resume: (id) => + Effect.tryPromise({ + try: async () => { + const startOp = new WithStartWorkflowOperation(WF.sessionExecution, { + taskQueue: TASK_QUEUE, + workflowId: workflowId(id), + args: [id], + workflowIdConflictPolicy: "USE_EXISTING", + }) + await client.workflow.executeUpdateWithStart(WF.resume, { + startWorkflowOperation: startOp, + args: [], + }) + started.add(id) + }, + catch: (e) => toRunError(id, e), + }), interrupt: (id) => Effect.promise(async () => { await client.workflow diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 02a396590cdc..d7d2e796850d 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -109,7 +109,13 @@ session runs as a Temporal workflow `session-exec-`. ### Known limits -- `resume` drives a forced run but does not yet carry the typed `RunError` back to the caller (a - follow-up: a Temporal update). `active` reports sessions this process started (process-local, like - the local coordinator), not a durable-visibility query. Layer construction connects to Temporal at - server startup, so the server needs Temporal reachable when `OPENCODE_SESSION_EXECUTION=temporal`. +- `resume` awaits the forced run (via Update-with-Start) and surfaces its failure to the caller as a + `RunError`, carried as a `ContextSnapshotDecodeError` with the original error text in `details`; + faithful per-member reconstruction of the exact tagged error across the durable boundary is a + further follow-up. The per-session workflow is long-lived and self-terminates after an idle period, + so `active` reports sessions this process started (process-local), not a durable-visibility query. + Layer construction connects to Temporal at server startup, so the server needs Temporal reachable + when `OPENCODE_SESSION_EXECUTION=temporal`. + +`scripts/resume-check.ts` verifies it: `resume` resolves on a healthy session and rejects on a +failing one (the run error reaches the caller instead of being swallowed). diff --git a/packages/temporal/scripts/resume-check.ts b/packages/temporal/scripts/resume-check.ts new file mode 100644 index 000000000000..edcbf2a4c6da --- /dev/null +++ b/packages/temporal/scripts/resume-check.ts @@ -0,0 +1,76 @@ +// Verifies the v2 SessionExecution `resume` path: it must AWAIT the forced run and surface its +// result — resolve on a healthy session, reject on a failing one (a run error is no longer +// swallowed). Drives the workflow's `resume` Update via Update-with-Start, exactly as the +// SessionExecutionTemporal layer does. Needs the v2 server (OPENCODE_SESSION_EXECUTION=temporal) +// on :4601 and a Temporal dev server on :7237. + +import { readFileSync } from "node:fs" +import { Client, Connection, WithStartWorkflowOperation } from "@temporalio/client" + +const TEMPORAL = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" +const QUEUE = process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? "opencode-session-exec" +const B = "http://127.0.0.1:4601/api" +const AUTH = + "Basic " + + Buffer.from("opencode:" + readFileSync(`${process.env.HOME}/.local/state/opencode/password`, "utf8").trim()).toString( + "base64", + ) + +const headers = { authorization: AUTH, "content-type": "application/json" } +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +async function createSession(model?: { providerID: string; id: string }): Promise { + const r = await fetch(`${B}/session`, { method: "POST", headers, body: JSON.stringify(model ? { model } : {}) }) + const d: any = await r.json() + return (d.data ?? d).id +} +async function prompt(sid: string, text: string): Promise { + await fetch(`${B}/session/${sid}/prompt`, { method: "POST", headers, body: JSON.stringify({ prompt: { text } }) }) +} + +async function resume(client: Client, sid: string): Promise { + const startOp = new WithStartWorkflowOperation("sessionExecution", { + taskQueue: QUEUE, + workflowId: `session-exec-${sid}`, + args: [sid], + workflowIdConflictPolicy: "USE_EXISTING" as any, + }) + await client.workflow.executeUpdateWithStart("resume", { startWorkflowOperation: startOp, args: [] }) +} + +async function main() { + const client = new Client({ connection: await Connection.connect({ address: TEMPORAL }) }) + + const good = await createSession({ providerID: "openai", id: "gpt-5-mini" }) + await prompt(good, "Reply with exactly: HI") + await wait(7000) + let healthy = "?" + try { + await resume(client, good) + healthy = "RESOLVED" + } catch (e: any) { + healthy = "REJECTED:" + (e?.message ?? String(e)) + } + console.log("resume(healthy) ->", healthy) + + const bad = await createSession() // no model -> default endpoint is unavailable, the run fails + await prompt(bad, "Reply with exactly: HI") + await wait(7000) + let failing = "?" + try { + await resume(client, bad) + failing = "RESOLVED (unexpected)" + } catch (e: any) { + failing = "REJECTED: " + String(e?.message ?? e).slice(0, 120) + } + console.log("resume(bad-model)->", failing) + + const pass = healthy === "RESOLVED" && failing.startsWith("REJECTED") + console.log("RESUME-TYPED-ERROR:", pass ? "PASS" : "FAIL") + process.exit(pass ? 0 : 1) +} + +main().catch((e) => { + console.error(e) + process.exit(1) +}) From 119d7033b027f406755e2e2ef1b590e45fd7ecbb Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 8 Aug 2026 00:50:19 -0700 Subject: [PATCH 006/103] Backed SessionExecution active with Temporal visibility. active listed a process-local set of sessions this process had started, so it was empty after a restart. It now queries Temporal for the open per-session workflows (WorkflowType 'sessionExecution', Running) and maps their ids back to session ids, so it reflects durable state and survives a restart. The process-local set is gone. --- .../core/src/session/execution/temporal.ts | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 393a875b46cf..8d1ef52438df 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -102,26 +102,37 @@ const layer = Layer.effect( (conn) => Effect.promise(() => conn.close().catch(() => {})), ) const client = new Client({ connection: clientConn, namespace: NAMESPACE }) - const started = new Set() + const SESSION_PREFIX = "session-exec-" const drive = (id: SessionSchema.ID) => - Effect.promise(async () => { - await client.workflow.signalWithStart(WF.sessionExecution, { + Effect.promise(() => + client.workflow.signalWithStart(WF.sessionExecution, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), args: [id], signal: WF.wake, signalArgs: [], - }) - started.add(id) - }) + }), + ) yield* Effect.logInfo("SessionExecutionTemporal ready").pipe( Effect.annotateLogs({ address: ADDRESS, taskQueue: TASK_QUEUE }), ) return SessionExecution.Service.of({ - active: Effect.sync(() => new Set(started)), + // Durable and restart-surviving: the open per-session workflows in Temporal ARE the active + // set (unlike a process-local Set, which is empty after a restart). + active: Effect.promise(async () => { + const ids = new Set() + for await (const wf of client.workflow.list({ + query: "WorkflowType = 'sessionExecution' AND ExecutionStatus = 'Running'", + })) { + if (wf.workflowId.startsWith(SESSION_PREFIX)) { + ids.add(SessionSchema.ID.make(wf.workflowId.slice(SESSION_PREFIX.length))) + } + } + return ids + }), wake: (id) => drive(id).pipe(Effect.asVoid), // resume = coordinator.run: drive a forced run via an Update-with-Start and AWAIT its result, // so a run error is surfaced to the caller (as a RunError) instead of being swallowed. @@ -138,18 +149,16 @@ const layer = Layer.effect( startWorkflowOperation: startOp, args: [], }) - started.add(id) }, catch: (e) => toRunError(id, e), }), interrupt: (id) => - Effect.promise(async () => { - await client.workflow + Effect.promise(() => + client.workflow .getHandle(workflowId(id)) .signal(WF.interrupt) - .catch(() => {}) - started.delete(id) - }), + .catch(() => {}), + ), }) }), ) From 0ee88182d71d5a3c6989a764a2905eba8c9d0cc5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 8 Aug 2026 00:59:26 -0700 Subject: [PATCH 007/103] Reconstructed the exact tagged RunError across the Temporal boundary. resume previously surfaced run failures as a generic ContextSnapshotDecodeError carrier. The activity now encodes the error through a Schema.Union of every RunError member (run-error-codec.ts) into the non-retryable failure details, and the layer walks the failure chain and decodes it back into the exact tagged instance (e.g. LLMError with its reason), falling back to the carrier only if decoding fails. Verified: a unit round-trip reconstructs an LLMError faithfully, and scripts/resume-check.ts shows a failing resume rejects with the encoded _tag = LLM.Error reaching the caller. --- .../src/session/execution/run-error-codec.ts | 52 +++++++++++++++++++ .../core/src/session/execution/temporal.ts | 26 +++++++--- packages/temporal/README.md | 26 +++++----- packages/temporal/scripts/resume-check.ts | 13 ++++- 4 files changed, 97 insertions(+), 20 deletions(-) create mode 100644 packages/core/src/session/execution/run-error-codec.ts diff --git a/packages/core/src/session/execution/run-error-codec.ts b/packages/core/src/session/execution/run-error-codec.ts new file mode 100644 index 000000000000..efe7d8178c71 --- /dev/null +++ b/packages/core/src/session/execution/run-error-codec.ts @@ -0,0 +1,52 @@ +// Faithful round-trip of a SessionRunner.RunError across the Temporal boundary. Every member of the +// union is a Schema.TaggedErrorClass, so we can encode the error to JSON in the activity and decode +// it back into the exact tagged instance in the layer, instead of flattening it to a carrier. + +import { Schema } from "effect" +import { LLMError } from "@opencode-ai/llm" +import { Integration } from "../../integration" +import { SystemContext } from "../../system-context/index" +import { ToolOutputStore } from "../../tool-output-store" +import { ContextSnapshotDecodeError, MessageDecodeError } from "../error" +import { + ModelNotSelectedError, + ModelUnavailableError, + UnsupportedApiError, + VariantUnavailableError, +} from "../runner/model" +import type { SessionRunner } from "../runner" + +const RunErrorSchema = Schema.Union([ + LLMError, + ModelNotSelectedError, + ModelUnavailableError, + VariantUnavailableError, + UnsupportedApiError, + Integration.AuthorizationError, + MessageDecodeError, + ContextSnapshotDecodeError, + SystemContext.InitializationBlocked, + ToolOutputStore.StorageError, +]) + +const encode = Schema.encodeSync(RunErrorSchema) +const decode = Schema.decodeUnknownSync(RunErrorSchema) + +// Returns the JSON encoding of a RunError, or undefined if the value is not a known member (e.g. a +// defect); the caller then falls back to a plain message. +export function encodeRunError(error: unknown): unknown | undefined { + try { + return (encode as (e: unknown) => unknown)(error) + } catch { + return undefined + } +} + +// Reconstructs the exact tagged RunError from its JSON encoding, or undefined if it does not decode. +export function decodeRunError(payload: unknown): SessionRunner.RunError | undefined { + try { + return decode(payload) as SessionRunner.RunError + } catch { + return undefined + } +} diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 8d1ef52438df..9f061e350aef 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -14,6 +14,7 @@ import { SessionStore } from "../store" import { SessionExecution } from "../execution" import { ContextSnapshotDecodeError } from "../error" import { makeActivities, type DrainInput } from "./temporal-activities" +import { encodeRunError, decodeRunError } from "./run-error-codec" import * as WF from "./temporal-workflow" const ADDRESS = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" @@ -25,9 +26,18 @@ const workflowId = (id: string) => `session-exec-${id}` // boundary is carried as a ContextSnapshotDecodeError with the original text in `details`. Faithful // per-member reconstruction (Schema round-trip of the exact tagged error) is a further follow-up. const toRunError = (sessionID: SessionSchema.ID, e: unknown): SessionRunner.RunError => { - const anyE = e as { cause?: { message?: string }; message?: string } | undefined - const message = anyE?.cause?.message ?? anyE?.message ?? String(e) - return new ContextSnapshotDecodeError({ sessionID, details: `session run failed: ${message}` }) + // Walk the failure chain (WorkflowUpdateFailedError -> ActivityFailure -> ApplicationFailure) to + // the encoded run error the activity attached, and reconstruct the exact tagged error. + let node: any = e + for (let depth = 0; node && depth < 6; depth++) { + if (Array.isArray(node.details) && node.details.length > 0) { + const decoded = decodeRunError(node.details[0]) + if (decoded) return decoded + return new ContextSnapshotDecodeError({ sessionID, details: `session run failed: ${node.message}` }) + } + node = node.cause + } + return new ContextSnapshotDecodeError({ sessionID, details: `session run failed: ${(e as any)?.message ?? String(e)}` }) } /** @@ -65,11 +75,15 @@ const layer = Layer.effect( const cause = exit.cause if (Cause.hasInterruptsOnly(cause)) throw new Error("session run interrupted") // A genuine run error is thrown non-retryable so Temporal surfaces it (to resume) rather than - // retrying; only crashes / task timeouts (never thrown here) go through the retry policy. + // retrying; only crashes / task timeouts (never thrown here) go through the retry policy. The + // error is encoded faithfully in `details` so the caller can reconstruct the exact RunError. + const squashed = Cause.squash(cause) as { _tag?: string; message?: string } + const encoded = encodeRunError(squashed) throw ApplicationFailure.create({ - message: Cause.pretty(cause), - type: "SessionRunError", + message: squashed?.message ?? Cause.pretty(cause), + type: squashed?._tag ?? "SessionRunError", nonRetryable: true, + details: encoded === undefined ? undefined : [encoded], }) } diff --git a/packages/temporal/README.md b/packages/temporal/README.md index d7d2e796850d..a9cfb651e601 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -107,15 +107,17 @@ session runs as a Temporal workflow `session-exec-`. embedded worker) mid-turn, then restarting, still completes the turn. Temporal re-drives `runContinuation` (attempt 2), the run continues from the event log, and the workflow completes. -### Known limits - -- `resume` awaits the forced run (via Update-with-Start) and surfaces its failure to the caller as a - `RunError`, carried as a `ContextSnapshotDecodeError` with the original error text in `details`; - faithful per-member reconstruction of the exact tagged error across the durable boundary is a - further follow-up. The per-session workflow is long-lived and self-terminates after an idle period, - so `active` reports sessions this process started (process-local), not a durable-visibility query. - Layer construction connects to Temporal at server startup, so the server needs Temporal reachable - when `OPENCODE_SESSION_EXECUTION=temporal`. - -`scripts/resume-check.ts` verifies it: `resume` resolves on a healthy session and rejects on a -failing one (the run error reaches the caller instead of being swallowed). +### Notes + +- `resume` awaits the forced run (via Update-with-Start) and surfaces its failure as the **exact + tagged `RunError`**: the activity encodes the error through a `Schema.Union` of every RunError + member (`run-error-codec.ts`) into the failure details, and the layer reconstructs it, falling + back to a `ContextSnapshotDecodeError` carrying the text only if decoding fails. +- `active` queries Temporal for the open per-session workflows, so it reflects durable state and + survives a restart (not a process-local set). +- The per-session workflow is long-lived and self-terminates after an idle period. Layer + construction connects to Temporal at startup, so the server needs Temporal reachable when + `OPENCODE_SESSION_EXECUTION=temporal`. + +`scripts/resume-check.ts` verifies resume: it resolves on a healthy session and rejects on a failing +one with the original tagged error (`LLM.Error`) reconstructed across the boundary. diff --git a/packages/temporal/scripts/resume-check.ts b/packages/temporal/scripts/resume-check.ts index edcbf2a4c6da..195216a69bfc 100644 --- a/packages/temporal/scripts/resume-check.ts +++ b/packages/temporal/scripts/resume-check.ts @@ -57,15 +57,24 @@ async function main() { await prompt(bad, "Reply with exactly: HI") await wait(7000) let failing = "?" + let encodedTag: string | undefined try { await resume(client, bad) failing = "RESOLVED (unexpected)" } catch (e: any) { - failing = "REJECTED: " + String(e?.message ?? e).slice(0, 120) + for (let node = e, d = 0; node && d < 6; node = node.cause, d++) { + if (Array.isArray(node.details) && node.details[0]?._tag) { + encodedTag = node.details[0]._tag + break + } + } + failing = "REJECTED (encoded _tag=" + encodedTag + "): " + String(e?.message ?? e).slice(0, 90) } console.log("resume(bad-model)->", failing) - const pass = healthy === "RESOLVED" && failing.startsWith("REJECTED") + // The activity encodes the real RunError faithfully into the failure details, so the caller can + // reconstruct the exact tagged error (LLM.Error here) instead of a generic carrier. + const pass = healthy === "RESOLVED" && failing.startsWith("REJECTED") && encodedTag === "LLM.Error" console.log("RESUME-TYPED-ERROR:", pass ? "PASS" : "FAIL") process.exit(pass ? 0 : 1) } From bffcd748262ca4e77992fbe7c534d8a23ca77568 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 9 Aug 2026 01:10:41 -0700 Subject: [PATCH 008/103] Added a shared, durable event store so any worker resumes any session. The v2 engine event-sources to SQLite, so a session was resumable only on the host with the file. Point every worker at one shared store and any worker resumes: a new libSQL SqlClient backend (sqlite.libsql.ts, over @libsql/client) selected by OPENCODE_DB_URL gives a networked SQLite (sqld/Turso) across hosts, and OPENCODE_DB already shares a file same-host. Same SQLite dialect, so the schema and all migrations are unchanged; the local-only PRAGMAs are skipped for the shared backend. Verified: a full turn runs against a libSQL file: store (migrations + event log), and scripts/shared-store-failover.sh shows turn 1 on worker A, A killed, then a fresh worker B recalls turn 1's code word (two distinct worker identities) purely from the shared store. Remote-URL transaction atomicity is documented as the remaining networked-writer step. Full typecheck 30/30. --- bun.lock | 35 ++++++ packages/core/package.json | 16 +-- packages/core/src/database/database.ts | 52 ++++++--- packages/core/src/database/sqlite.libsql.ts | 105 ++++++++++++++++++ packages/core/src/flag/flag.ts | 4 + packages/temporal/README.md | 31 ++++++ .../temporal/scripts/shared-store-failover.sh | 86 ++++++++++++++ 7 files changed, 306 insertions(+), 23 deletions(-) create mode 100644 packages/core/src/database/sqlite.libsql.ts create mode 100644 packages/temporal/scripts/shared-store-failover.sh diff --git a/bun.lock b/bun.lock index 52233e590cb5..c6df1094397f 100644 --- a/bun.lock +++ b/bun.lock @@ -317,6 +317,7 @@ "@effect/platform-node": "catalog:", "@effect/sql-sqlite-bun": "catalog:", "@ff-labs/fff-bun": "0.9.4", + "@libsql/client": "^0.17.0", "@lydell/node-pty": "catalog:", "@npmcli/arborist": "9.4.0", "@npmcli/config": "10.8.1", @@ -1854,6 +1855,32 @@ "@leichtgewicht/ip-codec": ["@leichtgewicht/ip-codec@2.0.5", "", {}, "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw=="], + "@libsql/client": ["@libsql/client@0.17.4", "", { "dependencies": { "@libsql/core": "^0.17.4", "@libsql/hrana-client": "^0.10.0", "js-base64": "^3.7.5", "libsql": "^0.5.28", "promise-limit": "^2.7.0" } }, "sha512-lYayFWasDV78A+TjlEhr6ubb3odBV6OHjb+wdp8VQcyWWAEIjuwbCHaraEUS4m4yWoo0BvZo96It4VdzZRmRWw=="], + + "@libsql/core": ["@libsql/core@0.17.4", "", { "dependencies": { "js-base64": "^3.7.5" } }, "sha512-LqF9gIvnJ38nmAH1y/ChizHqDO/MO1wLgA96XrraulEEbqXxLjleSH92YWTolbuJKgPUmGu4aJk9W3UnAcxLOQ=="], + + "@libsql/darwin-arm64": ["@libsql/darwin-arm64@0.5.29", "", { "os": "darwin", "cpu": "arm64" }, "sha512-K+2RIB1OGFPYQbfay48GakLhqf3ArcbHqPFu7EZiaUcRgFcdw8RoltsMyvbj5ix2fY0HV3Q3Ioa/ByvQdaSM0A=="], + + "@libsql/darwin-x64": ["@libsql/darwin-x64@0.5.29", "", { "os": "darwin", "cpu": "x64" }, "sha512-OtT+KFHsKFy1R5FVadr8FJ2Bb1mghtXTyJkxv0trocq7NuHntSki1eUbxpO5ezJesDvBlqFjnWaYYY516QNLhQ=="], + + "@libsql/hrana-client": ["@libsql/hrana-client@0.10.0", "", { "dependencies": { "@libsql/isomorphic-ws": "^0.1.5", "js-base64": "^3.7.5" } }, "sha512-OoA4EMqRAC7kn7V2P6EQqRcpZf2W+AjsNIyCizBg339Tq/aMC7sRnzs3SklderhmQWAqEzvv8A2vhxVmWpkVvw=="], + + "@libsql/isomorphic-ws": ["@libsql/isomorphic-ws@0.1.5", "", { "dependencies": { "@types/ws": "^8.5.4", "ws": "^8.13.0" } }, "sha512-DtLWIH29onUYR00i0GlQ3UdcTRC6EP4u9w/h9LxpUZJWRMARk6dQwZ6Jkd+QdwVpuAOrdxt18v0K2uIYR3fwFg=="], + + "@libsql/linux-arm-gnueabihf": ["@libsql/linux-arm-gnueabihf@0.5.29", "", { "os": "linux", "cpu": "arm" }, "sha512-CD4n4zj7SJTHso4nf5cuMoWoMSS7asn5hHygsDuhRl8jjjCTT3yE+xdUvI4J7zsyb53VO5ISh4cwwOtf6k2UhQ=="], + + "@libsql/linux-arm-musleabihf": ["@libsql/linux-arm-musleabihf@0.5.29", "", { "os": "linux", "cpu": "arm" }, "sha512-2Z9qBVpEJV7OeflzIR3+l5yAd4uTOLxklScYTwpZnkm2vDSGlC1PRlueLaufc4EFITkLKXK2MWBpexuNJfMVcg=="], + + "@libsql/linux-arm64-gnu": ["@libsql/linux-arm64-gnu@0.5.29", "", { "os": "linux", "cpu": "arm64" }, "sha512-gURBqaiXIGGwFNEaUj8Ldk7Hps4STtG+31aEidCk5evMMdtsdfL3HPCpvys+ZF/tkOs2MWlRWoSq7SOuCE9k3w=="], + + "@libsql/linux-arm64-musl": ["@libsql/linux-arm64-musl@0.5.29", "", { "os": "linux", "cpu": "arm64" }, "sha512-fwgYZ0H8mUkyVqXZHF3mT/92iIh1N94Owi/f66cPVNsk9BdGKq5gVpoKO+7UxaNzuEH1roJp2QEwsCZMvBLpqg=="], + + "@libsql/linux-x64-gnu": ["@libsql/linux-x64-gnu@0.5.29", "", { "os": "linux", "cpu": "x64" }, "sha512-y14V0vY0nmMC6G0pHeJcEarcnGU2H6cm21ZceRkacWHvQAEhAG0latQkCtoS2njFOXiYIg+JYPfAoWKbi82rkg=="], + + "@libsql/linux-x64-musl": ["@libsql/linux-x64-musl@0.5.29", "", { "os": "linux", "cpu": "x64" }, "sha512-gquqwA/39tH4pFl+J9n3SOMSymjX+6kZ3kWgY3b94nXFTwac9bnFNMffIomgvlFaC4ArVqMnOZD3nuJ3H3VO1w=="], + + "@libsql/win32-x64-msvc": ["@libsql/win32-x64-msvc@0.5.29", "", { "os": "win32", "cpu": "x64" }, "sha512-4/0CvEdhi6+KjMxMaVbFM2n2Z44escBRoEYpR+gZg64DdetzGnYm8mcNLcoySaDJZNaBd6wz5DNdgRmcI4hXcg=="], + "@lukeed/ms": ["@lukeed/ms@2.0.2", "", {}, "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA=="], "@lydell/node-pty": ["@lydell/node-pty@1.2.0-beta.12", "", { "optionalDependencies": { "@lydell/node-pty-darwin-arm64": "1.2.0-beta.12", "@lydell/node-pty-darwin-x64": "1.2.0-beta.12", "@lydell/node-pty-linux-arm64": "1.2.0-beta.12", "@lydell/node-pty-linux-x64": "1.2.0-beta.12", "@lydell/node-pty-win32-arm64": "1.2.0-beta.12", "@lydell/node-pty-win32-x64": "1.2.0-beta.12" } }, "sha512-qIK890UwPupoj07osVvgOIa++1mxeHbcGry4PKRHhNVNs81V2SCG34eJr46GybiOmBtc8Sj5PB1/GGM5PL549g=="], @@ -1908,6 +1935,8 @@ "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.4", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow=="], + "@neon-rs/load": ["@neon-rs/load@0.0.4", "", {}, "sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw=="], + "@noble/hashes": ["@noble/hashes@2.2.0", "", {}, "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg=="], "@nodable/entities": ["@nodable/entities@2.1.1", "", {}, "sha512-Pig3HxDIoMgjdEH8OCf/dkcTmLFjJRjWuq8jSnklu284/TKOPibSRERmOykiwmyXTtv61mP+44f3GMx0tLAyjg=="], @@ -4380,6 +4409,8 @@ "leac": ["leac@0.6.0", "", {}, "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg=="], + "libsql": ["libsql@0.5.29", "", { "dependencies": { "@neon-rs/load": "^0.0.4", "detect-libc": "2.0.2" }, "optionalDependencies": { "@libsql/darwin-arm64": "0.5.29", "@libsql/darwin-x64": "0.5.29", "@libsql/linux-arm-gnueabihf": "0.5.29", "@libsql/linux-arm-musleabihf": "0.5.29", "@libsql/linux-arm64-gnu": "0.5.29", "@libsql/linux-arm64-musl": "0.5.29", "@libsql/linux-x64-gnu": "0.5.29", "@libsql/linux-x64-musl": "0.5.29", "@libsql/win32-x64-msvc": "0.5.29" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "arm", "x64", "arm64", ] }, "sha512-8lMP8iMgiBzzoNbAPQ59qdVcj6UaE/Vnm+fiwX4doX4Narook0a4GPKWBEv+CR8a1OwbfkgL18uBfBjWdF0Fzg=="], + "light-my-request": ["light-my-request@6.6.0", "", { "dependencies": { "cookie": "^1.0.1", "process-warning": "^4.0.0", "set-cookie-parser": "^2.6.0" } }, "sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A=="], "lightningcss": ["lightningcss@1.30.1", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-darwin-arm64": "1.30.1", "lightningcss-darwin-x64": "1.30.1", "lightningcss-freebsd-x64": "1.30.1", "lightningcss-linux-arm-gnueabihf": "1.30.1", "lightningcss-linux-arm64-gnu": "1.30.1", "lightningcss-linux-arm64-musl": "1.30.1", "lightningcss-linux-x64-gnu": "1.30.1", "lightningcss-linux-x64-musl": "1.30.1", "lightningcss-win32-arm64-msvc": "1.30.1", "lightningcss-win32-x64-msvc": "1.30.1" } }, "sha512-xi6IyHML+c9+Q3W0S4fCQJOym42pyurFiJUHEcEyHS0CeKzia4yZDEsLlqOFykxOdHpNy0NmvVO31vcSqAxJCg=="], @@ -4970,6 +5001,8 @@ "promise-call-limit": ["promise-call-limit@3.0.2", "", {}, "sha512-mRPQO2T1QQVw11E7+UdCJu7S61eJVWknzml9sC1heAdj1jxl0fWMBypIt9ZOcLFf8FkG995ZD7RnVk7HH72fZw=="], + "promise-limit": ["promise-limit@2.7.0", "", {}, "sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw=="], + "promise-retry": ["promise-retry@2.0.1", "", { "dependencies": { "err-code": "^2.0.2", "retry": "^0.12.0" } }, "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g=="], "promise.allsettled": ["promise.allsettled@1.0.7", "", { "dependencies": { "array.prototype.map": "^1.0.5", "call-bind": "^1.0.2", "define-properties": "^1.2.0", "es-abstract": "^1.22.1", "get-intrinsic": "^1.2.1", "iterate-value": "^1.0.2" } }, "sha512-hezvKvQQmsFkOdrZfYxUxkyxl8mgFQeT259Ajj9PXdbg9VzBCWrItOev72JyWxkCD5VSSqAeHmlN3tWx4DlmsA=="], @@ -6566,6 +6599,8 @@ "lazystream/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + "libsql/detect-libc": ["detect-libc@2.0.2", "", {}, "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw=="], + "light-my-request/process-warning": ["process-warning@4.0.1", "", {}, "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q=="], "lightningcss/detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], diff --git a/packages/core/package.json b/packages/core/package.json index afd13a2e95a4..57273934df76 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -61,10 +61,6 @@ "drizzle-kit": "catalog:" }, "dependencies": { - "@temporalio/activity": "^1.21.0", - "@temporalio/client": "^1.21.0", - "@temporalio/worker": "^1.21.0", - "@temporalio/workflow": "^1.21.0", "@ai-sdk/alibaba": "1.0.17", "@ai-sdk/amazon-bedrock": "4.0.112", "@ai-sdk/anthropic": "3.0.82", @@ -89,15 +85,17 @@ "@effect/opentelemetry": "catalog:", "@effect/platform-node": "catalog:", "@effect/sql-sqlite-bun": "catalog:", - "@lydell/node-pty": "catalog:", "@ff-labs/fff-bun": "0.9.4", + "@libsql/client": "^0.17.0", + "@lydell/node-pty": "catalog:", "@npmcli/arborist": "9.4.0", "@npmcli/config": "10.8.1", "@opencode-ai/effect-drizzle-sqlite": "workspace:*", "@opencode-ai/effect-sqlite-node": "workspace:*", "@opencode-ai/llm": "workspace:*", - "@opencode-ai/schema": "workspace:*", "@opencode-ai/plugin": "workspace:*", + "@opencode-ai/schema": "workspace:*", + "@openrouter/ai-sdk-provider": "2.9.0", "@opentelemetry/api": "1.9.0", "@opentelemetry/context-async-hooks": "2.6.1", "@opentelemetry/exporter-trace-otlp-http": "0.214.0", @@ -105,6 +103,10 @@ "@parcel/watcher": "2.5.1", "@silvia-odwyer/photon-node": "0.3.4", "@openrouter/ai-sdk-provider": "2.9.0", + "@temporalio/activity": "^1.21.0", + "@temporalio/client": "^1.21.0", + "@temporalio/worker": "^1.21.0", + "@temporalio/workflow": "^1.21.0", "ai-gateway-provider": "3.2.0", "bun-pty": "0.4.8", "cross-spawn": "catalog:", @@ -117,8 +119,8 @@ "google-auth-library": "10.5.0", "gray-matter": "4.0.3", "htmlparser2": "8.0.2", - "immer": "11.1.4", "ignore": "7.0.5", + "immer": "11.1.4", "jsonc-parser": "3.3.1", "mime-types": "3.0.2", "minimatch": "10.2.5", diff --git a/packages/core/src/database/database.ts b/packages/core/src/database/database.ts index d61adf047eac..669b4831d487 100644 --- a/packages/core/src/database/database.ts +++ b/packages/core/src/database/database.ts @@ -2,6 +2,7 @@ export * as Database from "./database" import { EffectDrizzleSqlite } from "@opencode-ai/effect-drizzle-sqlite" import { layer as sqliteLayer } from "#sqlite" +import { layer as libsqlLayer } from "./sqlite.libsql" import { Context, Effect, Layer } from "effect" import { Global } from "../global" import { Flag } from "../flag/flag" @@ -19,25 +20,38 @@ export interface Interface { export class Service extends Context.Service()("@opencode/v2/storage/Database") {} -const layer = Layer.effect( - Service, - Effect.gen(function* () { - const db = yield* makeDatabase +function makeServiceLayer(localPragmas: boolean) { + return Layer.effect( + Service, + Effect.gen(function* () { + const db = yield* makeDatabase - yield* db.run("PRAGMA journal_mode = WAL") - yield* db.run("PRAGMA synchronous = NORMAL") - yield* db.run("PRAGMA busy_timeout = 5000") - yield* db.run("PRAGMA cache_size = -64000") - yield* db.run("PRAGMA foreign_keys = ON") - yield* db.run("PRAGMA wal_checkpoint(PASSIVE)") - yield* DatabaseMigration.apply(db) + // Local-file tuning. A shared/networked store (libSQL) manages journaling itself and may + // reject these, so they are skipped there; the schema and migrations are identical either way. + if (localPragmas) { + yield* db.run("PRAGMA journal_mode = WAL") + yield* db.run("PRAGMA synchronous = NORMAL") + yield* db.run("PRAGMA busy_timeout = 5000") + yield* db.run("PRAGMA cache_size = -64000") + yield* db.run("PRAGMA foreign_keys = ON") + yield* db.run("PRAGMA wal_checkpoint(PASSIVE)") + } + yield* DatabaseMigration.apply(db) - return { db } - }).pipe(Effect.orDie), -) + return { db } + }).pipe(Effect.orDie), + ) +} export function layerFromPath(filename: string) { - return layer.pipe(Layer.provide(sqliteLayer({ filename }))) + return makeServiceLayer(true).pipe(Layer.provide(sqliteLayer({ filename }))) +} + +// A shared/durable event store: point every worker at one libSQL URL (a self-hosted sqld or Turso) +// so any worker can resume any session from the same log. Same SQLite dialect, so nothing in the +// schema/migrations/queries changes. +export function layerFromLibsql(url: string, authToken?: string) { + return makeServiceLayer(false).pipe(Layer.provide(libsqlLayer({ url, authToken }))) } export function path() { @@ -54,4 +68,10 @@ export function path() { return join(Global.Path.data, `opencode-${InstallationChannel.replace(/[^a-zA-Z0-9._-]/g, "-")}.db`) } -export const node = makeGlobalNode({ service: Service, layer: layerFromPath(path()), deps: [] }) +export const node = makeGlobalNode({ + service: Service, + layer: Flag.OPENCODE_DB_URL + ? layerFromLibsql(Flag.OPENCODE_DB_URL, Flag.OPENCODE_DB_AUTH_TOKEN) + : layerFromPath(path()), + deps: [], +}) diff --git a/packages/core/src/database/sqlite.libsql.ts b/packages/core/src/database/sqlite.libsql.ts new file mode 100644 index 000000000000..18fde13f40b8 --- /dev/null +++ b/packages/core/src/database/sqlite.libsql.ts @@ -0,0 +1,105 @@ +// A libSQL-backed SqlClient, so the event store can live in a shared/networked SQLite (Turso or a +// self-hosted sqld) that every worker points at -- the basis for any-worker resume across hosts. +// It speaks the same SQLite dialect as the bun/node drivers, so the schema and all migrations are +// unchanged; only the transport differs. `makeDatabase` needs only the generic `SqlClient`, so this +// layer provides just that (no Sqlite.Native / Sqlite.Drizzle). + +import { createClient } from "@libsql/client" +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Fiber from "effect/Fiber" +import { identity } from "effect/Function" +import * as Layer from "effect/Layer" +import * as Scope from "effect/Scope" +import * as Semaphore from "effect/Semaphore" +import * as Stream from "effect/Stream" +import * as Reactivity from "effect/unstable/reactivity/Reactivity" +import * as Client from "effect/unstable/sql/SqlClient" +import type { Connection } from "effect/unstable/sql/SqlConnection" +import { classifySqliteError, SqlError } from "effect/unstable/sql/SqlError" +import * as Statement from "effect/unstable/sql/Statement" + +export interface LibsqlConfig { + readonly url: string + readonly authToken?: string + readonly transformResultNames?: (str: string) => string + readonly transformQueryNames?: (str: string) => string +} + +const make = (options: LibsqlConfig) => + Effect.gen(function* () { + // intMode "number": our columns are text ids + epoch-millis / sequence integers, all well under + // 2^53, so a JS number is exact and we avoid bigint round-tripping through the SqlClient. + const native = yield* Effect.acquireRelease( + Effect.sync(() => createClient({ url: options.url, authToken: options.authToken, intMode: "number" })), + (client) => Effect.sync(() => client.close()), + ) + + const compiler = Statement.makeCompilerSqlite(options.transformQueryNames) + const transformRows = options.transformResultNames + ? Statement.defaultTransforms(options.transformResultNames).array + : undefined + + const fail = (cause: unknown) => + new SqlError({ + reason: classifySqliteError(cause, { message: "Failed to execute statement", operation: "execute" }), + }) + + const run = (query: string, params: ReadonlyArray = []) => + Effect.tryPromise({ + try: () => + native + .execute({ sql: query, args: params as never[] }) + .then((r) => r.rows as unknown as Array>), + catch: fail, + }) + + const runValues = (query: string, params: ReadonlyArray = []) => + Effect.tryPromise({ + try: () => + native + .execute({ sql: query, args: params as never[] }) + .then((r) => r.rows.map((row) => r.columns.map((c) => (row as Record)[c])) as Array), + catch: fail, + }) + + const connection = identity({ + execute(query, params, transformRows) { + return transformRows ? Effect.map(run(query, params), transformRows) : run(query, params) + }, + executeRaw(query, params) { + return run(query, params) + }, + executeValues(query, params) { + return runValues(query, params) + }, + executeUnprepared(query, params, transformRows) { + return this.execute(query, params, transformRows) + }, + executeStream() { + return Stream.die("executeStream not implemented") + }, + }) + + const semaphore = yield* Semaphore.make(1) + const acquirer = semaphore.withPermits(1)(Effect.succeed(connection)) + const transactionAcquirer = Effect.uninterruptibleMask((restore) => { + const fiber = Fiber.getCurrent()! + const scope = Context.getUnsafe(fiber.context, Scope.Scope) + return Effect.as( + Effect.tap(restore(semaphore.take(1)), () => Scope.addFinalizer(scope, semaphore.release(1))), + connection, + ) + }) + + return yield* Client.make({ + acquirer, + compiler, + transactionAcquirer, + spanAttributes: [["db.system.name", "sqlite"]], + transformRows, + }) + }) + +export const layer = (config: LibsqlConfig) => + Layer.effect(Client.SqlClient, make(config)).pipe(Layer.provide(Reactivity.layer)) diff --git a/packages/core/src/flag/flag.ts b/packages/core/src/flag/flag.ts index a0eb78a13e2a..20364dccdfdb 100644 --- a/packages/core/src/flag/flag.ts +++ b/packages/core/src/flag/flag.ts @@ -45,6 +45,10 @@ export const Flag = { OPENCODE_MODELS_URL: process.env["OPENCODE_MODELS_URL"], OPENCODE_MODELS_PATH: process.env["OPENCODE_MODELS_PATH"], OPENCODE_DB: process.env["OPENCODE_DB"], + // A libSQL URL (self-hosted sqld or Turso) for a shared, durable event store so any worker can + // resume any session. Takes precedence over OPENCODE_DB (the local file) when set. + OPENCODE_DB_URL: process.env["OPENCODE_DB_URL"], + OPENCODE_DB_AUTH_TOKEN: process.env["OPENCODE_DB_AUTH_TOKEN"], OPENCODE_WORKSPACE_ID: process.env["OPENCODE_WORKSPACE_ID"], OPENCODE_EXPERIMENTAL_WORKSPACES: enabledByExperimental("OPENCODE_EXPERIMENTAL_WORKSPACES"), diff --git a/packages/temporal/README.md b/packages/temporal/README.md index a9cfb651e601..e6cda08a7c14 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -121,3 +121,34 @@ session runs as a Temporal workflow `session-exec-`. `scripts/resume-check.ts` verifies resume: it resolves on a healthy session and rejects on a failing one with the original tagged error (`LLM.Error`) reconstructed across the boundary. + +## Shared, durable event store (any-worker resume) + +The v2 engine event-sources each session to a SQLite store. By default that is a local file, so a +session can only be resumed on the host holding the file. Point every worker at one **shared** store +and any worker resumes any session: Temporal load-balances `runContinuation` across the fleet, +`active` is visibility-backed, and the resumed worker reads the session purely from the shared log. + +- **Same host**: set `OPENCODE_DB` to one absolute path on all workers (WAL + `busy_timeout` allow + multiple processes). No code change. +- **Across hosts**: set `OPENCODE_DB_URL` to a libSQL URL (self-hosted `sqld` or Turso), with + `OPENCODE_DB_AUTH_TOKEN` if needed. `packages/core/src/database/sqlite.libsql.ts` provides a + `SqlClient` over `@libsql/client`; it speaks the same SQLite dialect, so the schema and all + migrations are unchanged. Selection is one env check in `database.ts`. + +`scripts/shared-store-failover.sh` verifies it: worker A handles turn 1 (a code word), A is killed, +and a fresh worker B (same queue, same store, never saw the session) handles turn 2 and recalls the +code word, which it can only do by loading turn 1 from the shared store. The two turns run on two +distinct worker identities. + +### Caveats + +- Run migrations once as a deploy step before starting N workers; the first-open migration guard is + process-local, so N cold workers migrating at once can race. +- The PRAGMAs (`journal_mode` / `synchronous` / `busy_timeout` / `cache_size` / `wal_checkpoint`) + are local-file semantics and are skipped for the shared/libSQL backend, which manages journaling + itself. +- The libSQL client issues each statement as its own request, so a `BEGIN`/`COMMIT` transaction is + atomic against an embedded (`file:`) store but not against a remote URL without the libSQL + batch/transaction API; wiring that (or using Postgres) is the remaining step for a networked + writer. Verified here against an embedded `file:` store and against a shared local file. diff --git a/packages/temporal/scripts/shared-store-failover.sh b/packages/temporal/scripts/shared-store-failover.sh new file mode 100644 index 000000000000..946810a2755b --- /dev/null +++ b/packages/temporal/scripts/shared-store-failover.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Any-worker resume on a shared store, shown deterministically via cross-worker continuity: +# worker A handles turn 1 (a code word), A is killed entirely, then a FRESH worker B (same task +# queue, same shared store, never saw the session) handles turn 2 and can only answer by loading +# turn 1 from the shared store. This is the fleet-durability story: Temporal (cross-worker +# execution) + a shared store (cross-worker state) = any worker resumes any session. +# (Execution re-drive after a crash is covered separately by v2-crash-test.sh.) +# +# Prereqs: a Temporal dev server on :7237, an OpenAI key, bun. +set -u +REPO=$(cd "$(dirname "$0")/../../.." && pwd) +KEY_FILE=${OPENCODE_KEY_FILE:-$HOME/.config/ai363/llm.key} +SHARED=${OPENCODE_SHARED_DB:-/tmp/oc-shared/opencode.db} +AUTH=$(printf 'opencode:%s' "$(cat "$HOME/.local/state/opencode/password" 2>/dev/null)" | base64) +H="Authorization: Basic $AUTH" + +boot() { # boot + cd "$REPO" + nohup env OPENAI_API_KEY="$(cat "$KEY_FILE")" OPENCODE_SESSION_EXECUTION=temporal TEMPORAL_ADDRESS=127.0.0.1:7237 \ + OPENCODE_DB="$SHARED" \ + bun run --cwd packages/cli src/index.ts serve --port "$1" --hostname 127.0.0.1 >"/tmp/oc-worker-$1.log" 2>&1 & + until lsof -ti tcp:"$1" >/dev/null 2>&1 && grep -q "SessionExecutionTemporal ready" "/tmp/oc-worker-$1.log" 2>/dev/null; do sleep 1; done +} +kill_port() { lsof -ti tcp:"$1" 2>/dev/null | xargs -r kill -9 2>/dev/null; } + +# turn ; waits for a NEW step.ended (past the pre-prompt baseline); echoes the +# event count that existed BEFORE this turn (so callers can slice out just this turn's events). +turn() { + local base=$1 sid=$2 text=$3 pre + pre=$(curl -sS -m8 "$base/session/$sid/history" -H "$H" | python3 -c 'import sys,json;print(len(json.load(sys.stdin).get("data",[])))' 2>/dev/null) + pre=${pre:-0} + curl -sS -m10 -o /dev/null -X POST "$base/session/$sid/prompt" -H "$H" -H 'content-type: application/json' \ + -d "$(python3 -c 'import json,sys;print(json.dumps({"prompt":{"text":sys.argv[1]}}))' "$text")" + for _ in $(seq 1 30); do + sleep 2 + local done + done=$(curl -sS -m8 "$base/session/$sid/history" -H "$H" | PRE=$pre python3 -c ' +import sys,json,os +items=json.load(sys.stdin).get("data",[]) +new=items[int(os.environ["PRE"]):] +print("yes" if any(e.get("type","").endswith("step.ended") for e in new) else "no")' 2>/dev/null) + [[ "$done" == "yes" ]] && { echo "$pre"; return 0; } + done + echo "$pre"; return 1 +} + +mkdir -p "$(dirname "$SHARED")"; rm -f "$SHARED"* +echo "[1] boot worker A :4601"; boot 4601 +BA=http://127.0.0.1:4601/api +SID=$(curl -sS -m10 -X POST $BA/session -H "$H" -H 'content-type: application/json' -d '{"model":{"providerID":"openai","id":"gpt-5-mini"}}' | python3 -c 'import sys,json;print((json.load(sys.stdin).get("data") or {}).get("id",""))') +echo " SID=$SID" + +echo "[2] turn 1 on A: set a code word" +N1=$(turn "$BA" "$SID" "Remember this code word for later: BANANA47. Just reply OK.") +echo " turn 1 done (events=$N1)" + +echo "[3] KILL worker A entirely"; kill_port 4601; sleep 2 + +echo "[4] boot a FRESH worker B :4602 (same queue + shared store, never saw this session)"; boot 4602 +BB=http://127.0.0.1:4602/api + +echo "[5] turn 2 on B: recall the code word (only possible by loading turn 1 from the shared store)" +N2=$(turn "$BB" "$SID" "What was the code word I asked you to remember? Reply with only that word.") +echo " turn 2 done (events=$N2)" + +echo "[6] verify B's turn-2 reply used shared state, and ran on B" +python3 - "$BB" "$SID" "$N2" "$AUTH" <<'PY' +import sys,json,urllib.request +base,sid,n2,auth=sys.argv[1],sys.argv[2],int(sys.argv[3] or 0),sys.argv[4] +req=urllib.request.Request(f"{base}/session/{sid}/history",headers={"Authorization":"Basic "+auth}) +items=json.load(urllib.request.urlopen(req,timeout=8)).get("data",[]) +# only turn-2 events (from turn 2's baseline); pull assistant text +new=items[n2:] +text=" ".join(json.dumps(e.get("data",{})) for e in new) +ok = "BANANA47" in text +print(" turn-2 reply recalled the code word from the shared store:", ok) +PY +echo "[7] evidence: A was dead during turn 2; worker that ran it" +temporal workflow show --address 127.0.0.1:7237 --workflow-id "session-exec-$SID" --output json 2>/dev/null > /tmp/failover-wf.json +python3 - <<'PY' +import json +ev=json.load(open("/tmp/failover-wf.json")).get("events",[]) +ids=sorted(set(e["activityTaskStartedEventAttributes"].get("identity") for e in ev if e.get("activityTaskStartedEventAttributes"))) +print(" runContinuation ran on worker identities:", ids) +PY +kill_port 4602 \ No newline at end of file From eb1fc454165d823cd7b229f597f3dec31f861a95 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 9 Aug 2026 11:11:14 -0700 Subject: [PATCH 009/103] Added a per-step Temporal turn (each step its own activity). OPENCODE_SESSION_EXECUTION=temporal ran a whole turn as one activity. New temporal-turn mode drives the turn one step at a time: SessionRunner gains runStep (one iteration of run's loop, reusing runTurn), the sessionTurn workflow loops a runTurnStep activity, and each step (one provider attempt + its tools) is its own activity with its own retry/timeout/visibility. The step loop is workflow control flow; turn semantics are unchanged. Selected by one env check in routes.ts. Verified: a create-then-read-then-reply turn recorded three runTurnStep activities under a sessionTurn workflow and completed. Finer per-model-call / per-tool granularity is a larger rewrite left for later. Full typecheck 30/30. --- .../session/execution/temporal-activities.ts | 41 +++++++++++ .../session/execution/temporal-workflow.ts | 71 ++++++++++++++++++- .../core/src/session/execution/temporal.ts | 58 +++++++++++++-- packages/core/src/session/runner/index.ts | 21 ++++++ packages/core/src/session/runner/llm.ts | 29 ++++++++ packages/server/src/routes.ts | 9 ++- packages/temporal/README.md | 11 +++ 7 files changed, 226 insertions(+), 14 deletions(-) diff --git a/packages/core/src/session/execution/temporal-activities.ts b/packages/core/src/session/execution/temporal-activities.ts index 3300b8520abb..cd7e81470a50 100644 --- a/packages/core/src/session/execution/temporal-activities.ts +++ b/packages/core/src/session/execution/temporal-activities.ts @@ -34,3 +34,44 @@ export function makeActivities( }, } } + +// Per-step variant (OPENCODE_SESSION_EXECUTION=temporal-turn): one runTurnStep activity = one step +// (one provider attempt + its tools). The workflow loops it, so each step is its own activity with +// its own retry/timeout/visibility. `promotion` is null (not undefined) so it serializes cleanly. +export interface StepDrainInput { + sessionID: string + step: number + promotion: string | null + first: boolean + force: boolean +} + +export interface StepDrainResult { + ran: boolean + continue: boolean + step: number + promotion: string | null +} + +export type StepActivities = { + runTurnStep(input: StepDrainInput): Promise +} + +export function makeStepActivities( + stepDrain: (input: StepDrainInput, signal: AbortSignal) => Promise, +): StepActivities { + return { + async runTurnStep(input) { + const beat = setInterval(() => { + try { + heartbeat() + } catch {} + }, 3000) + try { + return await stepDrain(input, Context.current().cancellationSignal) + } finally { + clearInterval(beat) + } + }, + } +} diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index ede719737656..7b5d91c98ae5 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -18,16 +18,19 @@ import { CancellationScope, isCancellation, } from "@temporalio/workflow" -import type { Activities } from "./temporal-activities" +import type { Activities, StepActivities, StepDrainResult } from "./temporal-activities" -const { runContinuation } = proxyActivities({ +const activityOptions = { startToCloseTimeout: "30 minutes", // Short heartbeat so a dead worker's in-flight drain is re-driven quickly; the run re-reads the // durable log, so a retry is a safe re-attach. A genuine run error is thrown non-retryable by the // activity, so only crashes/timeouts actually retry. heartbeatTimeout: "10 seconds", retry: { maximumAttempts: 100 }, -}) +} as const + +const { runContinuation } = proxyActivities(activityOptions) +const { runTurnStep } = proxyActivities(activityOptions) export const wake = defineSignal("wake") export const interrupt = defineSignal("interrupt") @@ -88,3 +91,65 @@ export async function sessionExecution(sessionID: string): Promise { } } } + +// Per-step variant (OPENCODE_SESSION_EXECUTION=temporal-turn): identical lifecycle, but a turn is +// driven one step at a time -- each step (one provider attempt + its tools) is its own +// runTurnStep activity, and the step loop is workflow control flow. The loop state (step / +// promotion / first) mirrors SessionRunner.run's loop and lives in the (deterministic) workflow. +export async function sessionTurn(sessionID: string): Promise { + let pendingWake = true + let stopping = false + let draining = false + let handlers = 0 + + const drainTurn = async (force: boolean) => { + await condition(() => !draining || stopping) + if (stopping) return + draining = true + try { + let step = 1 + let promotion: string | null = null + let first = true + for (;;) { + const r: StepDrainResult = await runTurnStep({ sessionID, step, promotion, first, force }) + if (!r.continue) break + step = r.step + promotion = r.promotion + first = false + } + } finally { + draining = false + } + } + + setHandler(wake, () => { + pendingWake = true + }) + setHandler(interrupt, () => { + stopping = true + CancellationScope.current().cancel() + }) + setHandler(resume, async () => { + handlers++ + try { + await drainTurn(true) + } finally { + handlers-- + } + }) + + for (;;) { + const gotWork = await condition(() => pendingWake || stopping, IDLE_TIMEOUT) + if (stopping) return + if (!gotWork) { + if (!draining && handlers === 0) return + continue + } + pendingWake = false + try { + await drainTurn(false) + } catch (e) { + if (isCancellation(e)) return + } + } +} diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 9f061e350aef..f0560ce882d2 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -13,7 +13,14 @@ import { SessionSchema } from "../schema" import { SessionStore } from "../store" import { SessionExecution } from "../execution" import { ContextSnapshotDecodeError } from "../error" -import { makeActivities, type DrainInput } from "./temporal-activities" +import { + makeActivities, + makeStepActivities, + type DrainInput, + type StepDrainInput, + type StepDrainResult, +} from "./temporal-activities" +import type { SessionInput } from "../input" import { encodeRunError, decodeRunError } from "./run-error-codec" import * as WF from "./temporal-workflow" @@ -22,6 +29,12 @@ const NAMESPACE = process.env.TEMPORAL_NAMESPACE ?? "default" const TASK_QUEUE = process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? "opencode-session-exec" const workflowId = (id: string) => `session-exec-${id}` +// temporal = one activity per turn; temporal-turn = one activity per step (the model call + its +// tools), with the step loop as workflow control flow. +const PER_STEP = process.env.OPENCODE_SESSION_EXECUTION === "temporal-turn" +const WORKFLOW = PER_STEP ? WF.sessionTurn : WF.sessionExecution +const WORKFLOW_TYPE = PER_STEP ? "sessionTurn" : "sessionExecution" + // The v2 RunError union has no generic member, so a run failure surfaced across the durable // boundary is carried as a ContextSnapshotDecodeError with the original text in `details`. Faithful // per-member reconstruction (Schema round-trip of the exact tagged error) is a further follow-up. @@ -87,6 +100,39 @@ const layer = Layer.effect( }) } + // Per-step drain: run exactly one step of the turn (used by temporal-turn mode). Same context + // and error encoding as the whole-turn drain; returns the next loop state to the workflow. + const stepDrain = async (input: StepDrainInput, signal: AbortSignal): Promise => { + const exit = await Effect.runPromiseExit( + Effect.gen(function* () { + const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) + if (!session) return { ran: false, continue: false, step: input.step, promotion: null } + const r = yield* SessionRunner.Service.use((runner) => + runner.runStep({ + sessionID: session.id, + step: input.step, + promotion: (input.promotion ?? undefined) as SessionInput.Delivery | undefined, + first: input.first, + force: input.force, + }), + ).pipe(Effect.provide(locations.get(session.location))) + return { ran: r.ran, continue: r.continue, step: r.step, promotion: r.promotion ?? null } + }).pipe(Effect.provide(ctx), Effect.scoped), + { signal }, + ) + if (Exit.isSuccess(exit)) return exit.value + const cause = exit.cause + if (Cause.hasInterruptsOnly(cause)) throw new Error("session run interrupted") + const squashed = Cause.squash(cause) as { _tag?: string; message?: string } + const encoded = encodeRunError(squashed) + throw ApplicationFailure.create({ + message: squashed?.message ?? Cause.pretty(cause), + type: squashed?._tag ?? "SessionRunError", + nonRetryable: true, + details: encoded === undefined ? undefined : [encoded], + }) + } + // Worker connection (native) hosts the runContinuation activity + the workflow. const nativeConn = yield* Effect.acquireRelease( Effect.promise(() => NativeConnection.connect({ address: ADDRESS })), @@ -98,7 +144,7 @@ const layer = Layer.effect( namespace: NAMESPACE, taskQueue: TASK_QUEUE, workflowsPath: fileURLToPath(new URL("./temporal-workflow.ts", import.meta.url)), - activities: makeActivities(drain), + activities: { ...makeActivities(drain), ...makeStepActivities(stepDrain) }, }), ) const runHandle = worker.run() @@ -120,7 +166,7 @@ const layer = Layer.effect( const drive = (id: SessionSchema.ID) => Effect.promise(() => - client.workflow.signalWithStart(WF.sessionExecution, { + client.workflow.signalWithStart(WORKFLOW, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), args: [id], @@ -130,7 +176,7 @@ const layer = Layer.effect( ) yield* Effect.logInfo("SessionExecutionTemporal ready").pipe( - Effect.annotateLogs({ address: ADDRESS, taskQueue: TASK_QUEUE }), + Effect.annotateLogs({ address: ADDRESS, taskQueue: TASK_QUEUE, workflow: WORKFLOW_TYPE }), ) return SessionExecution.Service.of({ @@ -139,7 +185,7 @@ const layer = Layer.effect( active: Effect.promise(async () => { const ids = new Set() for await (const wf of client.workflow.list({ - query: "WorkflowType = 'sessionExecution' AND ExecutionStatus = 'Running'", + query: `WorkflowType = '${WORKFLOW_TYPE}' AND ExecutionStatus = 'Running'`, })) { if (wf.workflowId.startsWith(SESSION_PREFIX)) { ids.add(SessionSchema.ID.make(wf.workflowId.slice(SESSION_PREFIX.length))) @@ -153,7 +199,7 @@ const layer = Layer.effect( resume: (id) => Effect.tryPromise({ try: async () => { - const startOp = new WithStartWorkflowOperation(WF.sessionExecution, { + const startOp = new WithStartWorkflowOperation(WORKFLOW, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), args: [id], diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index 634075dd91b2..c842be7dca61 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -4,6 +4,7 @@ import type { LLMError } from "@opencode-ai/llm" import { Context, Effect } from "effect" import { SessionSchema } from "../schema" import type { ContextSnapshotDecodeError, MessageDecodeError } from "../error" +import type { SessionInput } from "../input" import { SessionRunnerModel } from "./model" import type { SystemContext } from "../../system-context/index" import type { ToolOutputStore } from "../../tool-output-store" @@ -16,6 +17,23 @@ export type RunError = | SystemContext.InitializationBlocked | ToolOutputStore.Error +/** Input for one step (one provider attempt + its tools) of a turn. */ +export interface StepInput { + readonly sessionID: SessionSchema.ID + readonly step: number + readonly promotion: SessionInput.Delivery | undefined + readonly first: boolean + readonly force: boolean +} + +/** Result of one step: whether it ran, whether to continue, and the next loop state. */ +export interface StepResult { + readonly ran: boolean + readonly continue: boolean + readonly step: number + readonly promotion: SessionInput.Delivery | undefined +} + /** Runs one local continuation from already-recorded Session history. */ export interface Interface { /** Drains eligible durable work. Explicit runs perform one provider attempt even when no work is eligible. */ @@ -23,6 +41,9 @@ export interface Interface { readonly sessionID: SessionSchema.ID readonly force: boolean }) => Effect.Effect + /** Run exactly one step and report the next loop state, so a caller (e.g. a Temporal workflow) + * can drive the turn one step at a time. Mirrors one iteration of `run`'s loop. */ + readonly runStep: (input: StepInput) => Effect.Effect } export class Service extends Context.Service()("@opencode/v2/SessionRunner") {} diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 72c761e10d93..5fe313c33a38 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -405,8 +405,37 @@ const layer = Layer.effect( } }) + // One iteration of `run`'s loop, exposed so a Temporal workflow can drive the turn one step at + // a time (each step = one runTurn = one provider attempt + its tools). Semantics match `run`. + const runStep = Effect.fn("SessionRunner.runStep")(function* (input: { + readonly sessionID: SessionSchema.ID + readonly step: number + readonly promotion: SessionInput.Delivery | undefined + readonly first: boolean + readonly force: boolean + }) { + let promotion = input.promotion + if (input.first) { + const hasSteer = yield* SessionInput.hasPending(db, input.sessionID, "steer") + const hasQueue = hasSteer ? false : yield* SessionInput.hasPending(db, input.sessionID, "queue") + if (!input.force && !hasSteer && !hasQueue) + return { ran: false, continue: false, step: input.step, promotion: undefined } + yield* failInterruptedTools(input.sessionID) + promotion = hasSteer ? "steer" : hasQueue ? "queue" : undefined + } + const result = yield* runTurn(input.sessionID, promotion, input.step) + let needsContinuation = result.needsContinuation + if (!needsContinuation) needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer") + if (needsContinuation) + return { ran: true, continue: true, step: result.step + 1, promotion: "steer" as SessionInput.Delivery } + const moreQueue = yield* SessionInput.hasPending(db, input.sessionID, "queue") + if (moreQueue) return { ran: true, continue: true, step: 1, promotion: "queue" as SessionInput.Delivery } + return { ran: true, continue: false, step: result.step + 1, promotion: undefined } + }) + return Service.of({ run, + runStep, }) }), ) diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index 0aacf03f406e..4b8460dbe13e 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -50,12 +50,11 @@ export function createEmbeddedRoutes() { } function makeRoutes(auth: Layer.Layer) { - // Opt in to the Temporal-backed durable execution with OPENCODE_SESSION_EXECUTION=temporal; - // otherwise the stock in-process coordinator is used. + // Opt in to Temporal-backed durable execution: "temporal" runs one activity per turn, + // "temporal-turn" runs one activity per step. Otherwise the stock in-process coordinator is used. + const exec = process.env.OPENCODE_SESSION_EXECUTION const executionNode = - process.env.OPENCODE_SESSION_EXECUTION === "temporal" - ? SessionExecutionTemporal.node - : SessionExecutionLocal.node + exec === "temporal" || exec === "temporal-turn" ? SessionExecutionTemporal.node : SessionExecutionLocal.node const serviceLayer = AppNodeBuilder.build(applicationServices, [[SessionExecution.node, executionNode]]) return HttpApiBuilder.layer(Api, { openapiPath: "/openapi.json" }).pipe( diff --git a/packages/temporal/README.md b/packages/temporal/README.md index e6cda08a7c14..a678c0d8a6ac 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -107,6 +107,17 @@ session runs as a Temporal workflow `session-exec-`. embedded worker) mid-turn, then restarting, still completes the turn. Temporal re-drives `runContinuation` (attempt 2), the run continues from the event log, and the workflow completes. +### Per-step turns (`temporal-turn`) + +`OPENCODE_SESSION_EXECUTION=temporal` runs a whole turn in one `runContinuation` activity. +`OPENCODE_SESSION_EXECUTION=temporal-turn` instead drives the turn one **step** at a time: the +`sessionTurn` workflow loops a `runTurnStep` activity, so each step (one provider attempt + its +tools) is its own activity with its own retry/timeout/visibility, and the step loop is workflow +control flow. It reuses `SessionRunner.runStep` (one iteration of `run`'s loop), so the turn +semantics are unchanged. Verified: a create-then-read-then-reply turn recorded three `runTurnStep` +activities under a `sessionTurn` workflow and completed. Finer granularity (the model call and each +tool as separate activities) is a larger rewrite left for later; this is the per-step increment. + ### Notes - `resume` awaits the forced run (via Update-with-Start) and surfaces its failure as the **exact From ce0cee49969999c69de6f37e0519b9e9750cfdf8 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 9 Aug 2026 17:23:58 -0700 Subject: [PATCH 010/103] Made remote libSQL writes atomic via interactive transactions. The libSQL client runs each statement as its own auto-commit request, so a multi-statement event append (the `event_sequence` upsert plus the `event` insert) could tear on a crash against a remote store. The transaction connection now drives a real interactive libSQL transaction, so those writes commit all-or-nothing. Verified against a `file:` store; networked crash-atomicity still needs a live `sqld`/Turso to test. --- packages/core/src/database/sqlite.libsql.ts | 113 +++++++++++++++--- .../test/database-libsql-transaction.test.ts | 67 +++++++++++ 2 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 packages/core/test/database-libsql-transaction.test.ts diff --git a/packages/core/src/database/sqlite.libsql.ts b/packages/core/src/database/sqlite.libsql.ts index 18fde13f40b8..e789aab5cdfa 100644 --- a/packages/core/src/database/sqlite.libsql.ts +++ b/packages/core/src/database/sqlite.libsql.ts @@ -4,7 +4,7 @@ // unchanged; only the transport differs. `makeDatabase` needs only the generic `SqlClient`, so this // layer provides just that (no Sqlite.Native / Sqlite.Drizzle). -import { createClient } from "@libsql/client" +import { createClient, type ResultSet, type Transaction } from "@libsql/client" import * as Context from "effect/Context" import * as Effect from "effect/Effect" import * as Fiber from "effect/Fiber" @@ -26,6 +26,17 @@ export interface LibsqlConfig { readonly transformQueryNames?: (str: string) => string } +// The transaction-control statements the effect-drizzle layer emits onto the reserved connection. +// A remote libSQL client runs each execute() as an independent auto-commit request, so forwarding +// `begin`/`commit` as plain statements would NOT bind a transaction across the writes in between. +// We intercept them and drive a real interactive libSQL transaction instead (one pinned stream), +// which is atomic all-or-nothing. `rollback to savepoint` is a savepoint op, not a rollback, so it +// must fall through to the transaction. +const BEGIN = /^\s*begin\b/i +const COMMIT = /^\s*(commit|end)\b/i +const ROLLBACK = /^\s*rollback\s*(transaction)?\s*;?\s*$/i +const EMPTY = { rows: [], columns: [] } as unknown as ResultSet + const make = (options: LibsqlConfig) => Effect.gen(function* () { // intMode "number": our columns are text ids + epoch-millis / sequence integers, all well under @@ -45,21 +56,16 @@ const make = (options: LibsqlConfig) => reason: classifySqliteError(cause, { message: "Failed to execute statement", operation: "execute" }), }) - const run = (query: string, params: ReadonlyArray = []) => - Effect.tryPromise({ - try: () => - native - .execute({ sql: query, args: params as never[] }) - .then((r) => r.rows as unknown as Array>), - catch: fail, - }) + const toRows = (r: ResultSet) => r.rows as unknown as Array> + const toValues = (r: ResultSet) => + r.rows.map((row) => r.columns.map((c) => (row as Record)[c])) as Array + // Auto-commit connection: each statement is its own request. Used outside transactions. + const run = (query: string, params: ReadonlyArray = []) => + Effect.tryPromise({ try: () => native.execute({ sql: query, args: params as never[] }).then(toRows), catch: fail }) const runValues = (query: string, params: ReadonlyArray = []) => Effect.tryPromise({ - try: () => - native - .execute({ sql: query, args: params as never[] }) - .then((r) => r.rows.map((row) => r.columns.map((c) => (row as Record)[c])) as Array), + try: () => native.execute({ sql: query, args: params as never[] }).then(toValues), catch: fail, }) @@ -81,14 +87,91 @@ const make = (options: LibsqlConfig) => }, }) + // A per-transaction connection bound to one interactive libSQL transaction. All statements in + // the drizzle transaction scope run on the same pinned stream, so the writes commit atomically. + // We always open in "write" mode: the client gives no read-only hint, and a write transaction is + // correct for both reads and writes (refining read-only spans to "read" mode is a follow-up). + const makeTxConnection = () => { + let tx: Transaction | null = null + const exec = async (query: string, params: ReadonlyArray): Promise => { + if (BEGIN.test(query)) { + tx = await native.transaction("write") + return EMPTY + } + if (COMMIT.test(query)) { + if (tx) { + await tx.commit() + tx = null + } + return EMPTY + } + if (ROLLBACK.test(query)) { + if (tx) { + await tx.rollback() + tx = null + } + return EMPTY + } + // Data statements and savepoint ops (savepoint / release / rollback to savepoint) run inside + // the transaction. This connection is only handed out inside a transaction scope, where the + // drizzle layer always emits `begin` first, so a statement with no open transaction would + // mean auto-committing it alone (the non-atomic behavior this backend exists to avoid). + if (!tx) throw new Error("libSQL: statement on a transaction connection before begin") + return tx.execute({ sql: query, args: params as never[] }) + } + const close = async () => { + // Safety net: if the scope unwinds with the transaction still open (no commit/rollback was + // emitted), roll it back so the stream is released and nothing partial lingers. + if (tx) { + try { + await tx.rollback() + } catch {} + try { + tx.close() + } catch {} + tx = null + } + } + const conn = identity({ + execute(query, params, transformRows) { + const e = Effect.tryPromise({ try: () => exec(query, params).then(toRows), catch: fail }) + return transformRows ? Effect.map(e, transformRows) : e + }, + executeRaw(query, params) { + return Effect.tryPromise({ try: () => exec(query, params).then(toRows), catch: fail }) + }, + executeValues(query, params) { + return Effect.tryPromise({ try: () => exec(query, params).then(toValues), catch: fail }) + }, + executeUnprepared(query, params, transformRows) { + return this.execute(query, params, transformRows) + }, + executeStream() { + return Stream.die("executeStream not implemented") + }, + }) + return { conn, close } + } + const semaphore = yield* Semaphore.make(1) const acquirer = semaphore.withPermits(1)(Effect.succeed(connection)) const transactionAcquirer = Effect.uninterruptibleMask((restore) => { const fiber = Fiber.getCurrent()! const scope = Context.getUnsafe(fiber.context, Scope.Scope) + const { conn, close } = makeTxConnection() return Effect.as( - Effect.tap(restore(semaphore.take(1)), () => Scope.addFinalizer(scope, semaphore.release(1))), - connection, + Effect.tap(restore(semaphore.take(1)), () => + // Close the transaction (rollback if still open) before releasing the permit, so the next + // transaction never starts against a half-finished one. + Scope.addFinalizer( + scope, + Effect.gen(function* () { + yield* Effect.promise(() => close()) + yield* semaphore.release(1) + }), + ), + ), + conn, ) }) diff --git a/packages/core/test/database-libsql-transaction.test.ts b/packages/core/test/database-libsql-transaction.test.ts new file mode 100644 index 000000000000..7a9eb403ed32 --- /dev/null +++ b/packages/core/test/database-libsql-transaction.test.ts @@ -0,0 +1,67 @@ +import { describe, expect } from "bun:test" +import path from "path" +import { Effect, Exit, Layer } from "effect" +import { Database } from "@opencode-ai/core/database/database" +import { testEffect } from "./lib/effect" +import { tmpdir } from "./fixture/tmpdir" + +// The libSQL backend runs each statement as its own request, so a transaction is only atomic if the +// backend pins one interactive libSQL transaction for the BEGIN..COMMIT span. This is the write +// shape the event store uses (event_sequence upsert + event insert must commit together). Exercised +// against an embedded `file:` store; the interactive-tx API is identical for a remote URL, whose +// crash-atomicity still needs a live sqld/Turso to test. +const withLibsqlDb = (body: (db: Database.Interface["db"]) => Effect.Effect) => + Effect.acquireUseRelease( + Effect.promise(() => tmpdir()), + (tmp) => + Effect.gen(function* () { + const { db } = yield* Database.Service + return yield* body(db) + }).pipe(Effect.provide(Database.layerFromLibsql(`file:${path.join(tmp.path, "events.db")}`))), + (tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()), + ) + +const it = testEffect(Layer.empty) + +describe("Database libSQL backend", () => { + it.live("commits a multi-statement transaction atomically", () => + withLibsqlDb((db) => + Effect.gen(function* () { + yield* db.run("CREATE TABLE tx_probe (id TEXT PRIMARY KEY, n INTEGER NOT NULL)") + yield* db.transaction( + () => + Effect.gen(function* () { + yield* db.run("INSERT INTO tx_probe (id, n) VALUES ('a', 1)") + yield* db.run("INSERT INTO tx_probe (id, n) VALUES ('b', 2)") + }), + { behavior: "immediate" }, + ) + const row = yield* db.get<{ c: number }>("SELECT COUNT(*) AS c FROM tx_probe") + expect(Number(row?.c ?? 0)).toBe(2) + }), + ), + ) + + it.live("rolls back every write when a statement in the transaction fails", () => + withLibsqlDb((db) => + Effect.gen(function* () { + yield* db.run("CREATE TABLE tx_probe (id TEXT PRIMARY KEY, n INTEGER NOT NULL)") + yield* db.run("INSERT INTO tx_probe (id, n) VALUES ('seed', 0)") + const exit = yield* db + .transaction( + () => + Effect.gen(function* () { + yield* db.run("INSERT INTO tx_probe (id, n) VALUES ('a', 1)") + yield* Effect.fail(new Error("boom")) + }), + { behavior: "immediate" }, + ) + .pipe(Effect.exit) + expect(Exit.isFailure(exit)).toBe(true) + // The seed survives; the in-transaction insert was rolled back with the failure. + const row = yield* db.get<{ c: number }>("SELECT COUNT(*) AS c FROM tx_probe") + expect(Number(row?.c ?? 0)).toBe(1) + }), + ), + ) +}) From ed0359afb3af2229a614f8aa39f49130705ca8f2 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 9 Aug 2026 17:25:10 -0700 Subject: [PATCH 011/103] Updated the shared-store caveat for atomic remote writes. Remote libSQL writes now go through interactive transactions, so the caveat about torn multi-statement writes no longer applies; only the networked crash test stays pending. --- packages/temporal/README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index a678c0d8a6ac..0a4e53b577cf 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -159,7 +159,10 @@ distinct worker identities. - The PRAGMAs (`journal_mode` / `synchronous` / `busy_timeout` / `cache_size` / `wal_checkpoint`) are local-file semantics and are skipped for the shared/libSQL backend, which manages journaling itself. -- The libSQL client issues each statement as its own request, so a `BEGIN`/`COMMIT` transaction is - atomic against an embedded (`file:`) store but not against a remote URL without the libSQL - batch/transaction API; wiring that (or using Postgres) is the remaining step for a networked - writer. Verified here against an embedded `file:` store and against a shared local file. +- Multi-statement writes commit atomically on both backends. The libSQL client runs each statement + as its own auto-commit request, so a `BEGIN`/`COMMIT` emitted as plain statements would not bind a + transaction over a remote URL. The shared backend instead routes a transaction through a real + interactive libSQL transaction (one pinned stream), which is all-or-nothing. A commit/rollback + atomicity test runs against an embedded (`file:`) store + (`packages/core/test/database-libsql-transaction.test.ts`). The networked crash-atomicity itself + still needs a live `sqld`/Turso to integration-test. From 33b0f6ac77c7bd8630ac3444af2294fdaa5d6b51 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 9 Aug 2026 19:50:22 -0700 Subject: [PATCH 012/103] Closed interrupted tools on every step re-drive, not just the first. In temporal-turn mode a Temporal step retry re-invokes `runStep` with `first=false`, so `failInterruptedTools` never ran and the re-drive re-streamed a request with a `tool_use` and no `tool_result`. The provider rejects that, and with `maximumAttempts: 100` it becomes a poison loop. It now runs before every turn; on a healthy step (whose prior tools already settled) it is a no-op. --- packages/core/src/session/runner/llm.ts | 6 +- .../core/test/session-runner-resume.test.ts | 165 ++++++++++++++++++ 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 packages/core/test/session-runner-resume.test.ts diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 5fe313c33a38..bbb57d7c151a 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -420,9 +420,13 @@ const layer = Layer.effect( const hasQueue = hasSteer ? false : yield* SessionInput.hasPending(db, input.sessionID, "queue") if (!input.force && !hasSteer && !hasQueue) return { ran: false, continue: false, step: input.step, promotion: undefined } - yield* failInterruptedTools(input.sessionID) promotion = hasSteer ? "steer" : hasQueue ? "queue" : undefined } + // Close tools left pending/running by an interrupted attempt before every turn, not just the + // first. A mid-turn re-drive (first=false, from a Temporal step retry) would otherwise + // re-stream a request with a dangling tool_use and no tool_result, which the provider rejects + // -- a retry poison loop. This is a no-op on a healthy step (the prior step settled its tools). + yield* failInterruptedTools(input.sessionID) const result = yield* runTurn(input.sessionID, promotion, input.step) let needsContinuation = result.needsContinuation if (!needsContinuation) needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer") diff --git a/packages/core/test/session-runner-resume.test.ts b/packages/core/test/session-runner-resume.test.ts new file mode 100644 index 000000000000..9f7d4e99fcf7 --- /dev/null +++ b/packages/core/test/session-runner-resume.test.ts @@ -0,0 +1,165 @@ +// Resumability of a per-step turn (temporal-turn): a Temporal step retry re-invokes runStep on the +// same durable log. These tests seed a crashed in-flight step (Step.Started + tool events, no +// Step.Ended) and drive runStep to check the two recovery behaviors: +// - Slice 1: a dangling tool left by an interrupted attempt is closed on every step entry, not +// just the first, so a re-drive never re-streams a request with a tool_use and no tool_result. +// - Slice 2: a step whose tools already ran is finalized from the log without re-calling the model +// (a dying mock LLM proves the model is never re-streamed) or re-running a completed tool. +import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" +import { LLMEvent } from "@opencode-ai/llm" +import { Database } from "@opencode-ai/core/database/database" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { EventV2 } from "@opencode-ai/core/event" +import { PermissionV2 } from "@opencode-ai/core/permission" +import { AgentV2 } from "@opencode-ai/core/agent" +import { Config } from "@opencode-ai/core/config" +import { Project } from "@opencode-ai/core/project" +import { ProjectTable } from "@opencode-ai/core/project/sql" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { SessionV2 } from "@opencode-ai/core/session" +import { Snapshot } from "@opencode-ai/core/snapshot" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import { SessionRunner } from "@opencode-ai/core/session/runner" +import * as SessionRunnerLLM from "@opencode-ai/core/session/runner/llm" +import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model" +import { createLLMEventPublisher } from "@opencode-ai/core/session/runner/publish-llm-event" +import { ToolRegistry } from "@opencode-ai/core/tool/registry" +import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionStore } from "@opencode-ai/core/session/store" +import { SessionMessage } from "@opencode-ai/core/session/message" +import { ModelV2 } from "@opencode-ai/core/model" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { Location } from "@opencode-ai/core/location" +import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" +import { SystemContext } from "@opencode-ai/core/system-context" +import { SkillGuidance } from "@opencode-ai/core/skill/guidance" +import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" +import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" +import { Auth } from "@opencode-ai/llm/route" +import { describe, expect } from "bun:test" +import { Effect, Layer, Stream } from "effect" +import { testEffect } from "./lib/effect" + +const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) + .model({ id: "gpt-4o-mini" }) +const models = SessionRunnerModel.layerWith(() => Effect.succeed(model)) +const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) +const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) +const permission = Layer.mock(PermissionV2.Service, {}) + +// The model call is what a resume must NOT repeat, so tests drive it explicitly: `dying` fails if +// the runner streams at all (proves resume skips the model); `empty` is a benign one-step response. +const mockClient = (stream: LLMClientShape["stream"]) => + Layer.succeed( + LLMClient.Service, + LLMClient.Service.of({ + prepare: () => Effect.die("LLMClient.prepare should not be called"), + generate: () => Effect.die("LLMClient.generate should not be called"), + stream, + }), + ) +const dying: LLMClientShape["stream"] = () => Stream.die("LLMClient.stream should not be called on resume") +const empty: LLMClientShape["stream"] = () => + Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })]) + +const harness = (stream: LLMClientShape["stream"]) => + testEffect( + AppNodeBuilder.build( + LayerNode.group([ + Database.node, + EventV2.node, + SessionProjector.node, + SessionStore.node, + AgentV2.node, + ToolRegistry.node, + SessionRunnerModel.node, + SystemContextRegistry.node, + SkillGuidance.node, + ReferenceGuidance.node, + Config.node, + Snapshot.node, + SessionRunnerLLM.node, + ]), + [ + [LayerNodePlatform.llmClient, mockClient(stream)], + [PermissionV2.node, permission], + [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [SessionRunnerModel.node, models], + [SystemContextRegistry.node, systemContext], + [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })], + [SkillGuidance.node, skillGuidance], + [ReferenceGuidance.node, referenceGuidance], + [Config.node, config], + [Snapshot.node, Snapshot.noopLayer], + ], + ), + ) + +const sessionID = SessionV2.ID.make("ses_runner_resume") + +const seedSession = Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ id: sessionID, project_id: Project.ID.global, slug: "t", directory: "/project", title: "t", version: "t" }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) +}) + +// Record a step that started and issued a tool call but never published Step.Ended -- exactly what a +// worker crash between tool dispatch and step settlement leaves in the log. `settle` controls +// whether the tool already recorded its result (completed) or was still running at the crash. +const seedCrashedStep = (settle: boolean) => + Effect.gen(function* () { + const events = yield* EventV2.Service + const publisher = createLLMEventPublisher(events, { + sessionID, + agent: "build", + model: { id: ModelV2.ID.make("gpt-4o-mini"), providerID: ProviderV2.ID.make("openai") }, + }) + yield* publisher.publish(LLMEvent.toolCall({ id: "call_1", name: "read", input: { path: "a.txt" } })) + if (settle) + yield* publisher.publish( + LLMEvent.toolResult({ + id: "call_1", + name: "read", + result: { type: "content", value: [{ type: "text", text: "seeded" }] }, + output: { structured: {}, content: [{ type: "text", text: "seeded" }] }, + }), + ) + }) + +const toolPart = (messages: ReadonlyArray, callID: string) => { + for (const message of messages) { + if (message.type !== "assistant") continue + for (const part of message.content) if (part.type === "tool" && part.id === callID) return part + } + return undefined +} + +describe("SessionRunner resume", () => { + harness(empty).effect("closes a dangling tool on a mid-turn (first=false) re-drive", () => + Effect.gen(function* () { + yield* seedSession + yield* seedCrashedStep(false) // running, never settled + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + yield* runner.runStep({ sessionID, step: 2, promotion: "steer", first: false, force: false }) + const part = toolPart(yield* store.context(sessionID), "call_1") + expect(part?.type === "tool" ? part.state.status : undefined).toBe("error") + }), + ) +}) From 57e9bea8ef2692ee081d03ff6718230e71c47957 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 9 Aug 2026 19:55:34 -0700 Subject: [PATCH 013/103] Resumed a crashed step from the log instead of re-running its tools. A Temporal step retry re-invokes `runStep` on the same durable log. If the in-flight step already dispatched tools (`Tool.Called` is recorded before the side effect runs), re-streaming would re-run those side effects and append a duplicate assistant message. `runStep` now finalizes such a step from the log: completed tools keep their results, still-unsettled ones are failed, and a synthesized `Step.Ended` closes it without re-calling the model. A step with no dispatched tools is still re-streamed, which is safe. Token metering is 0 for the resumed step; faithful metering would need a durable sealed marker. --- packages/core/src/session/runner/llm.ts | 64 +++++++++++++++ .../core/test/session-runner-resume.test.ts | 77 +++++++++++++------ 2 files changed, 118 insertions(+), 23 deletions(-) diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index bbb57d7c151a..ed3442f3611d 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -29,6 +29,7 @@ import { SessionCompaction } from "../compaction" import { SessionEvent } from "../event" import { SessionHistory } from "../history" import { SessionInput } from "../input" +import { SessionMessage } from "../message" import { SessionSchema } from "../schema" import { SessionStore } from "../store" import { type RunError, Service } from "./index" @@ -405,6 +406,65 @@ const layer = Layer.effect( } }) + // Resume a crashed step from the durable log instead of re-streaming it. A Temporal step retry + // re-invokes runStep on the same log; if the in-flight step already DISPATCHED tools (Tool.Called + // is recorded before the side effect runs, so a running/completed tool may have run), re-streaming + // would re-run that side effect and duplicate the assistant message. Instead we close the step + // from the log: keep completed tool results, fail the ones still unsettled (their result never + // committed -- we can't know if they ran, so the model redoes them), and publish a synthesized + // Step.Ended. The model is NOT re-called. Returns undefined when there is nothing to finalize (a + // fresh step, or a partial with no dispatched tools, which is safe to re-stream). Token/cost + // metering is 0 for the resumed step only; faithful metering would need a durable step-sealed + // marker carrying the provider usage. + const resumeCrashedStep = Effect.fn("SessionRunner.resumeCrashedStep")(function* (input: { + readonly sessionID: SessionSchema.ID + readonly step: number + }) { + const context = yield* getContext(input.sessionID) + // At most one assistant is in flight (the projector supersedes older ones); it only exists at + // step entry on a re-drive, never on a fresh step. + const inFlight = context.findLast( + (message): message is SessionMessage.Assistant => message.type === "assistant" && !message.time.completed, + ) + if (!inFlight) return undefined + const toolParts = inFlight.content.filter( + (part): part is SessionMessage.AssistantTool => part.type === "tool", + ) + const dispatched = toolParts.some( + (part) => part.state.status === "running" || part.state.status === "completed", + ) + if (!dispatched) return undefined + // Fail the still-open tools (never re-run them); completed tools keep their recorded results. + yield* failInterruptedTools(input.sessionID) + const startSnapshot = inFlight.snapshot?.start + const endSnapshot = yield* snapshots.capture() + const files = + startSnapshot && endSnapshot + ? yield* snapshots + .files({ from: Snapshot.ID.make(startSnapshot), to: endSnapshot }) + .pipe(Effect.catch(() => Effect.succeed(undefined))) + : undefined + yield* events.publish(SessionEvent.Step.Ended, { + sessionID: input.sessionID, + timestamp: yield* DateTime.now, + assistantMessageID: inFlight.id, + finish: "tool-calls", + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + snapshot: endSnapshot, + files, + }) + // Mirror runStep's continuation tail: a step with local tool calls continues so the model sees + // the (reused or failed) results. + let needsContinuation = toolParts.some((part) => part.provider?.executed !== true) + if (!needsContinuation) needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer") + if (needsContinuation) + return { ran: true, continue: true, step: input.step + 1, promotion: "steer" as SessionInput.Delivery } + const moreQueue = yield* SessionInput.hasPending(db, input.sessionID, "queue") + if (moreQueue) return { ran: true, continue: true, step: 1, promotion: "queue" as SessionInput.Delivery } + return { ran: true, continue: false, step: input.step + 1, promotion: undefined } + }) + // One iteration of `run`'s loop, exposed so a Temporal workflow can drive the turn one step at // a time (each step = one runTurn = one provider attempt + its tools). Semantics match `run`. const runStep = Effect.fn("SessionRunner.runStep")(function* (input: { @@ -414,6 +474,10 @@ const layer = Layer.effect( readonly first: boolean readonly force: boolean }) { + // Re-drive of a crashed step: finalize it from the log rather than re-calling the model and + // re-running its already-dispatched tools. + const resumed = yield* resumeCrashedStep(input) + if (resumed) return resumed let promotion = input.promotion if (input.first) { const hasSteer = yield* SessionInput.hasPending(db, input.sessionID, "steer") diff --git a/packages/core/test/session-runner-resume.test.ts b/packages/core/test/session-runner-resume.test.ts index 9f7d4e99fcf7..1f085bb25a8d 100644 --- a/packages/core/test/session-runner-resume.test.ts +++ b/packages/core/test/session-runner-resume.test.ts @@ -119,28 +119,37 @@ const seedSession = Effect.gen(function* () { .pipe(Effect.orDie) }) -// Record a step that started and issued a tool call but never published Step.Ended -- exactly what a -// worker crash between tool dispatch and step settlement leaves in the log. `settle` controls -// whether the tool already recorded its result (completed) or was still running at the crash. -const seedCrashedStep = (settle: boolean) => - Effect.gen(function* () { - const events = yield* EventV2.Service - const publisher = createLLMEventPublisher(events, { - sessionID, - agent: "build", - model: { id: ModelV2.ID.make("gpt-4o-mini"), providerID: ProviderV2.ID.make("openai") }, - }) - yield* publisher.publish(LLMEvent.toolCall({ id: "call_1", name: "read", input: { path: "a.txt" } })) - if (settle) - yield* publisher.publish( - LLMEvent.toolResult({ - id: "call_1", - name: "read", - result: { type: "content", value: [{ type: "text", text: "seeded" }] }, - output: { structured: {}, content: [{ type: "text", text: "seeded" }] }, - }), - ) +// Each seed records a step that started but never published Step.Ended -- what a worker crash mid +// turn leaves in the log. +const seededPublisher = Effect.gen(function* () { + const events = yield* EventV2.Service + return createLLMEventPublisher(events, { + sessionID, + agent: "build", + model: { id: ModelV2.ID.make("gpt-4o-mini"), providerID: ProviderV2.ID.make("openai") }, }) +}) +// Only a tool INPUT started -- the call was never dispatched, so no side effect ran and re-streaming +// is safe. This exercises slice 1 (close the dangling tool so the re-drive is not a poison loop). +const seedPendingStep = Effect.gen(function* () { + const publisher = yield* seededPublisher + yield* publisher.publish(LLMEvent.toolInputStart({ id: "call_pending", name: "read" })) +}) +// Tools were dispatched: `call_done` recorded its result (completed), `call_running` did not (its +// side effect may have run). This exercises slice 2 (finalize from the log, never re-stream). +const seedDispatchedStep = Effect.gen(function* () { + const publisher = yield* seededPublisher + yield* publisher.publish(LLMEvent.toolCall({ id: "call_done", name: "read", input: { path: "a.txt" } })) + yield* publisher.publish( + LLMEvent.toolResult({ + id: "call_done", + name: "read", + result: { type: "content", value: [{ type: "text", text: "done" }] }, + output: { structured: {}, content: [{ type: "text", text: "done" }] }, + }), + ) + yield* publisher.publish(LLMEvent.toolCall({ id: "call_running", name: "read", input: { path: "b.txt" } })) +}) const toolPart = (messages: ReadonlyArray, callID: string) => { for (const message of messages) { @@ -154,12 +163,34 @@ describe("SessionRunner resume", () => { harness(empty).effect("closes a dangling tool on a mid-turn (first=false) re-drive", () => Effect.gen(function* () { yield* seedSession - yield* seedCrashedStep(false) // running, never settled + yield* seedPendingStep const runner = yield* SessionRunner.Service const store = yield* SessionStore.Service yield* runner.runStep({ sessionID, step: 2, promotion: "steer", first: false, force: false }) - const part = toolPart(yield* store.context(sessionID), "call_1") + const part = toolPart(yield* store.context(sessionID), "call_pending") expect(part?.type === "tool" ? part.state.status : undefined).toBe("error") }), ) + + harness(dying).effect("finalizes a dispatched step from the log without re-calling the model", () => + Effect.gen(function* () { + yield* seedSession + yield* seedDispatchedStep + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + // A dying stream would fail this call if the runner re-streamed; it resolving proves resume. + const result = yield* runner.runStep({ sessionID, step: 3, promotion: "steer", first: false, force: false }) + expect(result.continue).toBe(true) + const context = yield* store.context(sessionID) + const done = toolPart(context, "call_done") + const running = toolPart(context, "call_running") + // The completed tool keeps its recorded result (not re-run); the unsettled one is failed. + expect(done?.type === "tool" ? done.state.status : undefined).toBe("completed") + expect(running?.type === "tool" ? running.state.status : undefined).toBe("error") + // The step is finalized in place (no duplicate assistant message). + const assistants = context.filter((message) => message.type === "assistant") + expect(assistants).toHaveLength(1) + expect(assistants[0]?.type === "assistant" ? Boolean(assistants[0].time.completed) : false).toBe(true) + }), + ) }) From d60c89e6e43c75f3981a01031ea0067e559b96e0 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 9 Aug 2026 19:56:40 -0700 Subject: [PATCH 014/103] Documented the resumable per-step turn in the temporal README. Covers the every-entry tool-close fix and the log-resume of a crashed step, with the in-flight-tool honest limit. --- packages/temporal/README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 0a4e53b577cf..90a24ba18685 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -115,8 +115,19 @@ session runs as a Temporal workflow `session-exec-`. tools) is its own activity with its own retry/timeout/visibility, and the step loop is workflow control flow. It reuses `SessionRunner.runStep` (one iteration of `run`'s loop), so the turn semantics are unchanged. Verified: a create-then-read-then-reply turn recorded three `runTurnStep` -activities under a `sessionTurn` workflow and completed. Finer granularity (the model call and each -tool as separate activities) is a larger rewrite left for later; this is the per-step increment. +activities under a `sessionTurn` workflow and completed. + +A per-step re-drive resumes from the durable event log rather than re-running work. `runStep` closes +any tool left dangling by an interrupted attempt on every entry, not just the first. Without that, a +mid-turn retry (`first=false`) re-streamed a request carrying a `tool_use` with no `tool_result`, +which the provider rejects, a retry poison loop. And if the crashed step had already dispatched tools +it is finalized from the log: completed tool results are kept, still-unsettled tools are failed, and +a synthesized `Step.Ended` closes the step without re-calling the model. A tool in flight at the +instant of the crash is marked interrupted (its result never committed, so the harness cannot know +whether it ran) and the model redoes it; true safety for that window needs per-tool idempotency +metadata. Finer granularity (the model call and each tool as separate Temporal activities) would +un-fuse the eager tool dispatch and is left for later. Verified by +`packages/core/test/session-runner-resume.test.ts`. ### Notes From 945b3907287830b7acb4ff1150a118b16e087b37 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 9 Aug 2026 22:30:27 -0700 Subject: [PATCH 015/103] Documented the real cross-host resume boundary. The runner rebuilds context only from the shared DB, so the conversation resumes on any worker. The working tree is the one host-local correctness constraint; snapshots and retained tool-output files are viewing-only. Corrects the earlier overstated tool-output gap. --- packages/temporal/README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 90a24ba18685..841850a2fcbc 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -177,3 +177,25 @@ distinct worker identities. atomicity test runs against an embedded (`file:`) store (`packages/core/test/database-libsql-transaction.test.ts`). The networked crash-atomicity itself still needs a live `sqld`/Turso to integration-test. + +### What resumes cross-host, and what does not + +The runner rebuilds a session's LLM context purely from the shared DB (`SessionHistory.entriesForRunner` +then `toLLMMessages`); it never reads local disk to reconstruct context. So the **conversation** resumes +on any worker: messages, tool results (the bounded preview and structured output that the model sees), +prompt attachments (stored inline as `data:` URIs in the prompt), and credentials (`CredentialTable`) +all ride the shared store. + +Host-local state that does NOT ride the DB, so it is not reconstructed on a different host: + +- **The project working tree.** File-touching tools (read/edit/bash) operate on the local worktree, so + a turn that keeps editing files must resume on a worker that has that worktree. This is the one real + cross-host correctness constraint. Three ways to satisfy it: co-locate a session's workers by worktree + (session affinity via a per-worktree Temporal task queue), share the worktree (a networked + filesystem), or reconstruct it from the last snapshot on resume (needs a shared snapshot store). This + is a deployment/design choice, not covered here yet. +- **The snapshot store (`${data}/snapshot`) and the retained full tool-output files + (`${data}/tool-output`).** The runner never reads these to rebuild context: snapshot file-diffs are + best-effort (`Effect.catch` to `undefined`), and the model sees the bounded tool-output preview, not + the file. They only affect the diff/restore/revert features and full-output viewing. Point `${data}` + (the XDG data dir) at shared storage to make them portable. From ba508e66333cec9e48ff8ff442a1719f8c135e2b Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 9 Aug 2026 22:41:16 -0700 Subject: [PATCH 016/103] Ran the Temporal worker standalone, decoupled from the HTTP server. `OPENCODE_TEMPORAL_ROLE` (`both` default, `client`, `worker`) gates the embedded activity worker and the workflow client, so serve can run client-only and a worker fleet can scale independently. `packages/server/src/worker.ts` builds the same application context serve uses (`createWorkerLayer`) without the HTTP API, so a worker resumes a session purely from the shared store. Verified by `scripts/standalone-worker-smoke.sh`: a worker comes up with no serve process and registers a poller on the task queue. --- .../core/src/session/execution/temporal.ts | 73 +++++++++++++------ packages/server/src/routes.ts | 23 +++++- packages/server/src/worker.ts | 27 +++++++ packages/temporal/README.md | 22 ++++++ .../scripts/standalone-worker-smoke.sh | 67 +++++++++++++++++ 5 files changed, 187 insertions(+), 25 deletions(-) create mode 100644 packages/server/src/worker.ts create mode 100755 packages/temporal/scripts/standalone-worker-smoke.sh diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index f0560ce882d2..c46fd6c6ed7f 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -35,6 +35,14 @@ const PER_STEP = process.env.OPENCODE_SESSION_EXECUTION === "temporal-turn" const WORKFLOW = PER_STEP ? WF.sessionTurn : WF.sessionExecution const WORKFLOW_TYPE = PER_STEP ? "sessionTurn" : "sessionExecution" +// Role split so the worker fleet can run separately from the HTTP server. `both` (default) hosts the +// activity worker AND the workflow client in one process (the serve process). `client` makes serve +// drive workflows without hosting a worker; `worker` runs a standalone activity worker with no HTTP +// surface (see packages/server/src/worker.ts). +const ROLE = process.env.OPENCODE_TEMPORAL_ROLE ?? "both" +const HOST_WORKER = ROLE !== "client" +const HOST_CLIENT = ROLE !== "worker" + // The v2 RunError union has no generic member, so a run failure surfaced across the durable // boundary is carried as a ContextSnapshotDecodeError with the original text in `details`. Faithful // per-member reconstruction (Schema round-trip of the exact tagged error) is a further follow-up. @@ -133,28 +141,48 @@ const layer = Layer.effect( }) } - // Worker connection (native) hosts the runContinuation activity + the workflow. - const nativeConn = yield* Effect.acquireRelease( - Effect.promise(() => NativeConnection.connect({ address: ADDRESS })), - (conn) => Effect.promise(() => conn.close().catch(() => {})), - ) - const worker = yield* Effect.promise(() => - Worker.create({ - connection: nativeConn, - namespace: NAMESPACE, - taskQueue: TASK_QUEUE, - workflowsPath: fileURLToPath(new URL("./temporal-workflow.ts", import.meta.url)), - activities: { ...makeActivities(drain), ...makeStepActivities(stepDrain) }, - }), - ) - const runHandle = worker.run() - runHandle.catch(() => {}) - yield* Effect.addFinalizer(() => - Effect.promise(async () => { - worker.shutdown() - await runHandle.catch(() => {}) - }), - ) + // Worker connection (native) hosts the runContinuation activity + the workflow. Skipped in + // client-only role so serve can run without an embedded worker. + if (HOST_WORKER) { + const nativeConn = yield* Effect.acquireRelease( + Effect.promise(() => NativeConnection.connect({ address: ADDRESS })), + (conn) => Effect.promise(() => conn.close().catch(() => {})), + ) + const worker = yield* Effect.promise(() => + Worker.create({ + connection: nativeConn, + namespace: NAMESPACE, + taskQueue: TASK_QUEUE, + workflowsPath: fileURLToPath(new URL("./temporal-workflow.ts", import.meta.url)), + activities: { ...makeActivities(drain), ...makeStepActivities(stepDrain) }, + }), + ) + const runHandle = worker.run() + runHandle.catch(() => {}) + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + worker.shutdown() + await runHandle.catch(() => {}) + }), + ) + } + + const SESSION_PREFIX = "session-exec-" + + // Worker-only process: it hosts activities but drives no workflows, so the client methods are + // unused. Return a service whose driving methods fail loudly if something unexpectedly calls them. + if (!HOST_CLIENT) { + yield* Effect.logInfo("SessionExecutionTemporal worker ready").pipe( + Effect.annotateLogs({ address: ADDRESS, taskQueue: TASK_QUEUE, workflow: WORKFLOW_TYPE, role: ROLE }), + ) + const clientOnly = Effect.die("SessionExecution client is not hosted when OPENCODE_TEMPORAL_ROLE=worker") + return SessionExecution.Service.of({ + active: Effect.succeed(new Set()), + wake: () => clientOnly, + resume: () => clientOnly, + interrupt: () => clientOnly, + }) + } // Client connection drives the per-session workflows. const clientConn = yield* Effect.acquireRelease( @@ -162,7 +190,6 @@ const layer = Layer.effect( (conn) => Effect.promise(() => conn.close().catch(() => {})), ) const client = new Client({ connection: clientConn, namespace: NAMESPACE }) - const SESSION_PREFIX = "session-exec-" const drive = (id: SessionSchema.ID) => Effect.promise(() => diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index 4b8460dbe13e..3d12c38f5463 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -49,13 +49,32 @@ export function createEmbeddedRoutes() { return makeRoutes(ServerAuth.Config.configLayer({ username: "opencode", password: Option.none() })) } -function makeRoutes(auth: Layer.Layer) { +// The application-service context (no HTTP surface), with the execution engine selected by env. +// Shared by the HTTP routes and the standalone worker entrypoint (src/worker.ts) so both build the +// exact same context. +export function createServiceLayer() { // Opt in to Temporal-backed durable execution: "temporal" runs one activity per turn, // "temporal-turn" runs one activity per step. Otherwise the stock in-process coordinator is used. const exec = process.env.OPENCODE_SESSION_EXECUTION const executionNode = exec === "temporal" || exec === "temporal-turn" ? SessionExecutionTemporal.node : SessionExecutionLocal.node - const serviceLayer = AppNodeBuilder.build(applicationServices, [[SessionExecution.node, executionNode]]) + return AppNodeBuilder.build(applicationServices, [[SessionExecution.node, executionNode]]) +} + +// The context for a standalone worker (src/worker.ts). Same services as serve, but with +// SessionExecution as a built member so constructing the layer eagerly starts the Temporal worker +// (with OPENCODE_TEMPORAL_ROLE=worker there is no HTTP handler to pull it in lazily). +export function createWorkerLayer() { + const exec = process.env.OPENCODE_SESSION_EXECUTION + const executionNode = + exec === "temporal" || exec === "temporal-turn" ? SessionExecutionTemporal.node : SessionExecutionLocal.node + return AppNodeBuilder.build(LayerNode.group([applicationServices, SessionExecution.node]), [ + [SessionExecution.node, executionNode], + ]) +} + +function makeRoutes(auth: Layer.Layer) { + const serviceLayer = createServiceLayer() return HttpApiBuilder.layer(Api, { openapiPath: "/openapi.json" }).pipe( Layer.provide(handlers), diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts new file mode 100644 index 000000000000..7cc8ef048e37 --- /dev/null +++ b/packages/server/src/worker.ts @@ -0,0 +1,27 @@ +// A standalone Temporal activity worker for the v2 durable execution, decoupled from the HTTP server. +// It builds the same application context serve uses (createServiceLayer) but hosts no HTTP surface: +// forcing SessionExecution constructs the Temporal worker, which then polls the task queue and runs +// session continuations. Run one or many of these next to (or instead of) an embedded worker. +// +// OPENCODE_TEMPORAL_ROLE=worker OPENCODE_SESSION_EXECUTION=temporal-turn \ +// TEMPORAL_ADDRESS=127.0.0.1:7237 OPENCODE_DB_URL=... \ +// bun run packages/server/src/worker.ts +// +// Note: file-touching tools run against the local working tree, so a worker must have the session's +// worktree present (co-locate by worktree, share the filesystem, or reconstruct from a snapshot). +import { Effect } from "effect" +import { createWorkerLayer } from "./routes" + +const program = Effect.gen(function* () { + // createWorkerLayer builds SessionExecution as a member, so providing it here starts the Temporal + // worker (OPENCODE_TEMPORAL_ROLE must be worker or both). + yield* Effect.logInfo("opencode v2 Temporal worker running").pipe( + Effect.annotateLogs({ role: process.env.OPENCODE_TEMPORAL_ROLE ?? "both" }), + ) + yield* Effect.never +}).pipe(Effect.provide(createWorkerLayer()), Effect.scoped) + +Effect.runPromise(program).catch((error) => { + console.error(error) + process.exit(1) +}) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 841850a2fcbc..3a37c5d8249d 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -144,6 +144,28 @@ un-fuse the eager tool dispatch and is left for later. Verified by `scripts/resume-check.ts` verifies resume: it resolves on a healthy session and rejects on a failing one with the original tagged error (`LLM.Error`) reconstructed across the boundary. +### Running workers separately + +By default the serve process hosts both the Temporal activity worker and the workflow client +(`OPENCODE_TEMPORAL_ROLE=both`). To scale workers independently of the HTTP server, run standalone +workers and point serve at client-only: + +```bash +# serve drives workflows, hosts no worker +OPENCODE_TEMPORAL_ROLE=client OPENCODE_SESSION_EXECUTION=temporal-turn ... serve --port 4601 + +# one or more standalone activity workers (no HTTP surface) +OPENCODE_TEMPORAL_ROLE=worker OPENCODE_SESSION_EXECUTION=temporal-turn \ + TEMPORAL_ADDRESS=127.0.0.1:7237 OPENCODE_DB_URL=... \ + bun run packages/server/src/worker.ts +``` + +`packages/server/src/worker.ts` builds the same application context serve uses (`createWorkerLayer`) +without the HTTP API, so a worker resumes a session purely from the shared store. +`scripts/standalone-worker-smoke.sh` verifies a worker comes up with no serve process and registers +on the task queue. Caveat: file-touching tools run against the local working tree, so a worker must +have the session's worktree present (see "What resumes cross-host"). + ## Shared, durable event store (any-worker resume) The v2 engine event-sources each session to a SQLite store. By default that is a local file, so a diff --git a/packages/temporal/scripts/standalone-worker-smoke.sh b/packages/temporal/scripts/standalone-worker-smoke.sh new file mode 100755 index 000000000000..1d178d774da1 --- /dev/null +++ b/packages/temporal/scripts/standalone-worker-smoke.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Proves the v2 Temporal worker runs standalone, decoupled from the HTTP server: with +# OPENCODE_TEMPORAL_ROLE=worker and no `serve` process, packages/server/src/worker.ts builds the app +# context, connects to Temporal, and polls the task queue. Boots a throwaway dev server, starts the +# worker, and asserts it comes up and registers a poller on the queue. No provider key needed (it +# registers without running a turn). +set -uo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +PORT=7241 +QUEUE="opencode-session-exec" +DB="$(mktemp -t worker-smoke-XXXX).db" +WLOG="$(mktemp -t worker-smoke-log-XXXX)" +TMPDIR_DEV="$(mktemp -d -t worker-smoke-dev-XXXX)" + +TEMPORAL_PID="" +WORKER_PID="" +cleanup() { + # `bun run` and `temporal` spawn children; kill children then the parent so nothing is orphaned. + [ -n "$WORKER_PID" ] && { pkill -P "$WORKER_PID" 2>/dev/null; kill "$WORKER_PID" 2>/dev/null; } + [ -n "$TEMPORAL_PID" ] && { pkill -P "$TEMPORAL_PID" 2>/dev/null; kill "$TEMPORAL_PID" 2>/dev/null; } + rm -f "$DB" "$WLOG" + rm -rf "$TMPDIR_DEV" +} +trap cleanup EXIT + +echo "starting temporal dev on :$PORT" +temporal server start-dev --port "$PORT" --db-filename "$TMPDIR_DEV/temporal.db" >/dev/null 2>&1 & +TEMPORAL_PID=$! +for i in $(seq 1 30); do + temporal operator cluster health --address "127.0.0.1:$PORT" >/dev/null 2>&1 && break + sleep 1 +done + +echo "starting standalone worker (role=worker, no serve)" +OPENCODE_TEMPORAL_ROLE=worker \ + OPENCODE_SESSION_EXECUTION=temporal-turn \ + TEMPORAL_ADDRESS="127.0.0.1:$PORT" \ + OPENCODE_DB="$DB" \ + bun run "$REPO/packages/server/src/worker.ts" >"$WLOG" 2>&1 & +WORKER_PID=$! + +up="" +for i in $(seq 1 40); do + if grep -q "Temporal worker running" "$WLOG" 2>/dev/null; then up="yes"; break; fi + kill -0 "$WORKER_PID" 2>/dev/null || { echo "worker exited early"; break; } + sleep 1 +done + +if [ -z "$up" ]; then + echo "WORKER-SMOKE: FAIL (worker did not come up)" + echo "--- worker log ---"; tail -30 "$WLOG" + exit 1 +fi +echo "worker up; giving it a moment to register pollers" +sleep 3 + +pollers="$(temporal task-queue describe --task-queue "$QUEUE" --address "127.0.0.1:$PORT" 2>/dev/null)" +echo "$pollers" | grep -qiE "poller|identity|@" && registered="yes" || registered="" + +if [ -n "$registered" ]; then + echo "WORKER-SMOKE: PASS (standalone worker up and polling queue=$QUEUE)" + exit 0 +fi +echo "WORKER-SMOKE: PARTIAL (worker up, but no poller reported by task-queue describe)" +echo "$pollers" | head -20 +exit 0 From 9b6863d960de5cfb94d36285aedacc7e090f9c36 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 9 Aug 2026 22:48:43 -0700 Subject: [PATCH 017/103] Serialized migrations across processes with a BEGIN IMMEDIATE lock. The migration guard was a process-local semaphore, so N cold workers pointing at one shared store raced: a TOCTOU table check let two processes both create the schema, or insert the same migration id. `apply` now runs its check-and-apply inside a single BEGIN IMMEDIATE transaction, so a concurrent start on another process waits and then observes the migrations already applied. A new test races five subprocesses against one fresh file. --- packages/core/script/migrate-once.ts | 17 +++++ packages/core/src/database/migration.ts | 69 +++++++++++++++---- packages/core/test/database-migration.test.ts | 24 +++++++ 3 files changed, 95 insertions(+), 15 deletions(-) create mode 100644 packages/core/script/migrate-once.ts diff --git a/packages/core/script/migrate-once.ts b/packages/core/script/migrate-once.ts new file mode 100644 index 000000000000..7d1895f9491f --- /dev/null +++ b/packages/core/script/migrate-once.ts @@ -0,0 +1,17 @@ +// Build the Database layer against the file in argv[2], which runs migrations, then exit. Used by +// the cross-process migration test to race several independent processes against one shared file. +import { Effect, Layer } from "effect" +import { Database } from "../src/database/database" + +const file = process.argv[2] +if (!file) { + console.error("usage: migrate-once.ts ") + process.exit(2) +} + +Effect.runPromise(Effect.scoped(Layer.build(Database.layerFromPath(file)))) + .then(() => process.exit(0)) + .catch((error) => { + console.error(error) + process.exit(1) + }) diff --git a/packages/core/src/database/migration.ts b/packages/core/src/database/migration.ts index 90dee8acbf3b..9e255dc7751d 100644 --- a/packages/core/src/database/migration.ts +++ b/packages/core/src/database/migration.ts @@ -16,27 +16,66 @@ export type Migration = { } export function apply(db: Database) { + // Serialize across processes, not just within one. N cold workers pointing at the same shared + // store would otherwise race: both create the schema, or both insert the same migration id (the + // table check was a read outside any lock, a TOCTOU). The process-local semaphore covers + // same-process concurrency; the single BEGIN IMMEDIATE transaction makes check-and-apply atomic + // and write-locked, so a concurrent start on another process waits and then observes the + // migrations already applied and does nothing. The incremental branch mirrors `applyOnly` on the + // shared transaction (keep the two in sync). return lock.withPermit( - Effect.gen(function* () { - const tables = yield* db.all<{ name: string }>( - sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`, - ) - if (tables.some((table) => table.name === "session")) return yield* applyOnly(db, migrations) - if (tables.length > 0) return yield* Effect.die("Database is not empty and has no session table") - yield* db.transaction((tx) => + db.transaction( + (tx) => Effect.gen(function* () { - yield* schema.up(tx) + const tables = yield* tx.all<{ name: string }>( + sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`, + ) + const hasSession = tables.some((table) => table.name === "session") + if (!hasSession && tables.length > 0) + return yield* Effect.die("Database is not empty and has no session table") + if (!hasSession) { + yield* schema.up(tx) + yield* tx.run( + sql`CREATE TABLE ${sql.identifier("migration")} (id TEXT PRIMARY KEY, time_completed INTEGER NOT NULL)`, + ) + yield* Effect.forEach(migrations, (migration) => + tx.run( + sql`INSERT INTO ${sql.identifier("migration")} (id, time_completed) VALUES (${migration.id}, ${Date.now()})`, + ), + ) + return + } yield* tx.run( - sql`CREATE TABLE ${sql.identifier("migration")} (id TEXT PRIMARY KEY, time_completed INTEGER NOT NULL)`, + sql`CREATE TABLE IF NOT EXISTS ${sql.identifier("migration")} (id TEXT PRIMARY KEY, time_completed INTEGER NOT NULL)`, ) - yield* Effect.forEach(migrations, (migration) => - tx.run( - sql`INSERT INTO ${sql.identifier("migration")} (id, time_completed) VALUES (${migration.id}, ${Date.now()})`, - ), + let completed = new Set( + (yield* tx.all<{ id: string }>(sql`SELECT id FROM ${sql.identifier("migration")}`)).map((row) => row.id), ) + if (completed.size === 0) { + if ( + yield* tx.get(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ${"__drizzle_migrations"}`) + ) { + yield* tx.run(sql` + INSERT OR IGNORE INTO ${sql.identifier("migration")} (id, time_completed) + SELECT name, ${Date.now()} + FROM ${sql.identifier("__drizzle_migrations")} + WHERE name IS NOT NULL + `) + completed = new Set( + (yield* tx.all<{ id: string }>(sql`SELECT id FROM ${sql.identifier("migration")}`)).map((row) => row.id), + ) + } + } + for (const migration of migrations) { + if (completed.has(migration.id)) continue + yield* migration.up(tx) + yield* tx.run( + sql`INSERT INTO ${sql.identifier("migration")} (id, time_completed) VALUES (${migration.id}, ${Date.now()})`, + ) + } }), - ) - }), + { behavior: "immediate" }, + ), ) } diff --git a/packages/core/test/database-migration.test.ts b/packages/core/test/database-migration.test.ts index b381cc7418a3..3f4b84162b2b 100644 --- a/packages/core/test/database-migration.test.ts +++ b/packages/core/test/database-migration.test.ts @@ -50,6 +50,30 @@ describe("DatabaseMigration", () => { ), ) }) + + test( + "serializes concurrent migration across processes on one path", + async () => { + await using tmp = await tmpdir() + const file = path.join(tmp.path, "cross-process.sqlite") + const script = fileURLToPath(new URL("../script/migrate-once.ts", import.meta.url)) + // Independent processes (not sharing the module-level semaphore) racing the same fresh file: + // the BEGIN IMMEDIATE transaction must serialize them so none errors on a duplicate CREATE or + // migration insert. + const runs = await Promise.all( + Array.from({ length: 5 }, () => $`bun ${script} ${file}`.quiet().nothrow()), + ) + for (const result of runs) expect(result.exitCode, result.stderr.toString()).toBe(0) + + await Effect.runPromise( + Effect.gen(function* () { + const { db } = yield* Database.Service + expect(yield* db.get(sql`SELECT count(*) as count FROM migration`)).toEqual({ count: migrations.length }) + }).pipe(Effect.provide(Database.layerFromPath(file)), Effect.scoped), + ) + }, + 60_000, + ) if (process.platform === "linux") { test("declared schema has no ungenerated migrations", async () => { const result = await $`bun ${fileURLToPath(new URL("../script/migration.ts", import.meta.url))} --check` From fd1a0e1c1d2ba200d0b1e0a62ee459a182022e9c Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 02:07:50 -0700 Subject: [PATCH 018/103] Re-ran idempotent tools on resume instead of failing them. A tool declares `idempotent` when it has no external side effect (the reads: `read`, `glob`, `grep`). On a crash resume a tool caught running is ambiguous (its result was never committed, so we cannot know if it ran), so a side-effecting tool is still failed and left for the model to redo. A declared-idempotent one is now re-settled for a real result, since re-running a pure read is safe. Extracted a shared `emitToolResult` so the resume path and the streaming publisher encode outcomes identically. --- packages/core/src/session/runner/llm.ts | 34 ++++++++- .../src/session/runner/publish-llm-event.ts | 73 +++++++++++++------ packages/core/src/tool/glob.ts | 1 + packages/core/src/tool/grep.ts | 1 + packages/core/src/tool/read.ts | 1 + packages/core/src/tool/registry.ts | 16 +++- packages/core/src/tool/tool.ts | 7 ++ .../core/test/session-runner-resume.test.ts | 45 +++++++++++- packages/temporal/README.md | 14 ++-- 9 files changed, 160 insertions(+), 32 deletions(-) diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index ed3442f3611d..d8885ea4b7db 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -34,7 +34,7 @@ import { SessionSchema } from "../schema" import { SessionStore } from "../store" import { type RunError, Service } from "./index" import { SessionRunnerModel } from "./model" -import { createLLMEventPublisher } from "./publish-llm-event" +import { createLLMEventPublisher, emitToolResult } from "./publish-llm-event" import { toLLMMessages } from "./to-llm-message" import { MAX_STEPS_PROMPT } from "./max-steps" import { Snapshot } from "../../snapshot" @@ -434,7 +434,37 @@ const layer = Layer.effect( (part) => part.state.status === "running" || part.state.status === "completed", ) if (!dispatched) return undefined - // Fail the still-open tools (never re-run them); completed tools keep their recorded results. + // A tool declared idempotent (a pure read) has no external side effect, so it is safe to + // re-run: re-settle it for a real result instead of failing it. Everything else still open is + // failed below -- we cannot know whether a side-effecting tool already ran. Completed tools + // keep their recorded results either way. + const session = yield* getSession(input.sessionID) + const agent = yield* agents.select(session.agent) + const materialization = yield* tools.materialize(agent.info?.permissions) + for (const part of toolParts) { + if (part.state.status !== "running") continue + if (!materialization.idempotent(part.name)) continue + const settlement = yield* materialization.settle({ + sessionID: session.id, + agent: agent.id, + assistantMessageID: inFlight.id, + call: LLMEvent.toolCall({ id: part.id, name: part.name, input: part.state.input }), + }) + yield* emitToolResult(events, { + sessionID: session.id, + assistantMessageID: inFlight.id, + callID: part.id, + result: settlement.result, + output: settlement.output, + outputPaths: settlement.outputPaths, + provider: { + executed: part.provider?.executed ?? false, + ...(part.provider?.metadata === undefined ? {} : { metadata: part.provider.metadata }), + }, + }) + } + // Fail whatever is still open (non-idempotent or never dispatched); completed and re-settled + // tools are terminal now and are skipped. yield* failInterruptedTools(input.sessionID) const startSnapshot = inFlight.snapshot?.start const endSnapshot = yield* snapshots.capture() diff --git a/packages/core/src/session/runner/publish-llm-event.ts b/packages/core/src/session/runner/publish-llm-event.ts index 33652a618c93..93dd3e86bb9c 100644 --- a/packages/core/src/session/runner/publish-llm-event.ts +++ b/packages/core/src/session/runner/publish-llm-event.ts @@ -50,6 +50,50 @@ const settledOutput = (value: ToolOutput | undefined, result: ToolResultValue): return { structured: record(settled.structured), content: settled.content } } +/** + * Publish the durable Tool.Success / Tool.Failed for one settled tool call. Shared by the streaming + * publisher below and the crash-resume path (which re-settles idempotent tools without a live + * publisher instance), so both encode the outcome identically. + */ +export const emitToolResult = ( + events: EventV2.Interface, + params: { + readonly sessionID: SessionSchema.ID + readonly assistantMessageID: SessionMessage.ID + readonly callID: string + readonly result: ToolResultValue + readonly output?: ToolOutput + readonly outputPaths?: ReadonlyArray + readonly provider: { readonly executed: boolean; readonly metadata?: ProviderMetadata } + }, +) => + Effect.gen(function* () { + const timestamp = yield* DateTime.now + const settled = settledOutput(params.output, params.result) + if ("error" in settled) { + yield* events.publish(SessionEvent.Tool.Failed, { + sessionID: params.sessionID, + timestamp, + assistantMessageID: params.assistantMessageID, + callID: params.callID, + error: settled.error, + result: params.result, + provider: params.provider, + }) + return + } + yield* events.publish(SessionEvent.Tool.Success, { + sessionID: params.sessionID, + timestamp, + assistantMessageID: params.assistantMessageID, + callID: params.callID, + ...settled, + outputPaths: params.outputPaths ?? [], + ...(params.provider.executed ? { result: params.result } : {}), + provider: params.provider, + }) + }) + /** Persist one provider turn without executing tools or starting a continuation turn. */ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) => { const tools = new Map< @@ -344,32 +388,17 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) return yield* Effect.die(`Duplicate tool result: ${event.id}`) } tool.settled = true - const result = settledOutput(event.output, event.result) - const provider = { - executed: event.providerExecuted === true || tool.providerExecuted, - ...(event.providerMetadata === undefined ? {} : { metadata: event.providerMetadata }), - } - if ("error" in result) { - yield* events.publish(SessionEvent.Tool.Failed, { - sessionID: input.sessionID, - timestamp: yield* timestamp, - assistantMessageID: tool.assistantMessageID, - callID: event.id, - error: result.error, - result: event.result, - provider, - }) - return - } - yield* events.publish(SessionEvent.Tool.Success, { + yield* emitToolResult(events, { sessionID: input.sessionID, - timestamp: yield* timestamp, assistantMessageID: tool.assistantMessageID, callID: event.id, - ...result, + result: event.result, + output: event.output, outputPaths, - ...(provider.executed ? { result: event.result } : {}), - provider, + provider: { + executed: event.providerExecuted === true || tool.providerExecuted, + ...(event.providerMetadata === undefined ? {} : { metadata: event.providerMetadata }), + }, }) return } diff --git a/packages/core/src/tool/glob.ts b/packages/core/src/tool/glob.ts index f8bd1869e11e..8762af84ec94 100644 --- a/packages/core/src/tool/glob.ts +++ b/packages/core/src/tool/glob.ts @@ -47,6 +47,7 @@ const layer = Layer.effectDiscard( [name]: Tool.make({ description: "Find files by glob pattern within the active Location. Returns concise relative file resources. Use a relative path to narrow the search and limit to bound the result count.", + idempotent: true, input: Input, output: Output, toModelOutput: ({ output }) => [ diff --git a/packages/core/src/tool/grep.ts b/packages/core/src/tool/grep.ts index f455bd4c8a0a..c56f8df81d9f 100644 --- a/packages/core/src/tool/grep.ts +++ b/packages/core/src/tool/grep.ts @@ -63,6 +63,7 @@ const layer = Layer.effectDiscard( [name]: Tool.make({ description: "Search file contents by regular expression within the active Location or an absolute managed tool-output file. Use a path to narrow the search, include to filter files by glob, and limit to bound the match count. Returns concise file resources, line numbers, and bounded line previews.", + idempotent: true, input: Input, output: Output, toModelOutput: ({ output }) => [ diff --git a/packages/core/src/tool/read.ts b/packages/core/src/tool/read.ts index 6961a8609118..9d71dad35f37 100644 --- a/packages/core/src/tool/read.ts +++ b/packages/core/src/tool/read.ts @@ -40,6 +40,7 @@ const layer = Layer.effectDiscard( [name]: Tool.make({ description: "Read a text file or supported image, page through a large UTF-8 text file by line offset, or list a directory page. Relative paths resolve from the current location; absolute paths inside it are accepted, while external absolute paths require external_directory approval.", + idempotent: true, input: Input, output: Output, toModelOutput: ({ input, output }) => { diff --git a/packages/core/src/tool/registry.ts b/packages/core/src/tool/registry.ts index 1c2dfe7ab459..de812e3c4eeb 100644 --- a/packages/core/src/tool/registry.ts +++ b/packages/core/src/tool/registry.ts @@ -9,7 +9,15 @@ import { SessionSchema } from "../session/schema" import { ToolOutputStore } from "../tool-output-store" import { Wildcard } from "../util/wildcard" import { ApplicationTools } from "./application-tools" -import { definition, permission, settle, validateName, type AnyTool, type RegistrationError } from "./tool" +import { + definition, + idempotent as toolIdempotent, + permission, + settle, + validateName, + type AnyTool, + type RegistrationError, +} from "./tool" import { Tools } from "./tools" import { makeLocationNode } from "../effect/app-node" @@ -29,6 +37,8 @@ export interface Interface { export interface Materialization { readonly definitions: ReadonlyArray readonly settle: (input: ExecuteInput) => Effect.Effect + /** Whether a tool declared itself side-effect-free (safe to re-run on a crash resume). */ + readonly idempotent: (name: string) => boolean } export interface Settlement { @@ -118,6 +128,10 @@ const registryLayer = Layer.effect( if (registration) return settleWith(input, registration.identity) return Effect.succeed({ result: { type: "error", value: `Unknown tool: ${input.call.name}` } }) }, + idempotent: (name) => { + const registration = registrations.get(name) + return registration ? toolIdempotent(registration.tool) : false + }, } }), }) diff --git a/packages/core/src/tool/tool.ts b/packages/core/src/tool/tool.ts index 1d9a82e9522d..803256c175a6 100644 --- a/packages/core/src/tool/tool.ts +++ b/packages/core/src/tool/tool.ts @@ -43,6 +43,10 @@ type Config< Structured extends SchemaType = Output, > = { readonly description: string + // Opt-in: this tool has no external side effect (a pure read), so it is safe to re-run. Used to + // re-settle a tool caught in flight at a crash instead of failing it. Defaults to false; anything + // that writes files, runs shell, or mutates a remote must leave it unset. + readonly idempotent?: boolean readonly input: Input readonly output: Output readonly structured?: Structured @@ -62,6 +66,7 @@ type Config< type Runtime = { readonly permission?: string + readonly idempotent: boolean readonly definition: (name: string) => ToolDefinition readonly settle: (call: ToolCall, context: Context) => Effect.Effect } @@ -76,6 +81,7 @@ export function make< const tool = Object.freeze({}) as Definition const definitions = new Map() runtimes.set(tool, { + idempotent: config.idempotent === true, definition: (name) => { const cached = definitions.get(name) if (cached) return cached @@ -146,6 +152,7 @@ export const withPermission = , Output extends Sch } export const permission = (tool: AnyTool, name: string) => runtimeOf(tool).permission ?? name +export const idempotent = (tool: AnyTool) => runtimeOf(tool).idempotent export const definition = (name: string, tool: AnyTool) => runtimeOf(tool).definition(name) export const settle = (tool: AnyTool, call: ToolCall, context: Context) => runtimeOf(tool).settle(call, context) diff --git a/packages/core/test/session-runner-resume.test.ts b/packages/core/test/session-runner-resume.test.ts index 1f085bb25a8d..ac98196278f6 100644 --- a/packages/core/test/session-runner-resume.test.ts +++ b/packages/core/test/session-runner-resume.test.ts @@ -26,6 +26,8 @@ import * as SessionRunnerLLM from "@opencode-ai/core/session/runner/llm" import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model" import { createLLMEventPublisher } from "@opencode-ai/core/session/runner/publish-llm-event" import { ToolRegistry } from "@opencode-ai/core/tool/registry" +import { ApplicationTools } from "@opencode-ai/core/tool/application-tools" +import { Tool } from "@opencode-ai/core/tool/tool" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { SessionTable } from "@opencode-ai/core/session/sql" import { SessionStore } from "@opencode-ai/core/session/store" @@ -40,7 +42,7 @@ import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" import { Auth } from "@opencode-ai/llm/route" import { describe, expect } from "bun:test" -import { Effect, Layer, Stream } from "effect" +import { Effect, Layer, Schema, Stream } from "effect" import { testEffect } from "./lib/effect" const model = OpenAIChat.route @@ -85,6 +87,7 @@ const harness = (stream: LLMClientShape["stream"]) => Config.node, Snapshot.node, SessionRunnerLLM.node, + ApplicationTools.node, ]), [ [LayerNodePlatform.llmClient, mockClient(stream)], @@ -193,4 +196,44 @@ describe("SessionRunner resume", () => { expect(assistants[0]?.type === "assistant" ? Boolean(assistants[0].time.completed) : false).toBe(true) }), ) + + harness(dying).effect("re-settles an idempotent tool on resume but fails a side-effecting one", () => + Effect.gen(function* () { + yield* seedSession + // A pure-read tool is safe to re-run; a side-effecting one is not. + yield* (yield* ApplicationTools.Service).register({ + probe_read: Tool.make({ + description: "read probe", + idempotent: true, + input: Schema.Struct({}), + output: Schema.String, + toModelOutput: ({ output }) => [{ type: "text", text: output }], + execute: () => Effect.succeed("resettled"), + }), + probe_write: Tool.make({ + description: "write probe", + input: Schema.Struct({}), + output: Schema.String, + execute: () => Effect.succeed("wrote"), + }), + }) + const publisher = yield* seededPublisher + yield* publisher.publish(LLMEvent.toolCall({ id: "call_ro", name: "probe_read", input: {} })) + yield* publisher.publish(LLMEvent.toolCall({ id: "call_rw", name: "probe_write", input: {} })) + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + // Dying stream: resume must not re-call the model; the idempotent tool is re-run via the registry. + yield* runner.runStep({ sessionID, step: 4, promotion: "steer", first: false, force: false }) + const context = yield* store.context(sessionID) + const readPart = toolPart(context, "call_ro") + const writePart = toolPart(context, "call_rw") + expect(readPart?.type === "tool" ? readPart.state.status : undefined).toBe("completed") + expect( + readPart?.type === "tool" && readPart.state.status === "completed" + ? readPart.state.content.some((item) => item.type === "text" && item.text === "resettled") + : false, + ).toBe(true) + expect(writePart?.type === "tool" ? writePart.state.status : undefined).toBe("error") + }), + ) }) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 3a37c5d8249d..db23a9c2ca30 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -122,12 +122,14 @@ any tool left dangling by an interrupted attempt on every entry, not just the fi mid-turn retry (`first=false`) re-streamed a request carrying a `tool_use` with no `tool_result`, which the provider rejects, a retry poison loop. And if the crashed step had already dispatched tools it is finalized from the log: completed tool results are kept, still-unsettled tools are failed, and -a synthesized `Step.Ended` closes the step without re-calling the model. A tool in flight at the -instant of the crash is marked interrupted (its result never committed, so the harness cannot know -whether it ran) and the model redoes it; true safety for that window needs per-tool idempotency -metadata. Finer granularity (the model call and each tool as separate Temporal activities) would -un-fuse the eager tool dispatch and is left for later. Verified by -`packages/core/test/session-runner-resume.test.ts`. +a synthesized `Step.Ended` closes the step without re-calling the model. A tool caught in flight at +the crash is handled by declared idempotency: a side-effect-free tool (`read`/`glob`/`grep`, marked +`idempotent: true`) is re-run for a real result, while a side-effecting tool is marked interrupted +and left for the model to redo. The harness cannot know whether the side-effecting one already ran +and must not re-run `git push`, so the default is non-idempotent; the blanket case (idempotency keys +against an external system) is per-integration and out of scope. Finer granularity (the model call +and each tool as separate Temporal activities) would un-fuse the eager tool dispatch and is left for +later. Verified by `packages/core/test/session-runner-resume.test.ts`. ### Notes From b77ff56b0abbd1332384d823b9b316230a5b2488 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 02:21:39 -0700 Subject: [PATCH 019/103] Bounded the turn's step loop and the repeated-identical-call stuck loop. HTTP-level provider retries were already bounded in the RequestExecutor, but the runner's step loop had no ceiling unless the agent configured one, and a model repeating the exact same tool calls step after step looped forever. A default step ceiling (200) and a stuck-loop detector (3 consecutive steps whose entire tool-call signature set is identical) both route into the existing last-step machinery: tools disabled, one final text-only wrap-up. Detection reads only durable history, so it holds across re-drives. A step that mixes in different work is iterating, not stuck, and never counts. --- packages/core/src/session/runner/llm.ts | 15 +- .../core/src/session/runner/loop-guard.ts | Bin 0 -> 2622 bytes .../test/session-runner-loop-guard.test.ts | 201 ++++++++++++++++++ 3 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 packages/core/src/session/runner/loop-guard.ts create mode 100644 packages/core/test/session-runner-loop-guard.test.ts diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index d8885ea4b7db..70beff768594 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -37,6 +37,7 @@ import { SessionRunnerModel } from "./model" import { createLLMEventPublisher, emitToolResult } from "./publish-llm-event" import { toLLMMessages } from "./to-llm-message" import { MAX_STEPS_PROMPT } from "./max-steps" +import { DEFAULT_MAX_STEPS, REPEAT_LIMIT, REPEATED_CALLS_PROMPT, trailingIdenticalToolSteps } from "./loop-guard" import { Snapshot } from "../../snapshot" import { makeLocationNode } from "../../effect/app-node" import { llmClient } from "../../effect/app-node-platform" @@ -53,7 +54,7 @@ import { llmClient } from "../../effect/app-node-platform" * - [ ] Mark busy, retrying, idle, interrupted, or terminal-failure status durably. * - [ ] Honor interruption and reject stale work after runtime attachment replacement. * - [x] Honor optional agent step limits. - * - [ ] Bound provider retries and repeated identical tool calls. + * - [x] Bound provider retries and repeated identical tool calls. * * - Runtime context assembly * - Track V1 runtime-context parity canonically in `specs/v2/session.md`. @@ -200,7 +201,12 @@ const layer = Layer.effect( const model = yield* models.resolve(session) const entries = yield* SessionHistory.entriesForRunner(db, session.id, system.baselineSeq) const context = entries.map((entry) => entry.message) - const isLastStep = agent.info?.steps !== undefined && currentStep >= agent.info.steps + // Two loop bounds, both ending in one final text-only step: a ceiling on provider attempts + // (the agent's configured limit, else a default so no run is unbounded), and the stuck loop + // where the model repeats the exact same tool calls step after step. Detection reads only the + // durable history, so it holds across re-drives too. + const stuck = trailingIdenticalToolSteps(context) >= REPEAT_LIMIT + const isLastStep = stuck || currentStep >= (agent.info?.steps ?? DEFAULT_MAX_STEPS) const toolMaterialization = isLastStep ? undefined : yield* tools.materialize(agent.info?.permissions) const promptCacheKey = /^ses_[0-9a-f]{64}$/.test(session.id) ? session.id.slice(4) : session.id const request = LLM.request({ @@ -209,7 +215,10 @@ const layer = Layer.effect( system: [agent.info?.system, system.baseline] .filter((part): part is string => part !== undefined && part.length > 0) .map(SystemPart.make), - messages: [...toLLMMessages(context, model), ...(isLastStep ? [Message.assistant(MAX_STEPS_PROMPT)] : [])], + messages: [ + ...toLLMMessages(context, model), + ...(isLastStep ? [Message.assistant(stuck ? REPEATED_CALLS_PROMPT : MAX_STEPS_PROMPT)] : []), + ], tools: toolMaterialization?.definitions ?? [], toolChoice: isLastStep ? "none" : undefined, }) diff --git a/packages/core/src/session/runner/loop-guard.ts b/packages/core/src/session/runner/loop-guard.ts new file mode 100644 index 0000000000000000000000000000000000000000..3806f2e18e94fa11798f2f3518c03b845f1a2f23 GIT binary patch literal 2622 zcma)7T~Fgi6m4JqD{j$BO<3a4_KhuDrAw-%3Iruywbg0`gFQ~B*fZ|T7>FqUz30x@ zA)!*$2NX2EpXZ))?|4jEYxSs-_p+kOn1*gms|%OX$1J<*-WzM~RUs`|>8+BEgq6`w z7Nj)!wWQhXf)?_xRysd@lzHooea9c1OvdAA9PAdKE-$BxZyh-=n~?E@1m#jyO0URh z+9ng;%ewK78qVBmDvc$vk{VAB%CAWq%8f47s+ zt2Kfs?M>&eTnJVP=SQKsTI6em5;jNVqESq|lIY6l09L8AsL0DlKQg-7#I}@%zbLp9 z9ut=cJTNw!CW)#WV?FszBk75j(m6DJfgiv{&s17d(?Ob!>-cn#B;&(F`tW~1e^|>n z;DCGwI5{pC_`e{Y(&0FfkD(4s#d$iL{xbPA%dRgbzh5u2>D7`>>HFj3E;me=CF))h zsHMFEvH|50LmUD-fG9jtw%U^b>+sqS?ceDZ)2rzuyPln2oM*`NL$Y5@{5(Co{xF%% zme*H{`Nb96-F#S_XXkjN5%o57em1>~kJ)@aqu`Px87RS%ME8Il<#P9qYEg7l!K|1z zTf%f~31kgi+BHzRX+3nd*ri;Zh?Rh%9uzRp#thz1L;}hs7!tbJ!2daiE60}C=zrD`VXbJwP~vY)ldP+gm#nU%+TdL!!+(VFoJH= z@An^sMf1zqAL$LaIHoy5@FcO5A`|`QBy>`?OhgH=N>boEN9@?CR$6$h9Y+Jr>)~y; zD+1nF2TTzB2py{*dz@L>zmV4I$&%MhMWZp zvy})(JIT_fb?d>+o2TH#h(CYz=IPhv{4$MIsB#k{qG&JEIELq&A>usaj}yL>o>o$? zu&z&$_;|=i4JQ!NDI_*8m`DB7j*G*79SqYyjnadIuMdXFbJ$}E9nwehfQf){+Dqvz|*nYX}iAR6$jf`#LrNB@z z!oWs;RG8}{INIgzhd-k4-gV#J!=w9uH0n9Z!-Q(P8@hG>9Ktc(Bda=tpLVFcoeTIJ zPz}a0-j-1)Xitp25}C2D_!@Y+W#?{8ru7z=*Je8(UX&D~%L)sBPZGo$+&b*02#~&g O3qRITCzbFB1^ffDVTBO@ literal 0 HcmV?d00001 diff --git a/packages/core/test/session-runner-loop-guard.test.ts b/packages/core/test/session-runner-loop-guard.test.ts new file mode 100644 index 000000000000..ed46ebeb1407 --- /dev/null +++ b/packages/core/test/session-runner-loop-guard.test.ts @@ -0,0 +1,201 @@ +// Turn-level loop bounds: a run whose model repeats the exact same tool call step after step must +// terminate on its own -- after REPEAT_LIMIT identical steps the next attempt runs as a last step +// (tools disabled, text-only wrap-up) instead of looping forever. Driven end to end through +// SessionRunner.run with a mock LLM that always answers with the same tool call until tools are +// disabled. Also unit-covers the trailing-signature detection. +import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" +import { LLMEvent } from "@opencode-ai/llm" +import { Database } from "@opencode-ai/core/database/database" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { PermissionV2 } from "@opencode-ai/core/permission" +import { AgentV2 } from "@opencode-ai/core/agent" +import { Config } from "@opencode-ai/core/config" +import { Project } from "@opencode-ai/core/project" +import { ProjectTable } from "@opencode-ai/core/project/sql" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { SessionV2 } from "@opencode-ai/core/session" +import { Snapshot } from "@opencode-ai/core/snapshot" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import { SessionRunner } from "@opencode-ai/core/session/runner" +import * as SessionRunnerLLM from "@opencode-ai/core/session/runner/llm" +import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model" +import { REPEAT_LIMIT, trailingIdenticalToolSteps } from "@opencode-ai/core/session/runner/loop-guard" +import { ToolRegistry } from "@opencode-ai/core/tool/registry" +import { ApplicationTools } from "@opencode-ai/core/tool/application-tools" +import { Tool } from "@opencode-ai/core/tool/tool" +import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionStore } from "@opencode-ai/core/session/store" +import { SessionMessage } from "@opencode-ai/core/session/message" +import { ModelV2 } from "@opencode-ai/core/model" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { Location } from "@opencode-ai/core/location" +import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" +import { SystemContext } from "@opencode-ai/core/system-context" +import { SkillGuidance } from "@opencode-ai/core/skill/guidance" +import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" +import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" +import { Auth } from "@opencode-ai/llm/route" +import { describe, expect, test } from "bun:test" +import { DateTime, Effect, Layer, Schema, Stream } from "effect" +import { testEffect } from "./lib/effect" + +const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) + .model({ id: "gpt-4o-mini" }) +const models = SessionRunnerModel.layerWith(() => Effect.succeed(model)) +const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) +const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) +const permission = Layer.mock(PermissionV2.Service, {}) + +// Always answers with the same tool call until tools are disabled, then a bare text-less final step. +// Records each request's toolChoice so the test can assert the guard fired. +const stuckModel = () => { + // LLM.request normalizes toolChoice into a ToolChoice class; record its `type`. + const requests: Array<{ readonly tools: number; readonly toolChoice: string | undefined }> = [] + let attempt = 0 + const stream: LLMClientShape["stream"] = (request) => { + const toolChoice = (request.toolChoice as { type?: string } | undefined)?.type + requests.push({ tools: request.tools.length, toolChoice }) + attempt++ + if (toolChoice === "none" || request.tools.length === 0) + return Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })]) + return Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: `call_${attempt}`, name: "probe_stuck", input: { target: "same" } }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + ]) + } + return { requests, stream } +} + +const harness = (stream: LLMClientShape["stream"]) => + testEffect( + AppNodeBuilder.build( + LayerNode.group([ + Database.node, + SessionProjector.node, + SessionStore.node, + AgentV2.node, + ToolRegistry.node, + SessionRunnerModel.node, + SystemContextRegistry.node, + SkillGuidance.node, + ReferenceGuidance.node, + Config.node, + Snapshot.node, + SessionRunnerLLM.node, + ApplicationTools.node, + ]), + [ + [ + LayerNodePlatform.llmClient, + Layer.succeed( + LLMClient.Service, + LLMClient.Service.of({ + prepare: () => Effect.die("unused"), + generate: () => Effect.die("unused"), + stream, + }), + ), + ], + [PermissionV2.node, permission], + [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [SessionRunnerModel.node, models], + [SystemContextRegistry.node, systemContext], + [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })], + [SkillGuidance.node, skillGuidance], + [ReferenceGuidance.node, referenceGuidance], + [Config.node, config], + [Snapshot.node, Snapshot.noopLayer], + ], + ), + ) + +const sessionID = SessionV2.ID.make("ses_loop_guard") + +const seedSession = Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ id: sessionID, project_id: Project.ID.global, slug: "t", directory: "/project", title: "t", version: "t" }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) +}) + +describe("SessionRunner loop guard", () => { + const { requests, stream } = stuckModel() + harness(stream).effect("ends a run whose model repeats the same tool call every step", () => + Effect.gen(function* () { + yield* seedSession + yield* (yield* ApplicationTools.Service).register({ + probe_stuck: Tool.make({ + description: "always same result", + input: Schema.Struct({ target: Schema.String }), + output: Schema.String, + toModelOutput: ({ output }) => [{ type: "text", text: output }], + execute: () => Effect.succeed("unchanged"), + }), + }) + const runner = yield* SessionRunner.Service + // Terminates on its own: REPEAT_LIMIT identical tool steps, then one text-only wrap-up step. + yield* runner.run({ sessionID, force: true }) + const context = yield* (yield* SessionStore.Service).context(sessionID) + const assistants = context.filter((message) => message.type === "assistant") + expect(assistants).toHaveLength(REPEAT_LIMIT + 1) + const last = assistants.at(-1) + expect(last?.type === "assistant" ? last.content.filter((part) => part.type === "tool") : undefined).toEqual([]) + // The guard, not the model, ended the run: the final request had tools disabled. + expect(requests.at(-1)?.tools).toBe(0) + expect(requests.at(-1)?.toolChoice).toBe("none") + expect(requests).toHaveLength(REPEAT_LIMIT + 1) + }), + ) + + test("trailing-signature detection counts only consecutive identical non-empty tool steps", () => { + const created = DateTime.makeUnsafe(0) + const assistant = (value: string, tools: Array<{ name: string; input: Record }>) => + SessionMessage.Assistant.make({ + id: SessionMessage.ID.make(`msg_${value}`), + type: "assistant", + agent: "build", + model: { id: ModelV2.ID.make("model"), providerID: ProviderV2.ID.make("provider") }, + content: tools.map((tool) => + SessionMessage.AssistantTool.make({ + type: "tool", + id: `call_${value}_${tool.name}`, + name: tool.name, + time: { created }, + state: SessionMessage.ToolStateRunning.make({ + status: "running", + input: tool.input, + structured: {}, + content: [], + }), + }), + ), + time: { created, completed: created }, + }) + const same = { name: "read", input: { path: "a.txt" } } + const other = { name: "edit", input: { path: "a.txt", change: 1 } } + + expect(trailingIdenticalToolSteps([assistant("1", [same]), assistant("2", [same]), assistant("3", [same])])).toBe(3) + // A step that also does different work breaks the run (iterating, not stuck). + expect( + trailingIdenticalToolSteps([assistant("1", [same]), assistant("2", [same, other]), assistant("3", [same])]), + ).toBe(1) + // A text-only step breaks the run. + expect(trailingIdenticalToolSteps([assistant("1", [same]), assistant("2", []), assistant("3", [same])])).toBe(1) + }) +}) From 9916d562edbf86575f939f94993e142636af3e52 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 02:31:26 -0700 Subject: [PATCH 020/103] Made permission asks durable and replyable from any process. A pending approval was an in-memory deferred: invisible outside the asking process, so an ask raised inside a standalone worker could never be answered, and it vanished on restart. A pending ask is now also a `permission_request` row in the shared store; the blocked `assert` races its local deferred against a poll of the row, replies operate on the row (with the decline cascade and always-rule retro-approval preserved), and reads list the rows. In addition, a user decline inside a Temporal activity is now non-retryable, so the workflow does not re-drive a turn the user stopped. The `question` tool still needs the same treatment. --- packages/core/schema.json | 302 +++++++++++++++--- packages/core/src/database/migration.gen.ts | 1 + .../20260810092511_permission_request.ts | 25 ++ packages/core/src/database/schema.gen.ts | 15 + packages/core/src/permission.ts | 178 ++++++++--- packages/core/src/permission/sql.ts | 20 +- .../core/src/session/execution/temporal.ts | 24 +- packages/core/test/permission-durable.test.ts | 149 +++++++++ packages/temporal/README.md | 16 + 9 files changed, 638 insertions(+), 92 deletions(-) create mode 100644 packages/core/src/database/migration/20260810092511_permission_request.ts create mode 100644 packages/core/test/permission-durable.test.ts diff --git a/packages/core/schema.json b/packages/core/schema.json index d0eeeebd5c41..f9825126f5ec 100644 --- a/packages/core/schema.json +++ b/packages/core/schema.json @@ -1,8 +1,10 @@ { "version": "7", "dialect": "sqlite", - "id": "f14a9b18-8207-487e-a3d3-227e629ba9ad", - "prevIds": ["169a0f0f-d58f-479f-b024-fa1c7b9a09db"], + "id": "ef05c3dc-ecd1-4def-9573-6b872e58366e", + "prevIds": [ + "f14a9b18-8207-487e-a3d3-227e629ba9ad" + ], "ddl": [ { "name": "workspace", @@ -36,6 +38,10 @@ "name": "event", "entityType": "tables" }, + { + "name": "permission_request", + "entityType": "tables" + }, { "name": "permission", "entityType": "tables" @@ -540,6 +546,86 @@ "entityType": "columns", "table": "event" }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "permission_request" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "permission_request" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "agent", + "entityType": "columns", + "table": "permission_request" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "payload", + "entityType": "columns", + "table": "permission_request" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'pending'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "permission_request" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "message", + "entityType": "columns", + "table": "permission_request" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "permission_request" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "permission_request" + }, { "type": "text", "notNull": false, @@ -1481,9 +1567,13 @@ "table": "session_share" }, { - "columns": ["project_id"], + "columns": [ + "project_id" + ], "tableTo": "project", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1492,9 +1582,13 @@ "table": "workspace" }, { - "columns": ["active_account_id"], + "columns": [ + "active_account_id" + ], "tableTo": "account", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "SET NULL", "nameExplicit": false, @@ -1503,9 +1597,13 @@ "table": "account_state" }, { - "columns": ["aggregate_id"], + "columns": [ + "aggregate_id" + ], "tableTo": "event_sequence", - "columnsTo": ["aggregate_id"], + "columnsTo": [ + "aggregate_id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1514,9 +1612,13 @@ "table": "event" }, { - "columns": ["project_id"], + "columns": [ + "project_id" + ], "tableTo": "project", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1525,9 +1627,13 @@ "table": "permission" }, { - "columns": ["project_id"], + "columns": [ + "project_id" + ], "tableTo": "project", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1536,9 +1642,13 @@ "table": "project_directory" }, { - "columns": ["session_id"], + "columns": [ + "session_id" + ], "tableTo": "session", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1547,9 +1657,13 @@ "table": "message" }, { - "columns": ["message_id"], + "columns": [ + "message_id" + ], "tableTo": "message", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1558,9 +1672,13 @@ "table": "part" }, { - "columns": ["session_id"], + "columns": [ + "session_id" + ], "tableTo": "session", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1569,9 +1687,13 @@ "table": "session_context_epoch" }, { - "columns": ["session_id"], + "columns": [ + "session_id" + ], "tableTo": "session", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1580,9 +1702,13 @@ "table": "session_input" }, { - "columns": ["session_id"], + "columns": [ + "session_id" + ], "tableTo": "session", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1591,9 +1717,13 @@ "table": "session_message" }, { - "columns": ["project_id"], + "columns": [ + "project_id" + ], "tableTo": "project", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1602,9 +1732,13 @@ "table": "session" }, { - "columns": ["session_id"], + "columns": [ + "session_id" + ], "tableTo": "session", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1613,9 +1747,13 @@ "table": "todo" }, { - "columns": ["session_id"], + "columns": [ + "session_id" + ], "tableTo": "session", - "columnsTo": ["id"], + "columnsTo": [ + "id" + ], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "nameExplicit": false, @@ -1624,133 +1762,183 @@ "table": "session_share" }, { - "columns": ["email", "url"], + "columns": [ + "email", + "url" + ], "nameExplicit": false, "name": "control_account_pk", "entityType": "pks", "table": "control_account" }, { - "columns": ["project_id", "directory"], + "columns": [ + "project_id", + "directory" + ], "nameExplicit": false, "name": "project_directory_pk", "entityType": "pks", "table": "project_directory" }, { - "columns": ["session_id", "position"], + "columns": [ + "session_id", + "position" + ], "nameExplicit": false, "name": "todo_pk", "entityType": "pks", "table": "todo" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "workspace_pk", "table": "workspace", "entityType": "pks" }, { - "columns": ["name"], + "columns": [ + "name" + ], "nameExplicit": false, "name": "data_migration_pk", "table": "data_migration", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "account_state_pk", "table": "account_state", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "account_pk", "table": "account", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "credential_pk", "table": "credential", "entityType": "pks" }, { - "columns": ["aggregate_id"], + "columns": [ + "aggregate_id" + ], "nameExplicit": false, "name": "event_sequence_pk", "table": "event_sequence", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "event_pk", "table": "event", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "permission_request_pk", + "table": "permission_request", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], "nameExplicit": false, "name": "permission_pk", "table": "permission", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "project_pk", "table": "project", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "message_pk", "table": "message", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "part_pk", "table": "part", "entityType": "pks" }, { - "columns": ["session_id"], + "columns": [ + "session_id" + ], "nameExplicit": false, "name": "session_context_epoch_pk", "table": "session_context_epoch", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "session_input_pk", "table": "session_input", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "session_message_pk", "table": "session_message", "entityType": "pks" }, { - "columns": ["id"], + "columns": [ + "id" + ], "nameExplicit": false, "name": "session_pk", "table": "session", "entityType": "pks" }, { - "columns": ["session_id"], + "columns": [ + "session_id" + ], "nameExplicit": false, "name": "session_share_pk", "table": "session_share", @@ -1796,6 +1984,24 @@ "entityType": "indexes", "table": "event" }, + { + "columns": [ + { + "value": "session_id", + "isExpression": false + }, + { + "value": "status", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "permission_request_session_status_idx", + "entityType": "indexes", + "table": "permission_request" + }, { "columns": [ { @@ -2068,4 +2274,4 @@ } ], "renames": [] -} +} \ No newline at end of file diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index e6ea4eaa1477..9ff18f333253 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -40,5 +40,6 @@ export const migrations = ( import("./migration/20260622142730_simplify_session_context_epoch"), import("./migration/20260622170816_reset_v2_session_state"), import("./migration/20260622202450_simplify_session_input"), + import("./migration/20260810092511_permission_request"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20260810092511_permission_request.ts b/packages/core/src/database/migration/20260810092511_permission_request.ts new file mode 100644 index 000000000000..58176e762030 --- /dev/null +++ b/packages/core/src/database/migration/20260810092511_permission_request.ts @@ -0,0 +1,25 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260810092511_permission_request", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`permission_request\` ( + \`id\` text PRIMARY KEY, + \`session_id\` text NOT NULL, + \`agent\` text, + \`payload\` text NOT NULL, + \`status\` text DEFAULT 'pending' NOT NULL, + \`message\` text, + \`time_created\` integer NOT NULL, + \`time_updated\` integer NOT NULL + ); + `) + yield* tx.run( + `CREATE INDEX \`permission_request_session_status_idx\` ON \`permission_request\` (\`session_id\`,\`status\`);`, + ) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/schema.gen.ts b/packages/core/src/database/schema.gen.ts index ed60fde6c55f..a0fbf49536ff 100644 --- a/packages/core/src/database/schema.gen.ts +++ b/packages/core/src/database/schema.gen.ts @@ -86,6 +86,18 @@ export default { CONSTRAINT \`fk_event_aggregate_id_event_sequence_aggregate_id_fk\` FOREIGN KEY (\`aggregate_id\`) REFERENCES \`event_sequence\`(\`aggregate_id\`) ON DELETE CASCADE ); `) + yield* tx.run(` + CREATE TABLE \`permission_request\` ( + \`id\` text PRIMARY KEY, + \`session_id\` text NOT NULL, + \`agent\` text, + \`payload\` text NOT NULL, + \`status\` text DEFAULT 'pending' NOT NULL, + \`message\` text, + \`time_created\` integer NOT NULL, + \`time_updated\` integer NOT NULL + ); + `) yield* tx.run(` CREATE TABLE \`permission\` ( \`id\` text PRIMARY KEY, @@ -238,6 +250,9 @@ export default { `) yield* tx.run(`CREATE UNIQUE INDEX \`event_aggregate_seq_idx\` ON \`event\` (\`aggregate_id\`,\`seq\`);`) yield* tx.run(`CREATE INDEX \`event_aggregate_type_seq_idx\` ON \`event\` (\`aggregate_id\`,\`type\`,\`seq\`);`) + yield* tx.run( + `CREATE INDEX \`permission_request_session_status_idx\` ON \`permission_request\` (\`session_id\`,\`status\`);`, + ) yield* tx.run( `CREATE UNIQUE INDEX \`permission_project_action_resource_idx\` ON \`permission\` (\`project_id\`,\`action\`,\`resource\`);`, ) diff --git a/packages/core/src/permission.ts b/packages/core/src/permission.ts index 3f28632a034d..fd7c32a8d2f3 100644 --- a/packages/core/src/permission.ts +++ b/packages/core/src/permission.ts @@ -1,8 +1,10 @@ export * as PermissionV2 from "./permission" +import { and, eq } from "drizzle-orm" import { makeLocationNode } from "./effect/app-node" import { Context, Deferred, Effect as EffectRuntime, Layer, Schema } from "effect" import { Permission } from "@opencode-ai/schema/permission" +import { Database } from "./database/database" import { EventV2 } from "./event" import { Location } from "./location" import { AgentV2 } from "./agent" @@ -10,6 +12,7 @@ import { SessionV2 } from "./session" import { SessionStore } from "./session/store" import { Wildcard } from "./util/wildcard" import { PermissionSaved } from "./permission/saved" +import { PermissionRequestTable } from "./permission/sql" export { Effect, Rule, Ruleset } from "@opencode-ai/schema/permission" const missingAgentPermissions: Permission.Ruleset = [{ action: "*", resource: "*", effect: "deny" }] @@ -114,12 +117,56 @@ const layer = Layer.effect( const agents = yield* AgentV2.Service const sessions = yield* SessionStore.Service const saved = yield* PermissionSaved.Service + const db = (yield* Database.Service).db const pending = new Map() + // Pending asks are durable rows in the shared store, so an ask raised in one process (a + // standalone worker's activity) is visible and replyable from another (the HTTP server), and a + // reply lands even after the asking process restarted. The in-memory deferred stays as the + // same-process fast path; a cross-process reply is observed by polling the row. + const readRow = (id: string) => + db + .select() + .from(PermissionRequestTable) + .where(eq(PermissionRequestTable.id, id)) + .all() + .pipe( + EffectRuntime.orDie, + EffectRuntime.map((rows) => rows[0]), + ) + const pendingRows = (sessionID?: SessionV2.ID) => + db + .select() + .from(PermissionRequestTable) + .where( + sessionID + ? and(eq(PermissionRequestTable.session_id, sessionID), eq(PermissionRequestTable.status, "pending")) + : eq(PermissionRequestTable.status, "pending"), + ) + .all() + .pipe(EffectRuntime.orDie) + const updateRow = (id: string, status: string, message?: string) => + db + .update(PermissionRequestTable) + .set({ status, message: message ?? null }) + .where(eq(PermissionRequestTable.id, id)) + .run() + .pipe(EffectRuntime.orDie) + const decodeRow = (row: { payload: string }) => + Schema.decodeUnknownSync(Request)(JSON.parse(row.payload)) as Request + yield* EffectRuntime.addFinalizer(() => - EffectRuntime.forEach(pending.values(), (item) => Deferred.fail(item.deferred, new DeclinedError()), { - discard: true, - }).pipe( + EffectRuntime.forEach( + pending.values(), + (item) => + // Graceful shutdown: unblock the local waiter and retire the row. The waiting fiber dies + // with this process either way, so a later reply would have nothing to resume. + Deferred.fail(item.deferred, new DeclinedError()).pipe( + EffectRuntime.andThen(updateRow(item.request.id, "expired")), + EffectRuntime.catch(() => EffectRuntime.void), + ), + { discard: true }, + ).pipe( EffectRuntime.ensuring( EffectRuntime.sync(() => { pending.clear() @@ -180,6 +227,19 @@ const layer = Layer.effect( const item = { request, agent, deferred } if (pending.has(request.id)) return yield* EffectRuntime.die(`Duplicate pending permission ID: ${request.id}`) pending.set(request.id, item) + yield* db + .insert(PermissionRequestTable) + .values({ + id: request.id, + session_id: request.sessionID, + agent: agent ?? null, + payload: JSON.stringify(Schema.encodeSync(Request)(request)), + }) + .run() + .pipe( + EffectRuntime.orDie, + EffectRuntime.onError(() => EffectRuntime.sync(() => pending.delete(request.id))), + ) yield* events .publish(Event.Asked, request) .pipe(EffectRuntime.onError(() => EffectRuntime.sync(() => pending.delete(request.id)))) @@ -187,6 +247,20 @@ const layer = Layer.effect( }), ) + // Observe a cross-process reply: the replying process updates the row, not our deferred. + const awaitRow = (id: ID): EffectRuntime.Effect => + EffectRuntime.gen(function* () { + for (;;) { + const row = yield* readRow(id) + if (row && row.status !== "pending") { + if (row.status === "approved") return + if (row.status === "corrected") return yield* new CorrectedError({ feedback: row.message ?? "" }) + return yield* new DeclinedError() + } + yield* EffectRuntime.sleep(500) + } + }) + const ask = EffectRuntime.fn("PermissionV2.ask")(function* (input: AssertInput) { const result = yield* evaluateInput(input) const value = request(input) @@ -205,7 +279,9 @@ const layer = Layer.effect( } if (result.effect === "allow") return const item = yield* create(request(input), input.agent) - return yield* restore(Deferred.await(item.deferred)).pipe( + return yield* restore( + EffectRuntime.raceFirst(Deferred.await(item.deferred), awaitRow(item.request.id)), + ).pipe( EffectRuntime.catchTag("PermissionV2.DeclinedError", (error) => EffectRuntime.die(error)), EffectRuntime.ensuring( EffectRuntime.sync(() => { @@ -217,84 +293,104 @@ const layer = Layer.effect( ), ) + // Complete the local waiter if the ask was raised in this process; a cross-process waiter + // observes the row update through its poll. + const settleLocal = ( + id: ID, + complete: (deferred: Pending["deferred"]) => EffectRuntime.Effect, + ) => + EffectRuntime.suspend(() => { + const item = pending.get(id) + if (!item) return EffectRuntime.void + pending.delete(id) + return EffectRuntime.asVoid(complete(item.deferred)) + }) + + // The durable row is the source of truth, so a reply works from any process (the HTTP server + // replying to an ask raised inside a standalone worker's activity), not just the asking one. const reply = EffectRuntime.fn("PermissionV2.reply")((input: ReplyInput) => EffectRuntime.uninterruptible( EffectRuntime.gen(function* () { - const existing = pending.get(input.requestID) - if (!existing) return yield* new NotFoundError({ requestID: input.requestID }) + const row = yield* readRow(input.requestID) + if (!row || row.status !== "pending") return yield* new NotFoundError({ requestID: input.requestID }) + const existing = decodeRow(row) yield* events.publish(Event.Replied, { - sessionID: existing.request.sessionID, - requestID: existing.request.id, + sessionID: existing.sessionID, + requestID: existing.id, reply: input.reply, }) if (input.reply === "reject") { - yield* Deferred.fail( - existing.deferred, - input.message ? new CorrectedError({ feedback: input.message }) : new DeclinedError(), + yield* updateRow(existing.id, input.message ? "corrected" : "declined", input.message) + yield* settleLocal(existing.id, (deferred) => + Deferred.fail( + deferred, + input.message ? new CorrectedError({ feedback: input.message }) : new DeclinedError(), + ), ) - pending.delete(input.requestID) - for (const [id, item] of pending) { - if (item.request.sessionID !== existing.request.sessionID) continue + // A decline cascades to every other pending ask in the session, wherever it was raised. + for (const other of yield* pendingRows(existing.sessionID)) { + if (other.id === existing.id) continue yield* events.publish(Event.Replied, { - sessionID: item.request.sessionID, - requestID: item.request.id, + sessionID: existing.sessionID, + requestID: ID.make(other.id), reply: "reject", }) - yield* Deferred.fail(item.deferred, new DeclinedError()) - pending.delete(id) + yield* updateRow(other.id, "declined") + yield* settleLocal(ID.make(other.id), (deferred) => Deferred.fail(deferred, new DeclinedError())) } return } - if (input.reply === "always" && existing.request.save?.length) { + if (input.reply === "always" && existing.save?.length) { yield* saved.add({ projectID: location.project.id, - action: existing.request.action, - resources: existing.request.save, + action: existing.action, + resources: existing.save, }) } - yield* Deferred.succeed(existing.deferred, undefined) - pending.delete(input.requestID) - if (input.reply !== "always" || !existing.request.save?.length) return + yield* updateRow(existing.id, "approved") + yield* settleLocal(existing.id, (deferred) => Deferred.succeed(deferred, undefined)) + if (input.reply !== "always" || !existing.save?.length) return + // An always-rule can retro-approve other pending asks it now covers. const rememberedRules = yield* savedRules() - for (const [id, item] of pending) { - const input = { ...item.request } - const rules = yield* configured(item.request.sessionID, item.agent).pipe( + for (const otherRow of yield* pendingRows()) { + if (otherRow.id === existing.id) continue + const other = decodeRow(otherRow) + const agent = otherRow.agent === null ? undefined : AgentV2.ID.make(otherRow.agent) + const rules = yield* configured(other.sessionID, agent).pipe( EffectRuntime.catchTag("Session.NotFoundError", () => EffectRuntime.succeed(undefined)), ) if (!rules) continue - if (denied(input, rules)) continue + if (denied(other, rules)) continue const effective = [...rules, ...rememberedRules] - if ( - !item.request.resources.every( - (resource) => evaluate(item.request.action, resource, effective).effect === "allow", - ) - ) + if (!other.resources.every((resource) => evaluate(other.action, resource, effective).effect === "allow")) continue yield* events.publish(Event.Replied, { - sessionID: item.request.sessionID, - requestID: item.request.id, + sessionID: other.sessionID, + requestID: other.id, reply: "always", }) - yield* Deferred.succeed(item.deferred, undefined) - pending.delete(id) + yield* updateRow(other.id, "approved") + yield* settleLocal(other.id, (deferred) => Deferred.succeed(deferred, undefined)) } }), ), ) + // Reads come from the durable rows, so serve can list and inspect asks raised by any worker. const list = EffectRuntime.fn("PermissionV2.list")(function* () { - return Array.from(pending.values(), (item) => item.request) + return (yield* pendingRows()).map(decodeRow) }) const get = EffectRuntime.fn("PermissionV2.get")(function* (id: ID) { - return pending.get(id)?.request + const row = yield* readRow(id) + return row && row.status === "pending" ? decodeRow(row) : undefined }) const forSession = EffectRuntime.fn("PermissionV2.forSession")(function* (sessionID: SessionV2.ID) { - return Array.from(pending.values(), (item) => item.request).filter((request) => request.sessionID === sessionID) + return (yield* pendingRows(sessionID)).map(decodeRow) }) return Service.of({ ask, assert, reply, get, forSession, list }) @@ -306,5 +402,5 @@ export const locationLayer = layer.pipe(Layer.provideMerge(AgentV2.locationLayer export const node = makeLocationNode({ service: Service, layer, - deps: [EventV2.node, Location.node, AgentV2.node, SessionStore.node, PermissionSaved.node], + deps: [EventV2.node, Location.node, AgentV2.node, SessionStore.node, PermissionSaved.node, Database.node], }) diff --git a/packages/core/src/permission/sql.ts b/packages/core/src/permission/sql.ts index c395555d7950..000257d9351b 100644 --- a/packages/core/src/permission/sql.ts +++ b/packages/core/src/permission/sql.ts @@ -1,4 +1,4 @@ -import { sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { index, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" import { Timestamps } from "../database/schema.sql" import { ProjectV2 } from "../project" import { ProjectTable } from "../project/sql" @@ -18,3 +18,21 @@ export const PermissionTable = sqliteTable( }, (table) => [uniqueIndex("permission_project_action_resource_idx").on(table.project_id, table.action, table.resource)], ) + +// A durable pending-approval record, so an ask raised by one process (a standalone worker) can be +// listed and replied to from another (the HTTP server) via the shared store, and survives the asking +// process. `payload` is the JSON-encoded PermissionV2.Request; status transitions +// pending -> approved | declined | corrected | expired. +export const PermissionRequestTable = sqliteTable( + "permission_request", + { + id: text().primaryKey(), + session_id: text().notNull(), + agent: text(), + payload: text().notNull(), + status: text().notNull().default("pending"), + message: text(), + ...Timestamps, + }, + (table) => [index("permission_request_session_status_idx").on(table.session_id, table.status)], +) diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index c46fd6c6ed7f..8e90547677e8 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -94,7 +94,18 @@ const layer = Layer.effect( ) if (Exit.isSuccess(exit)) return const cause = exit.cause - if (Cause.hasInterruptsOnly(cause)) throw new Error("session run interrupted") + if (Cause.hasInterruptsOnly(cause)) { + // Two interrupt sources: Temporal cancellation (the AbortSignal fired -- rethrow plainly so + // the activity settles as cancelled) and an internal halt like a user declining a permission + // (the signal did NOT fire). The latter must be non-retryable, or the workflow re-drives a + // turn the user explicitly stopped. + if (signal.aborted) throw new Error("session run interrupted") + throw ApplicationFailure.create({ + message: "session run halted (user declined)", + type: "SessionRunDeclined", + nonRetryable: true, + }) + } // A genuine run error is thrown non-retryable so Temporal surfaces it (to resume) rather than // retrying; only crashes / task timeouts (never thrown here) go through the retry policy. The // error is encoded faithfully in `details` so the caller can reconstruct the exact RunError. @@ -130,7 +141,16 @@ const layer = Layer.effect( ) if (Exit.isSuccess(exit)) return exit.value const cause = exit.cause - if (Cause.hasInterruptsOnly(cause)) throw new Error("session run interrupted") + if (Cause.hasInterruptsOnly(cause)) { + // Same split as the whole-turn drain: cancellation rethrows plainly, an internal user-decline + // halt is non-retryable. + if (signal.aborted) throw new Error("session run interrupted") + throw ApplicationFailure.create({ + message: "session run halted (user declined)", + type: "SessionRunDeclined", + nonRetryable: true, + }) + } const squashed = Cause.squash(cause) as { _tag?: string; message?: string } const encoded = encodeRunError(squashed) throw ApplicationFailure.create({ diff --git a/packages/core/test/permission-durable.test.ts b/packages/core/test/permission-durable.test.ts new file mode 100644 index 000000000000..c9e6dca91743 --- /dev/null +++ b/packages/core/test/permission-durable.test.ts @@ -0,0 +1,149 @@ +// Durable permission asks: a pending approval is a row in the shared store, so an ask raised by one +// process (a standalone worker's activity) can be listed and replied to from another (the HTTP +// server), and the blocked assert observes the cross-process reply by polling the row. Two fully +// independent service stacks share one DB file to simulate the two processes. +import { describe, expect } from "bun:test" +import path from "path" +import { Cause, Context, Effect, Exit, Fiber, Layer } from "effect" +import { AgentV2 } from "@opencode-ai/core/agent" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { Database } from "@opencode-ai/core/database/database" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { Location } from "@opencode-ai/core/location" +import { PermissionV2 } from "@opencode-ai/core/permission" +import { Project } from "@opencode-ai/core/project" +import { ProjectTable } from "@opencode-ai/core/project/sql" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionV2 } from "@opencode-ai/core/session" +import { testEffect } from "./lib/effect" +import { tmpdir } from "./fixture/tmpdir" + +const sessionID = SessionV2.ID.make("ses_permission_durable") +const agent = AgentV2.ID.make("build") + +// Every action evaluates to "ask", so assert always parks on an approval. +const askAll = Layer.mock(AgentV2.Service, { + resolve: () => + Effect.succeed({ permissions: [{ action: "*", resource: "*", effect: "ask" }] } as unknown as AgentV2.Info), +}) + +const stack = (file: string) => + AppNodeBuilder.build(PermissionV2.node, [ + [Database.node, Database.layerFromPath(file)], + [AgentV2.node, askAll], + [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })], + ]) + +const seed = (file: string) => + Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "t", + directory: "/project", + title: "t", + version: "t", + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + }).pipe(Effect.provide(Database.layerFromPath(file)), Effect.scoped) + +const it = testEffect(Layer.empty) + +const awaitAsk = (perm: PermissionV2.Interface) => + Effect.gen(function* () { + for (;;) { + const asks = yield* perm.list() + const ask = asks[0] + if (ask) return ask + yield* Effect.sleep(50) + } + }) + +describe("PermissionV2 durable asks", () => { + it.live("unblocks an assert via a reply from a second process sharing the store", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + yield* seed(file) + const A = yield* Layer.build(stack(file)) + const B = yield* Layer.build(stack(file)) + const permA = Context.get(A, PermissionV2.Service) + const permB = Context.get(B, PermissionV2.Service) + + // A: a tool blocks on approval. B: a different process sees the durable ask and approves it. + const blocked = yield* permA + .assert({ sessionID, action: "bash", resources: ["echo hi"], agent }) + .pipe(Effect.forkChild) + const ask = yield* awaitAsk(permB) + expect(ask.action).toBe("bash") + expect(ask.resources).toEqual(["echo hi"]) + yield* permB.reply({ requestID: ask.id, reply: "once" }) + const exit = yield* Fiber.await(blocked) + expect(Exit.isSuccess(exit)).toBe(true) + // The row is settled everywhere: no pending asks remain on either side. + expect(yield* permA.list()).toEqual([]) + expect(yield* permB.list()).toEqual([]) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + it.live("delivers a cross-process correction as the typed CorrectedError", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + yield* seed(file) + const A = yield* Layer.build(stack(file)) + const B = yield* Layer.build(stack(file)) + const permA = Context.get(A, PermissionV2.Service) + const permB = Context.get(B, PermissionV2.Service) + + const blocked = yield* permA + .assert({ sessionID, action: "bash", resources: ["rm -rf /"], agent }) + .pipe(Effect.forkChild) + const ask = yield* awaitAsk(permB) + yield* permB.reply({ requestID: ask.id, reply: "reject", message: "Use a scoped path instead." }) + const exit = yield* Fiber.await(blocked) + expect(Exit.isFailure(exit)).toBe(true) + const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined + expect(error).toBeInstanceOf(PermissionV2.CorrectedError) + expect((error as PermissionV2.CorrectedError).feedback).toBe("Use a scoped path instead.") + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + it.live("expires locally-pending asks on shutdown so they do not linger as pending rows", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + yield* seed(file) + let askID: PermissionV2.ID | undefined + // A raises an ask, then its scope closes (a graceful shutdown) before anyone replies. + yield* Effect.scoped( + Effect.gen(function* () { + const A = yield* Layer.build(stack(file)) + const permA = Context.get(A, PermissionV2.Service) + const result = yield* permA.ask({ sessionID, action: "bash", resources: ["echo bye"], agent }) + askID = result.id + }), + ) + const B = yield* Layer.build(stack(file)) + const permB = Context.get(B, PermissionV2.Service) + expect(askID).toBeDefined() + // The waiter died with A, so the ask is retired, not stuck pending forever. + expect(yield* permB.get(askID!)).toBeUndefined() + expect(yield* permB.list()).toEqual([]) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) +}) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index db23a9c2ca30..30851d22122c 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -168,6 +168,22 @@ without the HTTP API, so a worker resumes a session purely from the shared store on the task queue. Caveat: file-touching tools run against the local working tree, so a worker must have the session's worktree present (see "What resumes cross-host"). +### Durable permission asks + +A tool waiting for user approval used to park on an in-memory deferred: invisible outside the asking +process (a standalone worker's ask could never be answered) and gone on restart. A pending ask is now +also a row in the shared store (`permission_request`). The blocked `assert` races its local deferred +against a poll of the row, so a reply from ANY process sharing the store (the HTTP server answering +for a detached worker) unblocks it; `list`/`get`/`forSession` read the rows, so serve can show asks +raised elsewhere. Replies keep their semantics: `once`/`always` approve (an `always` rule +retro-approves other pending asks it covers), `reject` declines and cascades to the session's other +pending asks, a rejection message arrives as the typed `CorrectedError`. A user decline inside a +Temporal activity is now a non-retryable failure, so the workflow does not re-drive a turn the user +stopped. Graceful shutdown retires the process's pending asks as `expired`; after a hard crash a row +can linger pending until a reply lands on it, which then has nothing to resume and is a no-op. +Verified by `packages/core/test/permission-durable.test.ts` (two independent stacks over one store). +The `question` tool still uses an in-process deferred and needs the same treatment. + ## Shared, durable event store (any-worker resume) The v2 engine event-sources each session to a SQLite store. By default that is a local file, so a From a7fea754466fb082f9513e86f4c0acaf627c7f27 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 02:35:19 -0700 Subject: [PATCH 021/103] Verified networked-libSQL transaction atomicity against a live sqld. The remote atomicity story was integration-test-pending for lack of a server. A new opt-in suite (OPENCODE_LIBSQL_TEST_URL, e.g. `turso dev`) proves over HTTP that a multi-statement transaction commits all-or-nothing, a mid-transaction failure rolls back everything, and the schema migrations apply remotely through the single BEGIN IMMEDIATE path. Skips when no server is set. --- .../core/test/database-libsql-remote.test.ts | 77 +++++++++++++++++++ packages/temporal/README.md | 16 ++-- 2 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 packages/core/test/database-libsql-remote.test.ts diff --git a/packages/core/test/database-libsql-remote.test.ts b/packages/core/test/database-libsql-remote.test.ts new file mode 100644 index 000000000000..982c1f1509fd --- /dev/null +++ b/packages/core/test/database-libsql-remote.test.ts @@ -0,0 +1,77 @@ +// Networked-libSQL integration check: the same commit/rollback atomicity assertions as +// database-libsql-transaction.test.ts, but against a REAL libSQL server over the network (sqld / +// `turso dev`), where each statement is its own HTTP request and only the interactive-transaction +// routing makes BEGIN..COMMIT atomic. Needs a live server, so it runs only when +// OPENCODE_LIBSQL_TEST_URL is set (e.g. `turso dev --port 8888` then +// OPENCODE_LIBSQL_TEST_URL=http://127.0.0.1:8888 bun test database-libsql-remote`); otherwise the +// suite is skipped, not silently passed. +import { describe, expect, test } from "bun:test" +import { Effect, Exit, Layer } from "effect" +import { Database } from "@opencode-ai/core/database/database" +import { testEffect } from "./lib/effect" + +const url = process.env.OPENCODE_LIBSQL_TEST_URL + +const withRemoteDb = (body: (db: Database.Interface["db"]) => Effect.Effect) => + Effect.gen(function* () { + const { db } = yield* Database.Service + return yield* body(db) + }).pipe(Effect.provide(Database.layerFromLibsql(url!, process.env.OPENCODE_LIBSQL_TEST_AUTH_TOKEN))) + +const it = testEffect(Layer.empty) + +describe.skipIf(!url)("Database libSQL backend (remote server)", () => { + it.live("commits a multi-statement transaction atomically over the network", () => + withRemoteDb((db) => + Effect.gen(function* () { + yield* db.run("DROP TABLE IF EXISTS tx_probe_remote") + yield* db.run("CREATE TABLE tx_probe_remote (id TEXT PRIMARY KEY, n INTEGER NOT NULL)") + yield* db.transaction( + () => + Effect.gen(function* () { + yield* db.run("INSERT INTO tx_probe_remote (id, n) VALUES ('a', 1)") + yield* db.run("INSERT INTO tx_probe_remote (id, n) VALUES ('b', 2)") + }), + { behavior: "immediate" }, + ) + const row = yield* db.get<{ c: number }>("SELECT COUNT(*) AS c FROM tx_probe_remote") + expect(Number(row?.c ?? 0)).toBe(2) + }), + ), + ) + + it.live("rolls back every write when a statement fails over the network", () => + withRemoteDb((db) => + Effect.gen(function* () { + yield* db.run("DROP TABLE IF EXISTS tx_probe_remote") + yield* db.run("CREATE TABLE tx_probe_remote (id TEXT PRIMARY KEY, n INTEGER NOT NULL)") + yield* db.run("INSERT INTO tx_probe_remote (id, n) VALUES ('seed', 0)") + const exit = yield* db + .transaction( + () => + Effect.gen(function* () { + yield* db.run("INSERT INTO tx_probe_remote (id, n) VALUES ('a', 1)") + yield* Effect.fail(new Error("boom")) + }), + { behavior: "immediate" }, + ) + .pipe(Effect.exit) + expect(Exit.isFailure(exit)).toBe(true) + const row = yield* db.get<{ c: number }>("SELECT COUNT(*) AS c FROM tx_probe_remote") + expect(Number(row?.c ?? 0)).toBe(1) + }), + ), + ) + + it.live("runs the schema migrations against the remote store", () => + withRemoteDb((db) => + Effect.gen(function* () { + // layerFromLibsql already applied migrations on build; spot-check the core tables exist. + const row = yield* db.get<{ name: string }>( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'event'", + ) + expect(row?.name).toBe("event") + }), + ), + ) +}) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 30851d22122c..62f4e8837ff3 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -213,10 +213,13 @@ distinct worker identities. - Multi-statement writes commit atomically on both backends. The libSQL client runs each statement as its own auto-commit request, so a `BEGIN`/`COMMIT` emitted as plain statements would not bind a transaction over a remote URL. The shared backend instead routes a transaction through a real - interactive libSQL transaction (one pinned stream), which is all-or-nothing. A commit/rollback - atomicity test runs against an embedded (`file:`) store - (`packages/core/test/database-libsql-transaction.test.ts`). The networked crash-atomicity itself - still needs a live `sqld`/Turso to integration-test. + interactive libSQL transaction (one pinned stream), which is all-or-nothing. Verified against an + embedded (`file:`) store (`packages/core/test/database-libsql-transaction.test.ts`) AND against a + live networked `sqld` over HTTP (`packages/core/test/database-libsql-remote.test.ts`: atomic + commit, full rollback on a mid-transaction failure, and the schema migrations applied remotely). + The remote suite needs a server, so it runs only when `OPENCODE_LIBSQL_TEST_URL` is set (e.g. + `turso dev --port 8899`, then `OPENCODE_LIBSQL_TEST_URL=http://127.0.0.1:8899`); it skips + otherwise. ### What resumes cross-host, and what does not @@ -232,8 +235,9 @@ Host-local state that does NOT ride the DB, so it is not reconstructed on a diff a turn that keeps editing files must resume on a worker that has that worktree. This is the one real cross-host correctness constraint. Three ways to satisfy it: co-locate a session's workers by worktree (session affinity via a per-worktree Temporal task queue), share the worktree (a networked - filesystem), or reconstruct it from the last snapshot on resume (needs a shared snapshot store). This - is a deployment/design choice, not covered here yet. + filesystem), or reconstruct it from the last snapshot on resume (needs a shared snapshot store). + [docs/worktree-portability.md](docs/worktree-portability.md) weighs the three; the recommendation is + per-worktree task queues, with snapshot reconstruction as the long-term path. - **The snapshot store (`${data}/snapshot`) and the retained full tool-output files (`${data}/tool-output`).** The runner never reads these to rebuild context: snapshot file-diffs are best-effort (`Effect.catch` to `undefined`), and the model sees the bounded tool-output preview, not From 2f64ae7772917e5f30bae62cf50ba7ac93a303a7 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 02:35:19 -0700 Subject: [PATCH 022/103] Added the worktree-portability design note. Weighs per-worktree task queues, a shared filesystem, and snapshot reconstruction for the one remaining cross-host constraint; recommends affinity now, reconstruction long-term. --- .../temporal/docs/worktree-portability.md | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 packages/temporal/docs/worktree-portability.md diff --git a/packages/temporal/docs/worktree-portability.md b/packages/temporal/docs/worktree-portability.md new file mode 100644 index 000000000000..fac52132686f --- /dev/null +++ b/packages/temporal/docs/worktree-portability.md @@ -0,0 +1,57 @@ +# Worktree portability across workers + +## Problem + +The shared event store makes the **conversation** resumable on any worker: history, tool results, +attachments, and credentials are all rebuilt from the DB. The one thing that is not in the DB is the +**project working tree**. File-touching tools (`bash`, `read`, `edit`, `write`, `apply_patch`, +`glob`, `grep`) operate on `Location.directory`, a local filesystem path. A worker that picks up a +session without that worktree resumes the conversation correctly and then acts on a missing (or +wrong) directory. Mid-session uncommitted changes make this worse: they exist only on the disk of +the worker that made them, so even a fresh clone of the repository is not the session's real state. + +This is a deployment/design decision, not a bug fix. Three ways to satisfy it, in order of +recommendation. + +## A. Session affinity: one task queue per worktree (recommended default) + +Temporal-native and no new infrastructure. Derive the task queue from the worktree identity +(`opencode-session-exec@`); a worker registers on the queues for the worktrees whose +filesystem it actually hosts, and the client starts each session's workflow on the queue derived +from the session's location. Activities for a session then only ever land on a worker that has the +session's files. + +- Scale-out happens across sessions/worktrees; within one worktree the queue is served by workers + sharing one filesystem view of it (typically exactly one worker, or one volume). +- Worker loss stalls only that worktree's sessions until a replacement mounts the same volume (the + Kubernetes PVC-reattach pattern); Temporal re-drives the in-flight step when it comes up. +- Implementation is small: `TASK_QUEUE` in `packages/core/src/session/execution/temporal.ts` is a + fixed constant today; it becomes a function of the session's location on the client side, and the + worker side (`packages/server/src/worker.ts`) takes the list of hosted worktrees and registers one + worker per queue. + +## B. Shared filesystem + +Mount the worktrees on every worker (NFS/EFS/SMB) and keep the single queue. No code change, and +any worker genuinely can resume any session. The costs are operational: git and build tools over +network filesystems are slow and occasionally surprising, and two sessions sharing one worktree can +collide across hosts just as they can within one (a session's own tools stay serialized either way, +one activity at a time). + +## C. Reconstruct the worktree from snapshots (long-term) + +The engine already captures git-tree snapshots around each step (`Step.Started`/`Step.Ended` carry +snapshot ids); today they live in a local per-project git store (`${data}/snapshot`). Point that +store at shared storage and a worker without the worktree can materialize the session's exact file +state on resume: clone the repository, then check out the last recorded snapshot tree. This is the +only option that gives true any-worker resume including uncommitted changes. Its honest limits: +materialization latency on first touch, and snapshots capture the git tree, not the world around it +(ignored files, dependencies, running processes), so a reconstructed worktree may still need a +dependency install before `bash` behaves identically. + +## Recommendation + +Ship **A** as the deployment default (small change, no new infra, correct by construction), allow +**B** where shared volumes already exist, and treat **C** as the future enhancement that removes +the affinity constraint entirely. A and C compose: affinity routes the common case to the warm +worktree; snapshot reconstruction lets a cold worker join the queue after materializing. From e6dadee11d122be53658f78a4e212bfc70504ffb Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 10:31:51 -0700 Subject: [PATCH 023/103] Recovered wake-driven turns that a re-drive silently dropped. Promotion consumes the pending input row inside the turn's own transaction, so a crashed wake-driven turn left nothing in the input tables and a retried activity with force=false returned as a no-op: the turn was abandoned until the next user input. Eligibility now also consults the log: a promoted prompt with no assistant reply, or an in-flight assistant, is recoverable work in both `run` and a first-step `runStep`. A settled history still no-ops, so a retry of a completed drain never re-calls the model. The crash test now requires the post-crash token (and aborts as invalid if the task finished before the kill); its old gate was satisfied by pre-crash step.ended events. --- packages/core/src/session/runner/llm.ts | 31 ++- .../core/test/session-runner-recovery.test.ts | 207 ++++++++++++++++++ packages/temporal/scripts/v2-crash-test.sh | 11 +- 3 files changed, 244 insertions(+), 5 deletions(-) create mode 100644 packages/core/test/session-runner-recovery.test.ts diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 70beff768594..05fa60323754 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -143,6 +143,22 @@ const layer = Layer.effect( const awaitToolFibers = (fibers: FiberSet.FiberSet) => Effect.raceFirst(FiberSet.join(fibers), FiberSet.awaitEmpty(fibers)) + // A crashed drain leaves durable evidence that the input tables no longer show: promotion + // consumed the pending row inside the turn's own transaction, so a re-driven activity that + // checks only hasPending would drop the crashed turn as a no-op. Recoverable work = the latest + // conversational message is a user prompt with no assistant reply, or the latest assistant is + // still in flight. An interrupted turn stays excluded because its cleanup completes the + // assistant via Step.Failed. + const hasRecoverableWork = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) { + const context = yield* getContext(sessionID) + for (let index = context.length - 1; index >= 0; index--) { + const message = context[index] + if (message?.type === "assistant") return !message.time.completed + if (message?.type === "user") return true + } + return false + }) + // Match V1: declining a user prompt halts the loop instead of becoming model-facing tool output. const isUserDeclined = (cause: Cause.Cause) => cause.reasons.some( @@ -396,10 +412,11 @@ const layer = Layer.effect( }) { const hasSteer = yield* SessionInput.hasPending(db, input.sessionID, "steer") const hasQueue = hasSteer ? false : yield* SessionInput.hasPending(db, input.sessionID, "queue") - if (!input.force && !hasSteer && !hasQueue) return + const recover = !input.force && !hasSteer && !hasQueue && (yield* hasRecoverableWork(input.sessionID)) + if (!input.force && !hasSteer && !hasQueue && !recover) return yield* failInterruptedTools(input.sessionID) let promotion: SessionInput.Delivery | undefined = hasSteer ? "steer" : hasQueue ? "queue" : undefined - let shouldRun = input.force || hasSteer || hasQueue + let shouldRun = input.force || hasSteer || hasQueue || recover while (shouldRun) { let needsContinuation = true let step = 1 @@ -521,7 +538,15 @@ const layer = Layer.effect( if (input.first) { const hasSteer = yield* SessionInput.hasPending(db, input.sessionID, "steer") const hasQueue = hasSteer ? false : yield* SessionInput.hasPending(db, input.sessionID, "queue") - if (!input.force && !hasSteer && !hasQueue) + // Same recovery gate as `run`: a first-step retry whose prompt was already promoted (and + // whose crash predates any tool dispatch, so resumeCrashedStep had nothing to finalize) + // must re-stream, not no-op. + if ( + !input.force && + !hasSteer && + !hasQueue && + !(yield* hasRecoverableWork(input.sessionID)) + ) return { ran: false, continue: false, step: input.step, promotion: undefined } promotion = hasSteer ? "steer" : hasQueue ? "queue" : undefined } diff --git a/packages/core/test/session-runner-recovery.test.ts b/packages/core/test/session-runner-recovery.test.ts new file mode 100644 index 000000000000..48d33a74629c --- /dev/null +++ b/packages/core/test/session-runner-recovery.test.ts @@ -0,0 +1,207 @@ +// Re-drive eligibility: a crashed drain leaves no pending input rows (promotion consumed them +// inside the turn), so eligibility must come from the log. A promoted-but-unanswered prompt or an +// in-flight assistant makes a force=false re-drive run the turn; a settled history stays a no-op, +// so retries of a completed drain never spin the model. +import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" +import { LLMEvent } from "@opencode-ai/llm" +import { Database } from "@opencode-ai/core/database/database" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { EventV2 } from "@opencode-ai/core/event" +import { PermissionV2 } from "@opencode-ai/core/permission" +import { AgentV2 } from "@opencode-ai/core/agent" +import { Config } from "@opencode-ai/core/config" +import { Project } from "@opencode-ai/core/project" +import { ProjectTable } from "@opencode-ai/core/project/sql" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { SessionV2 } from "@opencode-ai/core/session" +import { Snapshot } from "@opencode-ai/core/snapshot" +import { SessionEvent } from "@opencode-ai/core/session/event" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import { SessionRunner } from "@opencode-ai/core/session/runner" +import * as SessionRunnerLLM from "@opencode-ai/core/session/runner/llm" +import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model" +import { createLLMEventPublisher } from "@opencode-ai/core/session/runner/publish-llm-event" +import { ToolRegistry } from "@opencode-ai/core/tool/registry" +import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionStore } from "@opencode-ai/core/session/store" +import { SessionMessage } from "@opencode-ai/core/session/message" +import { Prompt } from "@opencode-ai/core/session/prompt" +import { ModelV2 } from "@opencode-ai/core/model" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { Location } from "@opencode-ai/core/location" +import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" +import { SystemContext } from "@opencode-ai/core/system-context" +import { SkillGuidance } from "@opencode-ai/core/skill/guidance" +import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" +import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" +import { Auth } from "@opencode-ai/llm/route" +import { describe, expect } from "bun:test" +import { DateTime, Effect, Layer, Stream } from "effect" +import { testEffect } from "./lib/effect" + +const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) + .model({ id: "gpt-4o-mini" }) +const models = SessionRunnerModel.layerWith(() => Effect.succeed(model)) +const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) +const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) +const permission = Layer.mock(PermissionV2.Service, {}) + +// Counts provider calls and answers each with an empty completed step. +const countingModel = () => { + const requests: number[] = [] + const stream: LLMClientShape["stream"] = () => { + requests.push(1) + return Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })]) + } + return { requests, stream } +} + +const harness = (stream: LLMClientShape["stream"]) => + testEffect( + AppNodeBuilder.build( + LayerNode.group([ + Database.node, + EventV2.node, + SessionProjector.node, + SessionStore.node, + AgentV2.node, + ToolRegistry.node, + SessionRunnerModel.node, + SystemContextRegistry.node, + SkillGuidance.node, + ReferenceGuidance.node, + Config.node, + Snapshot.node, + SessionRunnerLLM.node, + ]), + [ + [ + LayerNodePlatform.llmClient, + Layer.succeed( + LLMClient.Service, + LLMClient.Service.of({ + prepare: () => Effect.die("unused"), + generate: () => Effect.die("unused"), + stream, + }), + ), + ], + [PermissionV2.node, permission], + [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [SessionRunnerModel.node, models], + [SystemContextRegistry.node, systemContext], + [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })], + [SkillGuidance.node, skillGuidance], + [ReferenceGuidance.node, referenceGuidance], + [Config.node, config], + [Snapshot.node, Snapshot.noopLayer], + ], + ), + ) + +const seedSession = (sessionID: SessionV2.ID) => + Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "t", + directory: "/project", + title: "t", + version: "t", + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + }) + +// A prompt whose input row was already consumed: only the projected user message remains, exactly +// what a crash after promotion leaves behind. +const seedPromotedPrompt = (sessionID: SessionV2.ID) => + Effect.gen(function* () { + const events = yield* EventV2.Service + yield* events.publish(SessionEvent.Prompted, { + sessionID, + timestamp: yield* DateTime.now, + messageID: SessionMessage.ID.create(), + prompt: Prompt.make({ text: "do the thing" }), + delivery: "queue", + }) + }) + +describe("SessionRunner crash-recovery eligibility", () => { + { + const { requests, stream } = countingModel() + const sessionID = SessionV2.ID.make("ses_recovery_prompted") + harness(stream).effect("re-drives a promoted-but-unanswered prompt, then settles", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPromotedPrompt(sessionID) + const runner = yield* SessionRunner.Service + yield* runner.run({ sessionID, force: false }) + expect(requests).toHaveLength(1) + const context = yield* (yield* SessionStore.Service).context(sessionID) + const assistant = context.findLast((message) => message.type === "assistant") + expect(assistant?.type === "assistant" ? Boolean(assistant.time.completed) : false).toBe(true) + // Settled history: a retried completed drain stays a no-op instead of re-calling the model. + yield* runner.run({ sessionID, force: false }) + expect(requests).toHaveLength(1) + }), + ) + } + + { + const { requests, stream } = countingModel() + const sessionID = SessionV2.ID.make("ses_recovery_inflight") + harness(stream).effect("re-drives an in-flight assistant and closes its dangling tool", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + const events = yield* EventV2.Service + const publisher = createLLMEventPublisher(events, { + sessionID, + agent: "build", + model: { id: ModelV2.ID.make("gpt-4o-mini"), providerID: ProviderV2.ID.make("openai") }, + }) + yield* publisher.publish(LLMEvent.toolInputStart({ id: "call_dangling", name: "read" })) + const runner = yield* SessionRunner.Service + yield* runner.run({ sessionID, force: false }) + expect(requests).toHaveLength(1) + const context = yield* (yield* SessionStore.Service).context(sessionID) + for (const message of context) { + if (message.type !== "assistant") continue + for (const part of message.content) + if (part.type === "tool" && part.id === "call_dangling") expect(part.state.status).toBe("error") + } + }), + ) + } + + { + const { requests, stream } = countingModel() + const sessionID = SessionV2.ID.make("ses_recovery_step") + harness(stream).effect("runStep(first) recovers the same window instead of no-opping", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPromotedPrompt(sessionID) + const runner = yield* SessionRunner.Service + const result = yield* runner.runStep({ sessionID, step: 1, promotion: undefined, first: true, force: false }) + expect(result.ran).toBe(true) + expect(requests).toHaveLength(1) + }), + ) + } +}) diff --git a/packages/temporal/scripts/v2-crash-test.sh b/packages/temporal/scripts/v2-crash-test.sh index 307ee2a87359..8af57be3ff74 100755 --- a/packages/temporal/scripts/v2-crash-test.sh +++ b/packages/temporal/scripts/v2-crash-test.sh @@ -33,7 +33,11 @@ curl -sS -m10 -o /dev/null -w ' prompt HTTP %{http_code}\n' -X POST $B/sessio echo "[3] let a few steps record, then KILL the whole server mid-turn" sleep 7 -echo " events before crash: $(curl -sS -m8 $B/session/$SID/history -H "$H" | python3 -c 'import sys,json;print(len(json.load(sys.stdin).get("data",[])))')" +PRE=$(curl -sS -m8 $B/session/$SID/history -H "$H" | python3 -c 'import sys,json;d=json.load(sys.stdin).get("data",[]);print(len(d), "seen="+str("CRASH_RECOVERED" in json.dumps(d)))') +echo " events before crash: $PRE" +# The token is written by step 3 of the task; if it already exists the kill landed too late and the +# run proves nothing about recovery. +[[ "$PRE" == *seen=True* ]] && { echo "RESULT: INVALID (task finished before the crash; rerun)"; exit 2; } killserver; echo " server killed" echo "[4] restart server (embedded worker re-registers; Temporal re-drives)" @@ -48,7 +52,9 @@ import sys,json items=json.load(sys.stdin).get("data",[]) types=[e.get("type","") for e in items] print("ENDED" if any(t.endswith("step.ended") for t in types) else "pending", "seen="+str("CRASH_RECOVERED" in json.dumps(items)))' 2>/dev/null) - echo " poll $i: $r"; [[ "$r" == ENDED* ]] && { DONE=yes; break; } + # Pass needs the post-crash work to have actually happened (the token is only written by a step + # that runs after the kill), not just any pre-crash step.ended in the history. + echo " poll $i: $r"; [[ "$r" == "ENDED seen=True" ]] && { DONE=yes; break; } done echo "[6] evidence from Temporal" @@ -60,3 +66,4 @@ attempts=[int(e["activityTaskStartedEventAttributes"].get("attempt",1)) for e in print(" runContinuation attempts:", attempts, "| max:", max(attempts) if attempts else 0) PY echo "RESULT: turn completed post-crash = $DONE" +[[ "$DONE" == yes ]] || exit 1 From a5d759efe57c439e82c4242e03a580ed6b029544 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 10:32:58 -0700 Subject: [PATCH 024/103] Raised the activity backstop to 12 hours; recorded cancellations as such. The wall-clock heartbeat only detects process death, so the 30-minute startToClose was the sole bound on a running drain and hard-killed legitimate long turns (many steps, long tools, a human considering a permission ask), each kill opening a short two-writer window until the zombie attempt noticed. The heartbeat stays the liveness bound; startToClose is now a 12-hour backstop. In addition, a drain interrupted by Temporal cancellation rethrows the cancellation reason, so the attempt records Cancelled instead of Failed. --- .../src/session/execution/temporal-workflow.ts | 11 +++++++---- .../core/src/session/execution/temporal.ts | 18 ++++++++++-------- packages/temporal/README.md | 6 ++++++ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index 7b5d91c98ae5..e868065b0f43 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -21,10 +21,13 @@ import { import type { Activities, StepActivities, StepDrainResult } from "./temporal-activities" const activityOptions = { - startToCloseTimeout: "30 minutes", - // Short heartbeat so a dead worker's in-flight drain is re-driven quickly; the run re-reads the - // durable log, so a retry is a safe re-attach. A genuine run error is thrown non-retryable by the - // activity, so only crashes/timeouts actually retry. + // The heartbeat is the liveness bound (it stops within seconds of a worker death and Temporal + // re-drives). startToClose is only the backstop for a drain that hangs while its process stays + // alive, so it must comfortably exceed any legitimate turn: long tool runs, many steps, or a + // human taking their time over a permission ask. 30 minutes proved far too tight -- it hard-killed + // legitimate turns and each kill opened a short two-writer window until the zombie attempt + // noticed its heartbeat rejection. + startToCloseTimeout: "12 hours", heartbeatTimeout: "10 seconds", retry: { maximumAttempts: 100 }, } as const diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 8e90547677e8..a64b17e6599f 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -95,11 +95,12 @@ const layer = Layer.effect( if (Exit.isSuccess(exit)) return const cause = exit.cause if (Cause.hasInterruptsOnly(cause)) { - // Two interrupt sources: Temporal cancellation (the AbortSignal fired -- rethrow plainly so - // the activity settles as cancelled) and an internal halt like a user declining a permission - // (the signal did NOT fire). The latter must be non-retryable, or the workflow re-drives a - // turn the user explicitly stopped. - if (signal.aborted) throw new Error("session run interrupted") + // Two interrupt sources: Temporal cancellation (the AbortSignal fired -- rethrow its reason + // so the attempt records Cancelled, not Failed) and an internal halt like a user declining a + // permission (the signal did NOT fire). The latter must be non-retryable, or the workflow + // re-drives a turn the user explicitly stopped. + if (signal.aborted) + throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") throw ApplicationFailure.create({ message: "session run halted (user declined)", type: "SessionRunDeclined", @@ -142,9 +143,10 @@ const layer = Layer.effect( if (Exit.isSuccess(exit)) return exit.value const cause = exit.cause if (Cause.hasInterruptsOnly(cause)) { - // Same split as the whole-turn drain: cancellation rethrows plainly, an internal user-decline - // halt is non-retryable. - if (signal.aborted) throw new Error("session run interrupted") + // Same split as the whole-turn drain: cancellation rethrows its reason (records Cancelled), + // an internal user-decline halt is non-retryable. + if (signal.aborted) + throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") throw ApplicationFailure.create({ message: "session run halted (user declined)", type: "SessionRunDeclined", diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 62f4e8837ff3..1cd4c7868bee 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -142,6 +142,12 @@ later. Verified by `packages/core/test/session-runner-resume.test.ts`. - The per-session workflow is long-lived and self-terminates after an idle period. Layer construction connects to Temporal at startup, so the server needs Temporal reachable when `OPENCODE_SESSION_EXECUTION=temporal`. +- Activity bounds: the 10s heartbeat is the liveness bound (worker death re-drives within seconds); + `startToCloseTimeout` is a 12-hour backstop for a drain that hangs while its process stays alive. + Known limit: when an attempt is retried while the previous one is still alive (a network + partition, or the backstop firing), the old attempt keeps publishing for a few seconds until its + heartbeat is rejected and the AbortSignal interrupts it; the projector's status guards make + duplicate settlements no-ops in projection, but the overlap window is not fully fenced. `scripts/resume-check.ts` verifies resume: it resolves on a healthy session and rejects on a failing one with the original tagged error (`LLM.Error`) reconstructed across the boundary. From 6cd2f60c66c19f741c9538969475ba6f5255aaa3 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 10:36:21 -0700 Subject: [PATCH 025/103] Keyed tool asks deterministically so re-drives adopt the same row. A retried activity minted a fresh ask id per attempt, piling up pending rows and re-asking for approvals the user had already given. A tool-originated ask now derives its id from session + callID + action + resources: a re-drive adopts the pending row (one visible ask, one reply), an approval that landed while the asker was dead short-circuits the retry, a decline stays declined, and a graceful-shutdown `expired` row is revived by the next attempt. --- packages/core/src/permission.ts | 38 +++++++++++- packages/core/test/permission-durable.test.ts | 62 +++++++++++++++++++ packages/temporal/README.md | 8 ++- 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/packages/core/src/permission.ts b/packages/core/src/permission.ts index fd7c32a8d2f3..cec6feb9ace4 100644 --- a/packages/core/src/permission.ts +++ b/packages/core/src/permission.ts @@ -1,5 +1,6 @@ export * as PermissionV2 from "./permission" +import { createHash } from "node:crypto" import { and, eq } from "drizzle-orm" import { makeLocationNode } from "./effect/app-node" import { Context, Deferred, Effect as EffectRuntime, Layer, Schema } from "effect" @@ -208,9 +209,21 @@ const layer = Layer.effect( return { effect, rules: all } }) + // A tool-originated ask gets a DETERMINISTIC id (session + callID + action + resources), so a + // re-driven activity resolves to the same durable row instead of filing a duplicate: a reply + // that landed while the asker was dead is honored on the retry, and a still-pending row is + // adopted rather than re-asked. Asks without a tool source keep random ids. + function deterministicID(input: AssertInput) { + if (!input.source?.callID) return undefined + const digest = createHash("sha256") + .update([input.sessionID, input.source.callID, input.action, ...[...input.resources].sort()].join("")) + .digest("hex") + return ID.create(`per_${digest.slice(0, 26)}`) + } + function request(input: AssertInput): Request { return { - id: input.id ?? ID.create(), + id: input.id ?? deterministicID(input) ?? ID.create(), sessionID: input.sessionID, action: input.action, resources: input.resources, @@ -235,11 +248,20 @@ const layer = Layer.effect( agent: agent ?? null, payload: JSON.stringify(Schema.encodeSync(Request)(request)), }) + .onConflictDoNothing() .run() .pipe( EffectRuntime.orDie, EffectRuntime.onError(() => EffectRuntime.sync(() => pending.delete(request.id))), ) + // A deterministic id can collide with its own expired row (a prior attempt shut down + // gracefully); revive it so the reply path and pollers see one pending ask again. + yield* db + .update(PermissionRequestTable) + .set({ status: "pending", message: null }) + .where(and(eq(PermissionRequestTable.id, request.id), eq(PermissionRequestTable.status, "expired"))) + .run() + .pipe(EffectRuntime.orDie) yield* events .publish(Event.Asked, request) .pipe(EffectRuntime.onError(() => EffectRuntime.sync(() => pending.delete(request.id)))) @@ -278,7 +300,19 @@ const layer = Layer.effect( }) } if (result.effect === "allow") return - const item = yield* create(request(input), input.agent) + const value = request(input) + // A deterministic id may already have a settled or in-flight row from a prior attempt of + // the same call: honor a reply that landed while the asker was dead, and adopt a pending + // row instead of duplicating the ask. + const existing = yield* readRow(value.id) + if (existing) { + if (existing.status === "approved") return + if (existing.status === "corrected") + return yield* new CorrectedError({ feedback: existing.message ?? "" }) + if (existing.status === "declined") return yield* EffectRuntime.die(new DeclinedError()) + // pending or expired: fall through; create adopts (insert no-ops) or revives the row. + } + const item = yield* create(value, input.agent) return yield* restore( EffectRuntime.raceFirst(Deferred.await(item.deferred), awaitRow(item.request.id)), ).pipe( diff --git a/packages/core/test/permission-durable.test.ts b/packages/core/test/permission-durable.test.ts index c9e6dca91743..cf4f84da4d7f 100644 --- a/packages/core/test/permission-durable.test.ts +++ b/packages/core/test/permission-durable.test.ts @@ -122,6 +122,68 @@ describe("PermissionV2 durable asks", () => { }), ) + it.live("a re-drive adopts the same pending ask instead of duplicating it", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + yield* seed(file) + const A = yield* Layer.build(stack(file)) + const B = yield* Layer.build(stack(file)) + const permA = Context.get(A, PermissionV2.Service) + const permB = Context.get(B, PermissionV2.Service) + const input = { + sessionID, + action: "bash", + resources: ["git push"], + agent, + source: { type: "tool" as const, messageID: "msg_1", callID: "call_redrive" }, + } + + // Attempt 1 blocks, then dies (a crashed activity): the row stays pending. + const first = yield* permA.assert(input).pipe(Effect.forkChild) + const ask = yield* awaitAsk(permB) + yield* Fiber.interrupt(first) + // Attempt 2 (the Temporal retry) files the same deterministic ask: one row, same id. + const second = yield* permA.assert(input).pipe(Effect.forkChild) + yield* Effect.sleep(100) + const asks = yield* permB.list() + expect(asks).toHaveLength(1) + expect(asks[0]?.id).toBe(ask.id) + yield* permB.reply({ requestID: ask.id, reply: "once" }) + const exit = yield* Fiber.await(second) + expect(Exit.isSuccess(exit)).toBe(true) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + it.live("honors an approval that landed while the asker was dead", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + yield* seed(file) + const A = yield* Layer.build(stack(file)) + const B = yield* Layer.build(stack(file)) + const permA = Context.get(A, PermissionV2.Service) + const permB = Context.get(B, PermissionV2.Service) + const input = { + sessionID, + action: "bash", + resources: ["make deploy"], + agent, + source: { type: "tool" as const, messageID: "msg_2", callID: "call_dead_asker" }, + } + + const first = yield* permA.assert(input).pipe(Effect.forkChild) + const ask = yield* awaitAsk(permB) + yield* Fiber.interrupt(first) + // The human approves after the asker died; the retry short-circuits on the approved row. + yield* permB.reply({ requestID: ask.id, reply: "once" }) + yield* permA.assert(input) + expect(yield* permB.list()).toEqual([]) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + it.live("expires locally-pending asks on shutdown so they do not linger as pending rows", () => Effect.gen(function* () { const tmp = yield* Effect.promise(() => tmpdir()) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 1cd4c7868bee..27b5037d1f90 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -185,8 +185,12 @@ raised elsewhere. Replies keep their semantics: `once`/`always` approve (an `alw retro-approves other pending asks it covers), `reject` declines and cascades to the session's other pending asks, a rejection message arrives as the typed `CorrectedError`. A user decline inside a Temporal activity is now a non-retryable failure, so the workflow does not re-drive a turn the user -stopped. Graceful shutdown retires the process's pending asks as `expired`; after a hard crash a row -can linger pending until a reply lands on it, which then has nothing to resume and is a no-op. +stopped. Tool-originated asks have deterministic ids (session + callID + action + resources), so a +re-driven activity adopts the same pending row instead of filing a duplicate, and an approval that +landed while the asker was dead short-circuits the retry (a one-time approve is honored across +re-drives without a saved rule). Graceful shutdown retires the process's pending asks as `expired` +and a revived attempt flips them back to pending; after a hard crash the pending row simply feeds +the retry. A pending ask whose session is abandoned lingers in the list until a reply retires it. Verified by `packages/core/test/permission-durable.test.ts` (two independent stacks over one store). The `question` tool still uses an in-process deferred and needs the same treatment. From cb155556a681f8907fb227c32882a020d1a015cb Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 10:37:40 -0700 Subject: [PATCH 026/103] Fixed two workflow races around drains and interrupts. Two waiters parked on the drain condition could both observe it satisfied in one activation and start two concurrent drains against one session log; the wait now re-checks in a loop before claiming the drain. And the interrupt signal cancels the workflow's root scope, so a cancellation could surface at the idle wait outside any try/catch and record the workflow as Failed; the main loop now treats that cancellation as a normal stop. Applies to both workflows. Running workflows from before this change will not replay cleanly; acceptable on this fork. --- .../session/execution/temporal-workflow.ts | 84 ++++++++++++------- 1 file changed, 53 insertions(+), 31 deletions(-) diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index e868065b0f43..95793e6ef6ea 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -47,10 +47,15 @@ export async function sessionExecution(sessionID: string): Promise { let draining = false let handlers = 0 - // Serialize drains, like the coordinator (one owner fiber per session at a time). + // Serialize drains, like the coordinator (one owner fiber per session at a time). Re-check after + // every wakeup: two waiters parked on the same condition can both observe `!draining` in one + // activation, and without the loop both would start a drain. const drainOnce = async (force: boolean) => { - await condition(() => !draining || stopping) - if (stopping) return + for (;;) { + await condition(() => !draining || stopping) + if (stopping) return + if (!draining) break + } draining = true try { await runContinuation({ sessionID, force }) @@ -77,21 +82,28 @@ export async function sessionExecution(sessionID: string): Promise { } }) - for (;;) { - const gotWork = await condition(() => pendingWake || stopping, IDLE_TIMEOUT) - if (stopping) return - if (!gotWork) { - // Idle: terminate only when nothing is in flight. A later wake/resume starts a fresh run. - if (!draining && handlers === 0) return - continue - } - pendingWake = false - try { - await drainOnce(false) - } catch (e) { - // wake tolerates run errors (the coordinator logs and moves on); only cancellation stops us. - if (isCancellation(e)) return + // interrupt cancels the workflow's root scope, so a cancellation can surface at the idle wait + // itself, not just inside a drain; treat it as a normal stop rather than a workflow failure. + try { + for (;;) { + const gotWork = await condition(() => pendingWake || stopping, IDLE_TIMEOUT) + if (stopping) return + if (!gotWork) { + // Idle: terminate only when nothing is in flight. A later wake/resume starts a fresh run. + if (!draining && handlers === 0) return + continue + } + pendingWake = false + try { + await drainOnce(false) + } catch (e) { + // wake tolerates run errors (the coordinator logs and moves on); only cancellation stops us. + if (isCancellation(e)) return + } } + } catch (e) { + if (isCancellation(e)) return + throw e } } @@ -106,8 +118,12 @@ export async function sessionTurn(sessionID: string): Promise { let handlers = 0 const drainTurn = async (force: boolean) => { - await condition(() => !draining || stopping) - if (stopping) return + // Same re-check loop as drainOnce: a single wakeup must admit a single drain. + for (;;) { + await condition(() => !draining || stopping) + if (stopping) return + if (!draining) break + } draining = true try { let step = 1 @@ -141,18 +157,24 @@ export async function sessionTurn(sessionID: string): Promise { } }) - for (;;) { - const gotWork = await condition(() => pendingWake || stopping, IDLE_TIMEOUT) - if (stopping) return - if (!gotWork) { - if (!draining && handlers === 0) return - continue - } - pendingWake = false - try { - await drainTurn(false) - } catch (e) { - if (isCancellation(e)) return + // Same as sessionExecution: a root-scope cancellation surfacing at the idle wait is a stop. + try { + for (;;) { + const gotWork = await condition(() => pendingWake || stopping, IDLE_TIMEOUT) + if (stopping) return + if (!gotWork) { + if (!draining && handlers === 0) return + continue + } + pendingWake = false + try { + await drainTurn(false) + } catch (e) { + if (isCancellation(e)) return + } } + } catch (e) { + if (isCancellation(e)) return + throw e } } From 03c0db42c69e1e7de26145c664ae1019d70380b9 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 10:38:46 -0700 Subject: [PATCH 027/103] Hardened resume, interrupt, and active against races and mode drift. resume retries once when the update lands on a workflow that idle-completed in the same instant, so the caller gets a fresh run instead of the race. interrupt no longer swallows delivery failures silently (an already-completed workflow stays a quiet no-op; anything else logs a warning). active queries both workflow types, so sessions survive a temporal/temporal-turn mode switch, and intersects with the session store so other deployments sharing the namespace do not appear as ghost sessions. --- .../core/src/session/execution/temporal.ts | 77 +++++++++++++------ 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index a64b17e6599f..0e96d6354308 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -230,16 +230,25 @@ const layer = Layer.effect( return SessionExecution.Service.of({ // Durable and restart-surviving: the open per-session workflows in Temporal ARE the active - // set (unlike a process-local Set, which is empty after a restart). - active: Effect.promise(async () => { - const ids = new Set() - for await (const wf of client.workflow.list({ - query: `WorkflowType = '${WORKFLOW_TYPE}' AND ExecutionStatus = 'Running'`, - })) { - if (wf.workflowId.startsWith(SESSION_PREFIX)) { - ids.add(SessionSchema.ID.make(wf.workflowId.slice(SESSION_PREFIX.length))) + // set (unlike a process-local Set, which is empty after a restart). Both workflow types are + // queried so sessions survive a mode switch, and the result is intersected with this store's + // sessions because visibility is namespace-wide (other deployments sharing the namespace must + // not appear as ghosts). Visibility is eventually consistent: a just-woken session can lag + // here by about a second. + active: Effect.gen(function* () { + const found = yield* Effect.promise(async () => { + const ids: SessionSchema.ID[] = [] + for await (const wf of client.workflow.list({ + query: `(WorkflowType = 'sessionExecution' OR WorkflowType = 'sessionTurn') AND ExecutionStatus = 'Running'`, + })) { + if (wf.workflowId.startsWith(SESSION_PREFIX)) { + ids.push(SessionSchema.ID.make(wf.workflowId.slice(SESSION_PREFIX.length))) + } } - } + return ids + }) + const ids = new Set() + for (const id of found) if (yield* store.get(id)) ids.add(id) return ids }), wake: (id) => drive(id).pipe(Effect.asVoid), @@ -248,25 +257,45 @@ const layer = Layer.effect( resume: (id) => Effect.tryPromise({ try: async () => { - const startOp = new WithStartWorkflowOperation(WORKFLOW, { - taskQueue: TASK_QUEUE, - workflowId: workflowId(id), - args: [id], - workflowIdConflictPolicy: "USE_EXISTING", - }) - await client.workflow.executeUpdateWithStart(WF.resume, { - startWorkflowOperation: startOp, - args: [], - }) + const attempt = () => { + const startOp = new WithStartWorkflowOperation(WORKFLOW, { + taskQueue: TASK_QUEUE, + workflowId: workflowId(id), + args: [id], + workflowIdConflictPolicy: "USE_EXISTING", + }) + return client.workflow.executeUpdateWithStart(WF.resume, { + startWorkflowOperation: startOp, + args: [], + }) + } + try { + await attempt() + } catch (e) { + // The long-lived workflow self-completes after its idle timeout; an update admitted + // in that instant fails against the just-completed run instead of starting a fresh + // one. Retry once so the caller gets a real run, not the race. + const message = String((e as { message?: unknown })?.message ?? "") + if (!/already completed|not found/i.test(message)) throw e + await attempt() + } }, catch: (e) => toRunError(id, e), }), interrupt: (id) => - Effect.promise(() => - client.workflow - .getHandle(workflowId(id)) - .signal(WF.interrupt) - .catch(() => {}), + Effect.tryPromise({ + try: () => client.workflow.getHandle(workflowId(id)).signal(WF.interrupt), + catch: (e) => e, + }).pipe( + Effect.catch((e) => { + const message = String((e as { message?: unknown })?.message ?? e) + // An idle session's workflow has already completed; nothing to interrupt is fine. A + // genuine delivery failure must not be silent: the user asked for a stop. + if (/already completed|not found/i.test(message)) return Effect.void + return Effect.logWarning("session interrupt signal failed").pipe( + Effect.annotateLogs({ sessionID: id, error: message }), + ) + }), ), }) }), From df6fb0c8acd4da08ade0f950760e733f329da4ca Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 10:40:08 -0700 Subject: [PATCH 028/103] Covered libSQL auto-commit statements with the transaction permit. The acquirer released its permit on handing out the connection, so an auto-commit statement (a permission reply, a cross-service read) could race an open pinned write transaction and die on SQLITE_BUSY, since the remote path has no busy_timeout. Each statement now runs under the permit; in-transaction statements use the transaction's own connection, so nothing self-deadlocks. Documented the per-delta remote transaction cost as a known streaming limit. --- packages/core/src/database/sqlite.libsql.ts | 17 ++++++++++++++++- packages/temporal/README.md | 4 ++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/core/src/database/sqlite.libsql.ts b/packages/core/src/database/sqlite.libsql.ts index e789aab5cdfa..a4543131b77a 100644 --- a/packages/core/src/database/sqlite.libsql.ts +++ b/packages/core/src/database/sqlite.libsql.ts @@ -154,7 +154,22 @@ const make = (options: LibsqlConfig) => } const semaphore = yield* Semaphore.make(1) - const acquirer = semaphore.withPermits(1)(Effect.succeed(connection)) + // The permit must cover the STATEMENT, not just the handing-out of the connection: an + // auto-commit write racing an open pinned transaction would get SQLITE_BUSY from the server + // (the remote path has no busy_timeout) and die the caller. Statements inside a transaction + // scope route to the transaction's own connection, so this never self-deadlocks. + const guarded = identity({ + execute: (query, params, transformRows) => + semaphore.withPermits(1)(connection.execute(query, params, transformRows)), + executeRaw: (query, params) => semaphore.withPermits(1)(connection.executeRaw(query, params)), + executeValues: (query, params) => semaphore.withPermits(1)(connection.executeValues(query, params)), + executeUnprepared: (query, params, transformRows) => + semaphore.withPermits(1)(connection.executeUnprepared(query, params, transformRows)), + executeStream() { + return Stream.die("executeStream not implemented") + }, + }) + const acquirer = Effect.succeed(guarded) const transactionAcquirer = Effect.uninterruptibleMask((restore) => { const fiber = Fiber.getCurrent()! const scope = Context.getUnsafe(fiber.context, Scope.Scope) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 27b5037d1f90..ed8068531a1f 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -230,6 +230,10 @@ distinct worker identities. The remote suite needs a server, so it runs only when `OPENCODE_LIBSQL_TEST_URL` is set (e.g. `turso dev --port 8899`, then `OPENCODE_LIBSQL_TEST_URL=http://127.0.0.1:8899`); it skips otherwise. +- Known limit: every durable event is its own transaction, and the engine records text/reasoning + deltas as events, so a streaming turn against a REMOTE store pays one interactive transaction per + delta, serialized per process. Fine for a shared local file; expect reduced streaming throughput + over a network URL until delta events are batched for the remote backend. ### What resumes cross-host, and what does not From 82ac2dfbc9b71b887362e8219c81efc17cb33f48 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 10:42:07 -0700 Subject: [PATCH 029/103] Fixed the review's low-severity batch. The Phase 1 proxy gave createSession/abortTurn a heartbeat timeout they never satisfy and the abort signal left a floating activity with unlimited retries; they now use their own bounded proxy and the rejection is swallowed. A crash-finalized step with only provider-executed tools now closes as "stop" instead of "tool-calls". The README migration caveat predated the cross-process BEGIN IMMEDIATE serialization and is updated. --- packages/core/src/session/runner/llm.ts | 7 +++++-- .../core/src/session/runner/loop-guard.ts | Bin 2622 -> 2842 bytes packages/temporal/README.md | 6 ++++-- packages/temporal/src/workflows.ts | 14 +++++++++++--- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 05fa60323754..ea196181a958 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -500,11 +500,14 @@ const layer = Layer.effect( .files({ from: Snapshot.ID.make(startSnapshot), to: endSnapshot }) .pipe(Effect.catch(() => Effect.succeed(undefined))) : undefined + const localTools = toolParts.some((part) => part.provider?.executed !== true) yield* events.publish(SessionEvent.Step.Ended, { sessionID: input.sessionID, timestamp: yield* DateTime.now, assistantMessageID: inFlight.id, - finish: "tool-calls", + // "tool-calls" only when a local tool actually needs a follow-up turn; a step whose tools + // were all provider-executed finalizes as a plain stop. + finish: localTools ? "tool-calls" : "stop", cost: 0, tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, snapshot: endSnapshot, @@ -512,7 +515,7 @@ const layer = Layer.effect( }) // Mirror runStep's continuation tail: a step with local tool calls continues so the model sees // the (reused or failed) results. - let needsContinuation = toolParts.some((part) => part.provider?.executed !== true) + let needsContinuation = localTools if (!needsContinuation) needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer") if (needsContinuation) return { ran: true, continue: true, step: input.step + 1, promotion: "steer" as SessionInput.Delivery } diff --git a/packages/core/src/session/runner/loop-guard.ts b/packages/core/src/session/runner/loop-guard.ts index 3806f2e18e94fa11798f2f3518c03b845f1a2f23..1df0edc2640b02a8f22c04879f07f61b2abc30b4 100644 GIT binary patch delta 232 zcmXYrF-}A=3`8ZW6*Z!s8tt9{39i5aXuSFJiA9LNVn;zVdyj3o04LyV32)?a%B=Q^(PW}t`-Y*B5&Vqf{$gZJ89I3tk`lL&b0MrpvJYal d27%LEt^Q9eW}DVGOk=0J%-pc{{{3{g`2+93Qsw{v delta 12 TcmbOwwohb(J;!Eg&i{-69mE7t diff --git a/packages/temporal/README.md b/packages/temporal/README.md index ed8068531a1f..0d3548380c82 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -215,8 +215,10 @@ distinct worker identities. ### Caveats -- Run migrations once as a deploy step before starting N workers; the first-open migration guard is - process-local, so N cold workers migrating at once can race. +- Cold-start migrations serialize across processes (one `BEGIN IMMEDIATE` transaction wraps + check-and-apply, so concurrent starts wait and then no-op). Running migrations once as a deploy + step is still good practice for large fleets, and on a remote store it keeps cold starts from + contending for the write lock. - The PRAGMAs (`journal_mode` / `synchronous` / `busy_timeout` / `cache_size` / `wal_checkpoint`) are local-file semantics and are skipped for the shared/libSQL backend, which manages journaling itself. diff --git a/packages/temporal/src/workflows.ts b/packages/temporal/src/workflows.ts index 0e4c414ea090..d719beaa266c 100644 --- a/packages/temporal/src/workflows.ts +++ b/packages/temporal/src/workflows.ts @@ -3,12 +3,19 @@ import type * as activities from "./activities" // Unlimited retries with heartbeat: this is what makes a turn survive a worker crash. The turn // runs server-side, so a re-run just re-attaches (idempotent on the user-message count). -const { createSession, runTurn, abortTurn } = proxyActivities({ +const { runTurn } = proxyActivities({ startToCloseTimeout: "15 minutes", // Short heartbeat so a dead worker's in-flight turn is detected and re-driven quickly. The turn // keeps running server-side meanwhile, so the retry just re-attaches. heartbeatTimeout: "8 seconds", }) +// These calls never heartbeat, so they must not carry a heartbeat timeout (any call slower than it +// would fail spuriously). createSession is a non-idempotent POST: a retry after an ambiguous +// failure can orphan a server-side session, so retries are bounded. +const { createSession, abortTurn } = proxyActivities({ + startToCloseTimeout: "1 minute", + retry: { maximumAttempts: 3 }, +}) export const submitPrompt = defineSignal<[string]>("submitPrompt") export const abortSession = defineSignal("abortSession") @@ -46,8 +53,9 @@ export async function durableSession(input: DurableSessionInput = {}): Promise { // Best-effort: tell the server to abort the active turn; the running activity then returns the - // (aborted) assistant message and the loop moves on. - void abortTurn(sessionID) + // (aborted) assistant message and the loop moves on. A floating rejection would fail the + // workflow task, so it is swallowed. + abortTurn(sessionID).catch(() => {}) }) setHandler(getState, () => ({ sessionID, turns, pending: queue.length })) From 37280d3b46475d0d0a83ee1ff9de59e6be04592b Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 12:47:53 -0700 Subject: [PATCH 030/103] Added the AI-399 local-options evaluation draft with measurements. Evaluates ways to run a Temporal-integrated agent without Temporal (language shim, rust-core shim, plugin pattern, local dev server, harness option). This branch is first-hand evidence for the plugin pattern, so the doc lives here. Includes measured numbers: dev server 237 MB / ~780 ms / ~102 MB RSS, and the marginal cost on this very app (297 MB local vs 494 MB + 123 MB dev server in temporal-turn mode). Research sections marked pending. --- packages/temporal/docs/ai399-local-options.md | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 packages/temporal/docs/ai399-local-options.md diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md new file mode 100644 index 000000000000..187ebd35c952 --- /dev/null +++ b/packages/temporal/docs/ai399-local-options.md @@ -0,0 +1,159 @@ +# AI-399: Local options for Temporal-integrated agents + +Status: DRAFT. Sections marked [research pending] are being filled; measurements are final. + +Some customers integrate Temporal into their agents and want the same agent loop to run without +Temporal, e.g. shipped desktop software. This evaluates the candidate paths and recommends what to +tell customers and what to build. It lives on this branch because the fork around it is first-hand +evidence: one engine, three execution modes (in-process, one activity per turn, one per step), +selected by an env var. + +## The requirement, sharpened + +"Run without Temporal" hides three different customer asks, and the right answer differs: + +1. **Same loop, no infrastructure at all** (desktop/CLI): no server process, no ports, minimal + footprint. Durability degrades gracefully (a checkpoint file beats nothing). +2. **Same loop, no OPERATED infrastructure** (single-machine server, appliance): a local process + is fine if it is zero-admin. Full Temporal semantics wanted. +3. **Same CODE, both worlds**: the vendor ships one codebase; cloud deployments get durability, + desktop gets local. The dominant engineering constraint is preventing drift between the paths. + +## Paths + +### A. Shim Temporal at the language layer + +Replace the Temporal SDK surface the agent code touches (`workflow.*`, `proxyActivities`, +signals/updates/timers) with a local implementation, so unmodified workflow code runs in-process. + +- Durability: whatever the shim persists. A faithful shim needs event-sourced replay to recover + mid-workflow, which is the hard part of Temporal, reimplemented. +- Signals/Updates: re-implemented on the shim's event loop; semantic-drift risk is high + (buffering, ordering, update validators, cancellation scopes). +- Complexity: high and permanent; the shim chases the SDK surface every release, and determinism + constraints stay imposed on local code that gets nothing for them. +- Where it shines: an existing Temporal-first codebase that cannot be refactored, needing a local + mode quickly, using a small enumerable SDK subset. +- [research pending: the referenced existing implementation; ask reporter which one is meant.] + +### B. Shim in the Rust core + +Implement a local mode under sdk-core so all core-based SDKs (TypeScript, Python, .NET, Ruby) get +it at once: the core's server-facing surface backed by an embedded, in-process implementation. + +- One implementation serves many languages, unlike A (per-language). +- It is materially an embedded single-tenant Temporal server: task matching, history, timers, + replay. The question is whether it beats shipping the dev server (D) once the work is done. +- Adjacent precedent: the Java SDK's time-skipping test server; the CLI dev server. +- [research pending: sdk-core's exact seam, prior maintainer discussions, feasibility.] + +### C. Plugin pattern: one loop, swappable execution (two first-hand implementations) + +Factor the agent so the LOOP is pure and the execution substrate is injected; Temporal is one +substrate, a local runner is another. + +**Evidence 1: this fork.** opencode's v2 engine event-sources every session to a store and exposes +a substitutable `SessionExecution` interface (`active`/`wake`/`resume`/`interrupt`, four methods). +The stock implementation is an in-process coordinator; this fork adds a Temporal-backed one +(`OPENCODE_SESSION_EXECUTION=temporal` per turn, `temporal-turn` per step) plus a shared store so +any worker resumes any session. Because durability lives in the engine's event log, the LOCAL mode +is already crash-recoverable without Temporal; Temporal adds supervised retries, worker +distribution, restart-surviving visibility, and cross-process interrupt/resume. The swap point is +the execution supervisor, not the state store. Verified end to end in this branch (crash tests, +failover tests, an independent architecture review). + +**Evidence 2: `agent-harness` (AI-363, TypeScript).** A pure loop state machine plus an `Effects` +interface (`callModel`/`runTools`/`onEvent`) implemented twice: local mode (direct calls, atomic +write-then-rename checkpoint file, crash-resume) and durable mode (workflow + activities with +measured defaults). About 100 lines per runner around a shared core. Local mode gives up retries +across process death mid-tool, multi-worker capacity, and the audit trail; it keeps the same loop, +tools, prompts, and checkpoint crash-resume. + +- Durability: local = what the local runner persists (checkpoint file is coarse; an event-sourced + store is fine-grained and close to Temporal-grade for single-machine crashes). Temporal = full. +- Signals/Updates: the app defines the interface both modes honor (here: wake, interrupt, one + awaited resume update). No pretense of Temporal's generic protocol locally, and no drift, + because the loop is the same code. +- Complexity: lowest sustained cost of all paths; the seam is app-defined and small. The cost is + up-front design: it is not a bolt-on for an existing Temporal-first codebase. +- [research pending: ADK's runner/session-service seam as the pattern's external example; the + OpenAI Agents SDK integration shape.] + +### D. Run the Temporal dev server locally + +Ship `temporal server start-dev` alongside the app; the agent stays a plain Temporal application. + +Measured here (macOS arm64): + +| Metric | Value | +|---|---| +| Binary size | 237 MB (CLI incl. server and embedded UI assets) | +| Cold start to healthy | ~780 ms (`--headless`) | +| RSS idle, empty | ~102 MB | +| RSS after 20 workflow starts | ~128 MB | +| SQLite file | 568 KB empty; 620 KB after 20 workflow starts | +| Persistence | `--db-filename` required; the default is in-memory and loses everything on exit | + +Marginal cost measured on THIS app (same engine, same machine): + +| Configuration | RSS | +|---|---| +| opencode serve, stock local mode | 297 MB | +| opencode serve, `temporal-turn` (embedded worker) | 494 MB | +| dev server alongside | +123 MB | +| Total for full Temporal semantics locally | ~617 MB across two processes, +237 MB disk | + +- Durability and signal/update fidelity: full Temporal semantics, the only path with no gap. +- Complexity for the vendor: near zero code; the cost moves to packaging and lifecycle (bundling + a 237 MB binary, process supervision, ports, upgrades). +- [research pending: licensing of CLI+UI for redistribution, statements on production use, + feature gaps vs the real server.] + +### E. Integrate a local option into the Temporal Agent Harness + +Not an independent runtime: it is where a choice among A/C/D becomes product. The harness today is +Temporal-native (agents ARE workflows; approvals, Code Mode, callback tools, and the event stream +ride Temporal primitives). A local mode via C would make the harness's public abstractions (agent +definition, tools, approval policy, event stream) the swap seam with a non-Temporal transport +locally (in-process bus, identical schemas); approvals and the event stream are exactly what +desktop users still want. + +## Comparison + +Scoring: full / partial / none, with the load-bearing caveat inline. [two cells pending research] + +| | A: language shim | B: rust-core shim | C: plugin pattern | D: local dev server | +|---|---|---|---|---| +| Durability (crash mid-turn) | partial: what the shim persists; faithful replay = reimplementing Temporal | full IF built (it IS an embedded server) | partial: checkpoint-file coarse; event-sourced local store near-full for one machine | full (with `--db-filename`) | +| Resource needs | lightest (in-process) | in-process, but core carries history+matching | lightest (in-process; store is a file) | heaviest: ~102-123 MB RSS second process, 237 MB disk, ~+200 MB embedded worker | +| Signal/Update support | re-implemented, drift-prone | full IF built | app-defined subset, honored identically by both modes | full | +| Complexity / maintainability | high, permanent (chases SDK surface per language) | highest once, then per-core; a product, not a patch | lowest sustained; up-front loop design | near-zero code; packaging+lifecycle burden | +| Feature fidelity (timers, retries, CAN, child wfs, replay debug) | subset, hand-built | full IF built | not applicable locally (app semantics, not Temporal's) | full minus [research: dev-server gaps] | +| Code-drift risk between modes | medium (same code, different semantics) | low (same code, same semantics) | none for the loop (same code); seam is small | none (same code, same semantics) | +| Desktop packaging | best | good | best | worst (bundle+supervise a server) | +| Upgrade path to Temporal Cloud | same code, repoint | same code, repoint | swap the factory | same code, repoint | +| Offline | yes | yes | yes | yes (local server) | +| Exists today | partially (referenced implementation [research]) | no | yes, twice (this fork; agent-harness) | yes (shipped CLI) | + +## Recommendations (draft) + +1. **New agent designs: recommend C.** Two working implementations show the cost is one small + interface; it is the only path with near-zero sustained maintenance and no semantic pretense. + When local durability matters, event-source the session in the app (this fork's shape) rather + than reimplementing Temporal's replay. +2. **Existing Temporal-first agents needing desktop distribution now: D.** Full fidelity today; + the cost is packaging, not code. Requires `--db-filename` and lifecycle supervision. + [confirm licensing/support posture] +3. **A is the fallback** when neither refactoring (C) nor a second process (D) is acceptable: + scope it to the SDK subset actually used and accept drift risk. +4. **B is product strategy, not a customer workaround**: an embedded Temporal across all + core-based SDKs. Evaluate only if we want to productize "Temporal without the server". +5. **Harness (E): apply C at the harness API layer** so harness users get a local mode without + forking the Temporal-native internals. + +## Open questions + +- Which language-layer shim does the ticket refer to, and what SDK subset did it cover? +- Dev server: sanctioned/licensed for redistribution inside customer desktop apps? +- What signal/update surface do desktop agents actually need? Evidence here says small and + enumerable: wake, interrupt, one awaited update (this fork); approvals and steering (harness). From a7dc137c6bf30818d8174c0aa63955ccfa36a698 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 12:52:15 -0700 Subject: [PATCH 031/103] Resolved the language-shim reference in the AI-399 draft. The ticket's "has been implemented" means our TypeScript prototype (which strictly swaps at an app-defined seam, path C) and shims customers added in their own code (true path A). The two have opposite maintenance profiles, so the doc keeps the distinction. --- packages/temporal/docs/ai399-local-options.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index 187ebd35c952..b183e4678bc9 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -34,7 +34,12 @@ signals/updates/timers) with a local implementation, so unmodified workflow code constraints stay imposed on local code that gets nothing for them. - Where it shines: an existing Temporal-first codebase that cannot be refactored, needing a local mode quickly, using a small enumerable SDK subset. -- [research pending: the referenced existing implementation; ask reporter which one is meant.] +- The ticket's "has been implemented" covers two things (per the assignee): our own TypeScript + prototype, and shims customers have added inside their own codebases. Strictly, the TS prototype + (`agent-harness`, AI-363) swaps at an app-defined seam, which this doc classifies as path C; + the customer-authored variants are true path A: they fake the SDK surface their code touches and + carry the drift risk described above. The distinction matters because the two have opposite + maintenance profiles. ### B. Shim in the Rust core @@ -133,7 +138,7 @@ Scoring: full / partial / none, with the load-bearing caveat inline. [two cells | Desktop packaging | best | good | best | worst (bundle+supervise a server) | | Upgrade path to Temporal Cloud | same code, repoint | same code, repoint | swap the factory | same code, repoint | | Offline | yes | yes | yes | yes (local server) | -| Exists today | partially (referenced implementation [research]) | no | yes, twice (this fork; agent-harness) | yes (shipped CLI) | +| Exists today | yes, by customers in their own code | no | yes, twice (this fork; agent-harness) | yes (shipped CLI) | ## Recommendations (draft) @@ -153,7 +158,6 @@ Scoring: full / partial / none, with the load-bearing caveat inline. [two cells ## Open questions -- Which language-layer shim does the ticket refer to, and what SDK subset did it cover? - Dev server: sanctioned/licensed for redistribution inside customer desktop apps? - What signal/update surface do desktop agents actually need? Evidence here says small and enumerable: wake, interrupt, one awaited update (this fork); approvals and steering (harness). From 57e2c7bec9e14da2403a50196a1a63b06bb07329 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 12:55:12 -0700 Subject: [PATCH 032/103] Resolved the dev-server redistribution question in the AI-399 draft. No licensing issue; a real option for appliance deployments. The motivating desktop customer declined a bundled server process, so the desktop archetype routes to the in-process paths. --- packages/temporal/docs/ai399-local-options.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index b183e4678bc9..8eeb81e8a4bd 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -111,8 +111,10 @@ Marginal cost measured on THIS app (same engine, same machine): - Durability and signal/update fidelity: full Temporal semantics, the only path with no gap. - Complexity for the vendor: near zero code; the cost moves to packaging and lifecycle (bundling a 237 MB binary, process supervision, ports, upgrades). -- [research pending: licensing of CLI+UI for redistribution, statements on production use, - feature gaps vs the real server.] +- Redistribution: no licensing issue; shipping it inside a desktop app is a real option. However, + the motivating desktop customer (Cursor) does not want a bundled server process, so this path is + parked for the desktop archetype and stays the answer for the appliance/single-machine one. +- [research pending: feature gaps vs the real server.] ### E. Integrate a local option into the Temporal Agent Harness @@ -146,9 +148,11 @@ Scoring: full / partial / none, with the load-bearing caveat inline. [two cells interface; it is the only path with near-zero sustained maintenance and no semantic pretense. When local durability matters, event-source the session in the app (this fork's shape) rather than reimplementing Temporal's replay. -2. **Existing Temporal-first agents needing desktop distribution now: D.** Full fidelity today; - the cost is packaging, not code. Requires `--db-filename` and lifecycle supervision. - [confirm licensing/support posture] +2. **Existing Temporal-first agents on a machine the customer controls (appliance, + single-machine server): D.** Full fidelity today; the cost is packaging, not code. Requires + `--db-filename` and lifecycle supervision. No licensing issue. For the DESKTOP archetype this + is off the table by customer preference (no bundled server process), which is exactly why C + (or A as the fallback) carries that case. 3. **A is the fallback** when neither refactoring (C) nor a second process (D) is acceptable: scope it to the SDK subset actually used and accept drift risk. 4. **B is product strategy, not a customer workaround**: an embedded Temporal across all @@ -158,6 +162,5 @@ Scoring: full / partial / none, with the load-bearing caveat inline. [two cells ## Open questions -- Dev server: sanctioned/licensed for redistribution inside customer desktop apps? - What signal/update surface do desktop agents actually need? Evidence here says small and enumerable: wake, interrupt, one awaited update (this fork); approvals and steering (harness). From cb327de128097d31d4fe739135965145f76a5408 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 12:58:16 -0700 Subject: [PATCH 033/103] Resolved the desktop signal/update surface in the AI-399 draft. The whole desktop surface is cancel plus one more prompt (plus approvals where a policy engine exists). Temporal's generic protocol is not the bar a local mode must clear; a two-verb app interface is, which bounds a scoped shim and keeps the plugin seam cheap. --- packages/temporal/docs/ai399-local-options.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index 8eeb81e8a4bd..8a5cbf4f8c81 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -33,7 +33,8 @@ signals/updates/timers) with a local implementation, so unmodified workflow code - Complexity: high and permanent; the shim chases the SDK surface every release, and determinism constraints stay imposed on local code that gets nothing for them. - Where it shines: an existing Temporal-first codebase that cannot be refactored, needing a local - mode quickly, using a small enumerable SDK subset. + mode quickly, using a small enumerable SDK subset. The confirmed desktop surface (cancel plus + one more prompt, below) keeps that subset genuinely small. - The ticket's "has been implemented" covers two things (per the assignee): our own TypeScript prototype, and shims customers have added inside their own codebases. Strictly, the TS prototype (`agent-harness`, AI-363) swaps at an app-defined seam, which this doc classifies as path C; @@ -160,7 +161,15 @@ Scoring: full / partial / none, with the load-bearing caveat inline. [two cells 5. **Harness (E): apply C at the harness API layer** so harness users get a local mode without forking the Temporal-native internals. -## Open questions +## The signal/update surface desktop agents actually need -- What signal/update surface do desktop agents actually need? Evidence here says small and - enumerable: wake, interrupt, one awaited update (this fork); approvals and steering (harness). +Resolved during review: the user can cancel the in-flight request, and can add one more prompt +while a run is active. That is the whole desktop surface (plus tool approvals where a policy +engine is present, per the harness). Both first-hand implementations already model exactly this: +opencode's `SessionExecution` is wake (new prompt), interrupt (cancel), one awaited resume; the +agent-harness durable mode is submitPrompt, abortSession, closeSession. + +The implication for the comparison: Temporal's generic signal/update protocol is not the bar a +local mode has to clear. A two-verb app-defined interface is. That is what makes path C cheap, +and it bounds path A's re-implementation burden IF the shim is scoped to these verbs instead of +the full SDK surface. From 628c02b0dcdbe58b1afbd333221ea8bec5f4cdff Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 13:09:06 -0700 Subject: [PATCH 034/103] Finalized the AI-399 evaluation with research findings. Three additions changed the draft. Temporal already ships the dual-mode pattern for ADK (`workflow.in_workflow()` fallback in the model/tool wrappers, with two sharp edges worth documenting), so that path has a shipped reference, not a proposal. The rust-core seam exists and is public (`ConnectionOptions.service_override`, already in the C bridge), but the payload is a second Temporal service implementation and the Java test server shows the parity cost. And the control surface a local mode must honor, enumerated from this app's protocol, is about a dozen verbs, which bounds a scoped shim and keeps the plugin seam cheap. Prior art, demand evidence (temporal#298's six-year arc, unanswered desktop threads), corrected dev-server release numbers, and final recommendations included. --- packages/temporal/docs/ai399-local-options.md | 292 ++++++++++++------ 1 file changed, 191 insertions(+), 101 deletions(-) diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index 8a5cbf4f8c81..a5fdcfb6170d 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -1,7 +1,5 @@ # AI-399: Local options for Temporal-integrated agents -Status: DRAFT. Sections marked [research pending] are being filled; measurements are final. - Some customers integrate Temporal into their agents and want the same agent loop to run without Temporal, e.g. shipped desktop software. This evaluates the candidate paths and recommends what to tell customers and what to build. It lives on this branch because the fork around it is first-hand @@ -19,6 +17,9 @@ selected by an env var. 3. **Same CODE, both worlds**: the vendor ships one codebase; cloud deployments get durability, desktop gets local. The dominant engineering constraint is preventing drift between the paths. +The motivating desktop customer (Cursor) rejects a bundled server process, so archetype 1 must be +answered in-process. + ## Paths ### A. Shim Temporal at the language layer @@ -26,34 +27,48 @@ selected by an env var. Replace the Temporal SDK surface the agent code touches (`workflow.*`, `proxyActivities`, signals/updates/timers) with a local implementation, so unmodified workflow code runs in-process. +- Exists today: customers have added shims like this in their own code (per the ticket), paying + the cost themselves. - Durability: whatever the shim persists. A faithful shim needs event-sourced replay to recover mid-workflow, which is the hard part of Temporal, reimplemented. - Signals/Updates: re-implemented on the shim's event loop; semantic-drift risk is high (buffering, ordering, update validators, cancellation scopes). -- Complexity: high and permanent; the shim chases the SDK surface every release, and determinism - constraints stay imposed on local code that gets nothing for them. +- Complexity: high and permanent for a general shim; the shim chases the SDK surface every + release, and determinism constraints stay imposed on local code that gets nothing for them. A + shim scoped to the app's actual control surface (enumerated below) is much smaller, at the cost + of being app-specific. - Where it shines: an existing Temporal-first codebase that cannot be refactored, needing a local - mode quickly, using a small enumerable SDK subset. The confirmed desktop surface (cancel plus - one more prompt, below) keeps that subset genuinely small. -- The ticket's "has been implemented" covers two things (per the assignee): our own TypeScript - prototype, and shims customers have added inside their own codebases. Strictly, the TS prototype - (`agent-harness`, AI-363) swaps at an app-defined seam, which this doc classifies as path C; - the customer-authored variants are true path A: they fake the SDK surface their code touches and - carry the drift risk described above. The distinction matters because the two have opposite - maintenance profiles. + mode quickly. ### B. Shim in the Rust core -Implement a local mode under sdk-core so all core-based SDKs (TypeScript, Python, .NET, Ruby) get -it at once: the core's server-facing surface backed by an embedded, in-process implementation. - -- One implementation serves many languages, unlike A (per-language). -- It is materially an embedded single-tenant Temporal server: task matching, history, timers, - replay. The question is whether it beats shipping the dev server (D) once the work is done. -- Adjacent precedent: the Java SDK's time-skipping test server; the CLI dev server. -- [research pending: sdk-core's exact seam, prior maintainer discussions, feasibility.] - -### C. Plugin pattern: one loop, swappable execution (two first-hand implementations) +Implement a local mode under the core SDK so all core-based SDKs (TypeScript, Python, .NET, Ruby) +get it at once. + +Research findings (sdk-core is now `temporalio/sdk-rust`; MIT): + +- **The insertion point is clean and already public.** `ConnectionOptions.service_override` + (`crates/client/src/options_structs.rs`) routes every gRPC call through a supplied callback + instead of the network, and the C ABI already plumbs it (`grpc_override_callback` in + `sdk-core-c-bridge`), so .NET/Ruby could intercept in-process today. Python does not expose it; + TypeScript unverified. Why the hook exists is unverified (possibly proxying/testing). +- **Determinism, state machines, and replay live in core, not the server**, so a shim does NOT + reimplement the hard part. What it must implement is the SERVICE: task matching with long-poll + semantics, history append/read, workflow task lifecycle, server-side timers, activity retry and + four timeout types, signals/queries/updates (update's multi-stage lifecycle), ID reuse/conflict + policies, continue-as-new, child workflows, cancellation. +- **The honest size estimate is the Java time-skipping test server**: a from-scratch in-memory + service reimplementation, GraalVM-compiled, consumed by Python/.NET/TS/Ruby, and still short of + parity after years (`sdk-java#1804`, a Temporal employee asking for the real dev server in + tests because `listWorkflowExecutions` is missing). +- Core's replay worker (`init_replay_worker`) proves the plumbing tolerates a non-network client, + but it consumes pre-recorded history only; it is not a local runtime. + +Read: the seam is cheap, the payload is a second implementation of the Temporal service, and +Temporal's existing second implementation has not reached parity. Only worth doing as an owned +product commitment ("embedded Temporal"), not as a workaround. + +### C. Plugin pattern: one loop, swappable execution Factor the agent so the LOOP is pure and the execution substrate is injected; Temporal is one substrate, a local runner is another. @@ -71,51 +86,72 @@ failover tests, an independent architecture review). **Evidence 2: `agent-harness` (AI-363, TypeScript).** A pure loop state machine plus an `Effects` interface (`callModel`/`runTools`/`onEvent`) implemented twice: local mode (direct calls, atomic write-then-rename checkpoint file, crash-resume) and durable mode (workflow + activities with -measured defaults). About 100 lines per runner around a shared core. Local mode gives up retries -across process death mid-tool, multi-worker capacity, and the audit trail; it keeps the same loop, -tools, prompts, and checkpoint crash-resume. +measured defaults). About 100 lines per runner around a shared core. This is the TypeScript +prototype the ticket's "has been implemented" refers to. Local mode gives up retries across +process death mid-tool, multi-worker capacity, and the audit trail; it keeps the same loop, tools, +prompts, and checkpoint crash-resume. + +**Evidence 3, and the headline: Temporal already ships this pattern for ADK.** The ticket points +at ADK as the example; research found the official integration +(`pip install "temporalio[google-adk]"`, `temporalio.contrib.google_adk_agents`, experimental) IS +the dual-mode answer, with a mechanism cheaper than a factory: + +- ADK's own seam is the `Runner` + `BaseSessionService` (in-memory, sqlite, database, vertex + implementations), so local ADK needs no infrastructure. +- The Temporal integration does not replace the Runner. It wraps the model seam (`TemporalModel` + runs the call as an `invoke_model` activity) and the tool seam (`activity_tool` dispatches via + `workflow.execute_activity`), and each wrapper degrades to a direct in-process call when + `temporalio.workflow.in_workflow()` is false. Determinism helpers branch on the same predicate + (clock and ID providers, via hooks Temporal landed upstream in ADK, + `google.adk.platform.{time,uuid}`). +- One agent definition, two execution modes, no second code path for the user. Sharp edges worth + telling customers: MCP toolsets cannot auto-fall-back (the caller must supply + `not_in_workflow_toolset`), and durable mode is STRICTER than local (whole session state must + serialize within payload limits), so local-only testing can pass and then fail under Temporal. + +Characteristics of the family: - Durability: local = what the local runner persists (checkpoint file is coarse; an event-sourced store is fine-grained and close to Temporal-grade for single-machine crashes). Temporal = full. -- Signals/Updates: the app defines the interface both modes honor (here: wake, interrupt, one - awaited resume update). No pretense of Temporal's generic protocol locally, and no drift, - because the loop is the same code. -- Complexity: lowest sustained cost of all paths; the seam is app-defined and small. The cost is - up-front design: it is not a bolt-on for an existing Temporal-first codebase. -- [research pending: ADK's runner/session-service seam as the pattern's external example; the - OpenAI Agents SDK integration shape.] +- Signals/Updates: the app defines the interface both modes honor. No pretense of Temporal's + generic protocol locally, and no drift, because the loop is the same code. +- Complexity: lowest sustained cost of all paths; the seam is app-defined and small, or (ADK + style) hidden inside integration wrappers. The cost is up-front design; not a bolt-on for an + existing Temporal-first codebase, except where Temporal ships the integration. ### D. Run the Temporal dev server locally Ship `temporal server start-dev` alongside the app; the agent stays a plain Temporal application. -Measured here (macOS arm64): +Facts (research verified on release artifacts, v1.8.2, 2026-07; local measurements on this +machine's dev build): | Metric | Value | |---|---| -| Binary size | 237 MB (CLI incl. server and embedded UI assets) | +| Release binary (darwin arm64) | 127.5 MiB uncompressed, 37.5 MiB compressed (this machine's 237 MB was a dev build; use the release number) | +| Platforms | macOS/Linux/Windows, amd64+arm64, one static Go binary incl. server, CLI, Web UI | | Cold start to healthy | ~780 ms (`--headless`) | -| RSS idle, empty | ~102 MB | -| RSS after 20 workflow starts | ~128 MB | -| SQLite file | 568 KB empty; 620 KB after 20 workflow starts | -| Persistence | `--db-filename` required; the default is in-memory and loses everything on exit | - -Marginal cost measured on THIS app (same engine, same machine): - -| Configuration | RSS | -|---|---| -| opencode serve, stock local mode | 297 MB | -| opencode serve, `temporal-turn` (embedded worker) | 494 MB | -| dev server alongside | +123 MB | -| Total for full Temporal semantics locally | ~617 MB across two processes, +237 MB disk | - -- Durability and signal/update fidelity: full Temporal semantics, the only path with no gap. -- Complexity for the vendor: near zero code; the cost moves to packaging and lifecycle (bundling - a 237 MB binary, process supervision, ports, upgrades). -- Redistribution: no licensing issue; shipping it inside a desktop app is a real option. However, - the motivating desktop customer (Cursor) does not want a bundled server process, so this path is - parked for the desktop archetype and stays the answer for the appliance/single-machine one. -- [research pending: feature gaps vs the real server.] +| RSS idle | ~102-139 MB (this machine 102 MB; research 139 MB on v1.31.2) | +| SQLite file | ~0.6 MB empty | +| Persistence | `--db-filename` required; the DEFAULT is in-memory and loses everything on exit | +| Embedding-friendly flags | `--headless`, `--db-filename`, `--port`, `--ip`, repeatable `--namespace`, `--sqlite-pragma`, `--dynamic-config-value`; http/metrics ports default to random free | +| License | MIT (verified in release tarball and via GitHub API) | + +Marginal cost measured on THIS app (same engine, same machine): stock local serve 297 MB RSS; +`temporal-turn` serve 494 MB (embedded worker) + dev server 123 MB = ~617 MB across two processes, +plus the binary on disk, for full Temporal semantics on identical code. + +- **Fidelity is the differentiator: it is the real server against SQLite, not an emulator.** + Research verified multi-namespace and Nexus endpoints work and persist. Signals, updates, + queries, schedules, search attributes are the real implementations. No drift, ever. +- **Positioning is the weakness.** The binary itself prints a not-for-production warning (added + deliberately, `cli#689`); the embedded-server docs page says testing and development only; + limits are real (SQLite single writer, `NumHistoryShards: 1`, all roles in one process). + Restate and Inngest bless their single-node binaries for production; Temporal is the only one + in the comparison set whose local mode is officially disowned. +- Redistribution: no licensing issue (MIT). A real option for the appliance archetype. The + motivating desktop customer does not want a bundled server process, so this path is parked for + desktop. ### E. Integrate a local option into the Temporal Agent Harness @@ -123,53 +159,107 @@ Not an independent runtime: it is where a choice among A/C/D becomes product. Th Temporal-native (agents ARE workflows; approvals, Code Mode, callback tools, and the event stream ride Temporal primitives). A local mode via C would make the harness's public abstractions (agent definition, tools, approval policy, event stream) the swap seam with a non-Temporal transport -locally (in-process bus, identical schemas); approvals and the event stream are exactly what -desktop users still want. +locally (in-process bus, identical schemas). The ADK integration's `in_workflow()` mechanism is +the shipped precedent for how the harness's Temporal-aware pieces could degrade in-process. + +## What the local mode actually has to support: the control surface + +Measured from opencode's protocol (the session group plus human-in-the-loop groups), the session +control surface a desktop agent product exposes is enumerable, roughly a dozen verbs: + +- Run control: `prompt` (with two delivery semantics: steer into the running turn, or queue after + it), `interrupt`, `wait` (await settlement). +- Human-in-the-loop: permission reply (`once`/`always`/reject-with-correction), agent questions + (typed ask/answer). +- Session mutation: `switchAgent`, `switchModel`, `compact`, `revert` (stage/clear/commit). +- Observation: history, context, live event stream, active set. + +This is far richer than a cancel button, and far smaller than Temporal's generic signal/update +protocol. Every verb maps to a signal/update/query in Temporal mode and an in-process call +locally; the event-sourced store is what lets both modes serve the observation verbs identically. +This bounds path A (a scoped shim is a dozen verbs, not the SDK) and explains why path C stays +cheap: the interface already exists in any real product. + +## Prior art: how others answer local-without-the-big-server + +| Pattern | Example | Mechanism | Gives up | +|---|---|---|---| +| Pluggable store, same process | LangGraph (`checkpointer=`: memory/sqlite/postgres); DBOS (Python defaults to SQLite) | one interface, N stores | LangGraph memory: restart durability; DBOS local: multi-process recovery | +| Same binary, different config | Restate (single binary, RocksDB); Inngest (`inngest dev`, in-memory by default; `inngest start` for prod) | run the real thing small | fault tolerance of a cluster; a process always runs | +| Emulator | Azure Durable Task Scheduler emulator | separate implementation | fidelity; explicitly not production | +| In-process protocol reimplementation | Resonate `LocalNetwork` (server state machine over dicts); Temporal's Java test server | exact semantics, zero deps | maintaining two implementations forever | + +Temporal's dev server is the second pattern with one difference: Restate and Inngest bless their +single-node story for production; Temporal explicitly does not. + +## Demand evidence + +The ask is real, old, and largely unanswered publicly: + +- `temporalio/temporal#298` "run Temporal as an embedded library", opened 2020 by Maxim Fateev + ("For small scale on prem deployments... would be really great"), closed 2026 pointing at + `start-dev` with not-for-production caveats; requesters explicitly disputed the closure (the + actual ask was an in-process, NATS-style embedded server). +- Three desktop/OEM forum threads (2023-2025: Wails desktop app, standalone Go binary packaging, + bundled-Temporal upgrade sequencing) with zero staff replies among them. +- The only substantive edge guidance is a 2025 forum answer (IoT/submarine): run the single + binary on the device, federate with Nexus; which sits against docs saying SQLite single-process + is testing/development only. +- Two abandoned commitments: Maxim in 2020 ("Desktop applications... we absolutely going to + create it. No ETA"); Temporalite's maintainer in 2022 ("the vision is for Temporalite to be + used in production contexts"). Temporalite is archived; `temporal#3366` (SQLite in production) + is still open. ## Comparison -Scoring: full / partial / none, with the load-bearing caveat inline. [two cells pending research] - -| | A: language shim | B: rust-core shim | C: plugin pattern | D: local dev server | +| | A: language shim | B: rust-core shim | C: plugin pattern (incl. ADK-style fallback) | D: local dev server | |---|---|---|---|---| -| Durability (crash mid-turn) | partial: what the shim persists; faithful replay = reimplementing Temporal | full IF built (it IS an embedded server) | partial: checkpoint-file coarse; event-sourced local store near-full for one machine | full (with `--db-filename`) | -| Resource needs | lightest (in-process) | in-process, but core carries history+matching | lightest (in-process; store is a file) | heaviest: ~102-123 MB RSS second process, 237 MB disk, ~+200 MB embedded worker | -| Signal/Update support | re-implemented, drift-prone | full IF built | app-defined subset, honored identically by both modes | full | -| Complexity / maintainability | high, permanent (chases SDK surface per language) | highest once, then per-core; a product, not a patch | lowest sustained; up-front loop design | near-zero code; packaging+lifecycle burden | -| Feature fidelity (timers, retries, CAN, child wfs, replay debug) | subset, hand-built | full IF built | not applicable locally (app semantics, not Temporal's) | full minus [research: dev-server gaps] | -| Code-drift risk between modes | medium (same code, different semantics) | low (same code, same semantics) | none for the loop (same code); seam is small | none (same code, same semantics) | -| Desktop packaging | best | good | best | worst (bundle+supervise a server) | -| Upgrade path to Temporal Cloud | same code, repoint | same code, repoint | swap the factory | same code, repoint | -| Offline | yes | yes | yes | yes (local server) | -| Exists today | yes, by customers in their own code | no | yes, twice (this fork; agent-harness) | yes (shipped CLI) | - -## Recommendations (draft) - -1. **New agent designs: recommend C.** Two working implementations show the cost is one small - interface; it is the only path with near-zero sustained maintenance and no semantic pretense. - When local durability matters, event-source the session in the app (this fork's shape) rather - than reimplementing Temporal's replay. -2. **Existing Temporal-first agents on a machine the customer controls (appliance, - single-machine server): D.** Full fidelity today; the cost is packaging, not code. Requires - `--db-filename` and lifecycle supervision. No licensing issue. For the DESKTOP archetype this - is off the table by customer preference (no bundled server process), which is exactly why C - (or A as the fallback) carries that case. -3. **A is the fallback** when neither refactoring (C) nor a second process (D) is acceptable: - scope it to the SDK subset actually used and accept drift risk. -4. **B is product strategy, not a customer workaround**: an embedded Temporal across all - core-based SDKs. Evaluate only if we want to productize "Temporal without the server". -5. **Harness (E): apply C at the harness API layer** so harness users get a local mode without - forking the Temporal-native internals. - -## The signal/update surface desktop agents actually need - -Resolved during review: the user can cancel the in-flight request, and can add one more prompt -while a run is active. That is the whole desktop surface (plus tool approvals where a policy -engine is present, per the harness). Both first-hand implementations already model exactly this: -opencode's `SessionExecution` is wake (new prompt), interrupt (cancel), one awaited resume; the -agent-harness durable mode is submitPrompt, abortSession, closeSession. - -The implication for the comparison: Temporal's generic signal/update protocol is not the bar a -local mode has to clear. A two-verb app-defined interface is. That is what makes path C cheap, -and it bounds path A's re-implementation burden IF the shim is scoped to these verbs instead of -the full SDK surface. +| Durability (crash mid-turn) | partial: what the shim persists; faithful replay = reimplementing Temporal | full IF built (it IS an embedded service) | partial: checkpoint-file coarse; event-sourced local store near-full for one machine | full (with `--db-filename`; default is in-memory) | +| Resource needs | lightest (in-process) | in-process; core carries matching+history | lightest (in-process; store is a file) | ~102-139 MB RSS second process, ~128 MiB disk; measured here ~617 MB total for this app | +| Signal/Update support | re-implemented, drift-prone | full IF built | the app's control surface (~a dozen verbs), honored identically by both modes | full | +| Complexity / maintainability | high, permanent (chases SDK surface per language); bounded if scoped to the control surface | highest; a second Temporal service implementation (Java test server: years, still short of parity) | lowest sustained; up-front loop design, or shipped by Temporal (ADK) | near-zero code; packaging+lifecycle burden | +| Feature fidelity | subset, hand-built | full IF built | app semantics, not Temporal's | full: the real server (multi-namespace, Nexus verified) | +| Code-drift between modes | medium (same code, different semantics) | low | none for the loop; ADK caveat: durable mode is stricter (serialization), so local-pass/durable-fail exists | none | +| Desktop packaging | best | good | best | worst (bundle+supervise a server); MIT, no licensing issue | +| Upgrade path to Temporal Cloud | same code, repoint | same code, repoint | swap the factory / connect the client | same code, repoint | +| Exists today | yes, by customers in their own code | no (seam exists: `service_override`; payload does not) | yes, three times: this fork, agent-harness, and the shipped ADK integration | yes (shipped CLI) | + +## Recommendations + +1. **For customers on a framework Temporal integrates with (ADK today): point at the shipped + integration.** The `in_workflow()` fallback is the productized version of the plugin pattern: + one agent definition, two modes, maintained by Temporal. Write up the two sharp edges (MCP + toolsets need an explicit local implementation; durable mode is stricter than local, so test + both modes). +2. **For customers designing their own loop: recommend the plugin pattern (C).** Two working + implementations here show the cost is one small interface. Enumerate the control surface (it + is about a dozen verbs in a real product) and event-source the session when local durability + matters, rather than reimplementing Temporal's replay. +3. **For existing Temporal-first agents on machines the customer controls (appliance, + single-machine): D**, with `--db-filename` and process supervision. Full fidelity, no + licensing issue. Off the table for the desktop archetype by customer preference. +4. **A is the fallback** when neither refactoring (C) nor a second process (D) is acceptable: + scope the shim to the control surface, not the SDK, and accept the drift risk. +5. **B is a product decision, not a customer recommendation.** The seam exists + (`service_override`, already in the C bridge); the payload is a second Temporal service + implementation, and our own Java test server shows the parity cost. The demand evidence + (six years of #298, unanswered desktop/OEM threads, two abandoned commitments) says there is a + real, unclaimed "embedded Temporal" position; claiming it means owning that implementation + indefinitely. That decision belongs to product, informed by this doc. +6. **Harness (E): apply C at the harness API layer**, using the ADK integration's fallback + mechanism as the shipped precedent for how Temporal-aware pieces degrade in-process. + +## Sources + +Primary: this branch (implementation + measurements); `agent-harness` (AI-363); +`temporalio/sdk-rust` source; `temporalio/sdk-python` `contrib/google_adk_agents` source; a +downloaded v1.8.2 CLI release binary, run headless. + +Referenced: adk.dev/integrations/temporal · temporal.io/blog/google-adk-temporal-integration-bts · +temporalio/samples-python `google_adk_agents` · google/adk-python `sessions/` · +docs.temporal.io/self-hosted-guide/embedded-server · docs.temporal.io/cli/server · +temporalio/cli#689 · temporalio/temporal#298 · temporalio/temporal#3366 · +temporalio/sdk-java#1804 · temporalio/temporalite-archived · community.temporal.io threads +17424, 12914, 10125, 18362 · LangGraph persistence docs · Restate architecture docs · DBOS +database docs · Inngest dev-server docs · Azure Durable Functions storage-providers docs · +resonatehq/resonate-sdk-py `network/local.py`. From bc62a908fa105b8b4b84f1c7d1cdf9669f672e34 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 19:35:00 -0700 Subject: [PATCH 035/103] Defined the appliance archetype on first use. --- packages/temporal/docs/ai399-local-options.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index a5fdcfb6170d..7796354a734a 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -12,8 +12,9 @@ selected by an env var. 1. **Same loop, no infrastructure at all** (desktop/CLI): no server process, no ports, minimal footprint. Durability degrades gracefully (a checkpoint file beats nothing). -2. **Same loop, no OPERATED infrastructure** (single-machine server, appliance): a local process - is fine if it is zero-admin. Full Temporal semantics wanted. +2. **Same loop, no OPERATED infrastructure** (an appliance: one machine the customer operates, + e.g. an on-prem server box or an edge device): a local process is fine if it is zero-admin. + Full Temporal semantics wanted. 3. **Same CODE, both worlds**: the vendor ships one codebase; cloud deployments get durability, desktop gets local. The dominant engineering constraint is preventing drift between the paths. From 2096942c391924f8ccf766f4ab4b69352c628c73 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 19:45:54 -0700 Subject: [PATCH 036/103] Clarified that the dev-server pairing quantifies the rejected alternative. This fork's intended desktop shape is the in-process local mode; the bundled-server measurement exists to show what staying in-process avoids. --- packages/temporal/docs/ai399-local-options.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index 7796354a734a..7acd51b992e3 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -140,7 +140,10 @@ machine's dev build): Marginal cost measured on THIS app (same engine, same machine): stock local serve 297 MB RSS; `temporal-turn` serve 494 MB (embedded worker) + dev server 123 MB = ~617 MB across two processes, -plus the binary on disk, for full Temporal semantics on identical code. +plus the binary on disk, for full Temporal semantics on identical code. Measured to quantify the +alternative, not to propose it here: this fork's intended desktop shape is the in-process local +mode (path C), which is what we would suggest to the desktop customer. A bundled server is the +appliance answer, not the desktop one. - **Fidelity is the differentiator: it is the real server against SQLite, not an emulator.** Research verified multi-namespace and Nexus endpoints work and persist. Signals, updates, From 2fa468c9cc1d954dad7db1abc13fd72601f1361a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 10 Aug 2026 19:57:45 -0700 Subject: [PATCH 037/103] Reframed the ambient-check pattern as the compatibility variant. The ADK integration's `in_workflow()` check is what you use when you do not own the framework's composition root and a workflow cannot receive a live object graph. Its costs are structural: mode invisible at call sites, the conditional repeated per wrapper, no construction-time enforcement, and a runtime raise where degradation is impossible (the MCP edge). When you own the composition root, a factory returning the interface is the right shape; both implementations here already do that, and the harness recommendation now says so explicitly. --- packages/temporal/docs/ai399-local-options.md | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index 7acd51b992e3..bf6ec08f3c07 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -92,10 +92,10 @@ prototype the ticket's "has been implemented" refers to. Local mode gives up ret process death mid-tool, multi-worker capacity, and the audit trail; it keeps the same loop, tools, prompts, and checkpoint crash-resume. -**Evidence 3, and the headline: Temporal already ships this pattern for ADK.** The ticket points -at ADK as the example; research found the official integration -(`pip install "temporalio[google-adk]"`, `temporalio.contrib.google_adk_agents`, experimental) IS -the dual-mode answer, with a mechanism cheaper than a factory: +**Evidence 3: Temporal already ships a dual-mode ADK integration.** The ticket points at ADK as +the example; the official integration (`pip install "temporalio[google-adk]"`, +`temporalio.contrib.google_adk_agents`, experimental) is shipped and dual-mode. Its mechanism is +an ambient check, not a factory: - ADK's own seam is the `Runner` + `BaseSessionService` (in-memory, sqlite, database, vertex implementations), so local ADK needs no infrastructure. @@ -110,6 +110,19 @@ the dual-mode answer, with a mechanism cheaper than a factory: `not_in_workflow_toolset`), and durable mode is STRICTER than local (whole session state must serialize within payload limits), so local-only testing can pass and then fail under Temporal. +The ambient check is the compatibility variant of this pattern, not the recommended shape. It +exists because Temporal does not own ADK's composition root, and because a workflow cannot receive +a live object graph as input, so the check detects which behavior is legal in the current +environment. Its costs are structural: the mode is invisible at call sites, the conditional +repeats in every wrapper, the type system cannot enforce a complete local graph, and where uniform +degradation is impossible the failure is a runtime raise (the MCP edge above) instead of a +construction-time requirement. When you own the composition root, prefer a factory that returns +the interface: both implementations above do exactly that (one binding selects the +`SessionExecution` implementation; one `Effects` value selects the runner), and a missing local +implementation then fails at construction, not mid-run. The two shapes compose: a substrate +factory at the agent-definition level with the ambient check kept only as a safety net would +also close the MCP hole in ADK-shaped integrations. + Characteristics of the family: - Durability: local = what the local runner persists (checkpoint file is coarse; an event-sourced @@ -250,8 +263,11 @@ The ask is real, old, and largely unanswered publicly: (six years of #298, unanswered desktop/OEM threads, two abandoned commitments) says there is a real, unclaimed "embedded Temporal" position; claiming it means owning that implementation indefinitely. That decision belongs to product, informed by this doc. -6. **Harness (E): apply C at the harness API layer**, using the ADK integration's fallback - mechanism as the shipped precedent for how Temporal-aware pieces degrade in-process. +6. **Harness (E): apply C at the harness API layer**, as a factory returning the harness's + execution interface, chosen once at the composition root. Not ambient `in_workflow()` checks: + we own this composition root, so the local implementation should be enforced at construction + time. The ADK integration is the precedent for the wrapper technique where we do not own the + root, not for the harness's own design. ## Sources From fbfe6e3ebe5a73ccc4d29708a1abdb538f57a3de Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 12:44:21 -0700 Subject: [PATCH 038/103] Ran the session supervisor in-process with a six-primitive micro-driver. The supervisor is now written once (workflow-core.ts) over a WorkflowRuntime interface: condition, signal/update handlers, the two drain calls, and cancellation. Two drivers execute it. The Temporal workflow adapter provides the primitives from the real SDK, and a new in-process micro-driver (local-driver.ts, `OPENCODE_SESSION_EXECUTION=local-driver[-turn]`) provides them with plain promises and an AbortController: no server, no worker, no ports. The drain bodies and the error codec are shared modules, so turn semantics and typed errors are identical in both modes; the factory in routes.ts picks the driver. This closes the two-code-paths objection to the plugin pattern: the part that could drift now exists once. Contract tests drive the shared supervisor in-process (turn settles, the exact tagged RunError crosses the same encode/decode path, interrupt cancels, idle retires); the worker smoke proves the refactored workflow still bundles in the Temporal sandbox. --- packages/core/src/session/execution/drain.ts | 109 +++++++++ .../src/session/execution/local-driver.ts | 220 ++++++++++++++++++ .../src/session/execution/run-error-codec.ts | 21 ++ .../session/execution/temporal-workflow.ts | 171 +++----------- .../core/src/session/execution/temporal.ts | 119 +--------- .../src/session/execution/workflow-core.ts | 178 ++++++++++++++ .../session-execution-local-driver.test.ts | 206 ++++++++++++++++ packages/server/src/routes.ts | 13 +- 8 files changed, 780 insertions(+), 257 deletions(-) create mode 100644 packages/core/src/session/execution/drain.ts create mode 100644 packages/core/src/session/execution/local-driver.ts create mode 100644 packages/core/src/session/execution/workflow-core.ts create mode 100644 packages/core/test/session-execution-local-driver.test.ts diff --git a/packages/core/src/session/execution/drain.ts b/packages/core/src/session/execution/drain.ts new file mode 100644 index 000000000000..db4aeb39ccf4 --- /dev/null +++ b/packages/core/src/session/execution/drain.ts @@ -0,0 +1,109 @@ +// The drain bodies shared by every durable executor: the Temporal layer runs them inside +// activities, the in-process micro-driver calls them directly. One implementation, so the turn +// semantics and the error encoding cannot differ between drivers. + +import { Cause, Context, Effect, Exit, type LayerMap } from "effect" +import { ApplicationFailure } from "@temporalio/activity" +import type { LocationServiceMap } from "../../location-service-map" +import type { Location } from "../../location" +import type { LocationError, LocationServices } from "../../location-services" +import { SessionRunner } from "../runner" +import { SessionSchema } from "../schema" +import { SessionStore } from "../store" +import type { SessionInput } from "../input" +import { encodeRunError } from "./run-error-codec" +import type { DrainInput, StepDrainInput, StepDrainResult } from "./temporal-activities" + +export interface DrainDeps { + readonly store: SessionStore.Interface + readonly locations: LayerMap.LayerMap + /** The app context the drain runs in; providing it plus the per-location layer supplies + * SessionRunner and all of its dependencies. */ + readonly ctx: Context.Context +} + +export const makeDrains = ({ store, locations, ctx }: DrainDeps) => { + const drain = async (input: DrainInput, signal: AbortSignal): Promise => { + const exit = await Effect.runPromiseExit( + Effect.gen(function* () { + const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) + if (!session) return + yield* SessionRunner.Service.use((runner) => + runner.run({ sessionID: session.id, force: input.force }), + ).pipe(Effect.provide(locations.get(session.location))) + }).pipe(Effect.provide(ctx), Effect.scoped), + { signal }, + ) + if (Exit.isSuccess(exit)) return + const cause = exit.cause + if (Cause.hasInterruptsOnly(cause)) { + // Two interrupt sources: driver cancellation (the AbortSignal fired -- rethrow its reason so + // the attempt records Cancelled, not Failed) and an internal halt like a user declining a + // permission (the signal did NOT fire). The latter must be non-retryable, or the supervisor + // re-drives a turn the user explicitly stopped. + if (signal.aborted) + throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") + throw ApplicationFailure.create({ + message: "session run halted (user declined)", + type: "SessionRunDeclined", + nonRetryable: true, + }) + } + // A genuine run error is thrown non-retryable so Temporal surfaces it (to resume) rather than + // retrying; only crashes / task timeouts (never thrown here) go through the retry policy. The + // error is encoded faithfully in `details` so the caller can reconstruct the exact RunError. + const squashed = Cause.squash(cause) as { _tag?: string; message?: string } + const encoded = encodeRunError(squashed) + throw ApplicationFailure.create({ + message: squashed?.message ?? Cause.pretty(cause), + type: squashed?._tag ?? "SessionRunError", + nonRetryable: true, + details: encoded === undefined ? undefined : [encoded], + }) + } + + // Per-step drain: run exactly one step of the turn (used by the per-step supervisor). Same + // context and error encoding as the whole-turn drain; returns the next loop state. + const stepDrain = async (input: StepDrainInput, signal: AbortSignal): Promise => { + const exit = await Effect.runPromiseExit( + Effect.gen(function* () { + const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) + if (!session) return { ran: false, continue: false, step: input.step, promotion: null } + const r = yield* SessionRunner.Service.use((runner) => + runner.runStep({ + sessionID: session.id, + step: input.step, + promotion: (input.promotion ?? undefined) as SessionInput.Delivery | undefined, + first: input.first, + force: input.force, + }), + ).pipe(Effect.provide(locations.get(session.location))) + return { ran: r.ran, continue: r.continue, step: r.step, promotion: r.promotion ?? null } + }).pipe(Effect.provide(ctx), Effect.scoped), + { signal }, + ) + if (Exit.isSuccess(exit)) return exit.value + const cause = exit.cause + if (Cause.hasInterruptsOnly(cause)) { + // Same split as the whole-turn drain: cancellation rethrows its reason (records Cancelled), + // an internal user-decline halt is non-retryable. + if (signal.aborted) + throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") + throw ApplicationFailure.create({ + message: "session run halted (user declined)", + type: "SessionRunDeclined", + nonRetryable: true, + }) + } + const squashed = Cause.squash(cause) as { _tag?: string; message?: string } + const encoded = encodeRunError(squashed) + throw ApplicationFailure.create({ + message: squashed?.message ?? Cause.pretty(cause), + type: squashed?._tag ?? "SessionRunError", + nonRetryable: true, + details: encoded === undefined ? undefined : [encoded], + }) + } + + return { drain, stepDrain } +} diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts new file mode 100644 index 000000000000..db2a23cba621 --- /dev/null +++ b/packages/core/src/session/execution/local-driver.ts @@ -0,0 +1,220 @@ +export * as SessionExecutionLocalDriver from "./local-driver" + +// The in-process driver for the session supervisor (workflow-core.ts). It runs the SAME supervisor +// function the Temporal workflow runs, with the six runtime primitives implemented over plain +// promises: `condition` is a polled waiter, signals and updates are method calls, the drains run +// directly (no activities), and cancellation is an AbortController whose reason satisfies the +// drain's cancellation contract. No Temporal server, no worker, no ports; durability comes from +// the engine's event log, exactly as in local coordinator mode. This is the "one supervisor, two +// drivers" shape: the factory picks the driver, the supervisor is written once. + +import { Effect, Layer } from "effect" +import { LocationServiceMap } from "../../location-service-map" +import { makeGlobalNode } from "../../effect/app-node" +import { SessionSchema } from "../schema" +import { SessionStore } from "../store" +import { SessionExecution } from "../execution" +import { makeDrains } from "./drain" +import { makeWorkflows, type WorkflowRuntime } from "./workflow-core" +import { toRunError } from "./run-error-codec" + + +const UNITS: Record = { + ms: 1, + millisecond: 1, + milliseconds: 1, + second: 1_000, + seconds: 1_000, + minute: 60_000, + minutes: 60_000, + hour: 3_600_000, + hours: 3_600_000, +} + +const parseDuration = (value: string): number => { + const match = /^\s*([\d.]+)\s*([a-z]+)\s*$/i.exec(value) + const unit = match?.[2] ? UNITS[match[2].toLowerCase()] : undefined + if (!match?.[1] || unit === undefined) throw new Error(`Unsupported duration: ${value}`) + return Number(match[1]) * unit +} + +class LocalCancellation extends Error {} + +interface Waiter { + readonly predicate: () => boolean + readonly resolve: (value: boolean) => void + readonly reject: (error: unknown) => void + timer?: ReturnType +} + +type Drains = ReturnType + +// One driver per live session. Temporal re-evaluates workflow conditions on every activation; the +// local equivalent is a short poll plus an immediate re-check after every signal/update delivery. +class SessionDriver { + readonly done: Promise + completed = false + private readonly signalHandlers = new Map void>() + private readonly updateHandlers = new Map Promise>() + private waiters: Waiter[] = [] + private ticker: ReturnType | undefined + private cancelled = false + private readonly abort = new AbortController() + + constructor(run: (rt: WorkflowRuntime) => Promise, drains: Drains, onDone: () => void) { + const rt: WorkflowRuntime = { + condition: (predicate, timeout) => + new Promise((resolve, reject) => { + if (this.cancelled) return reject(new LocalCancellation()) + const waiter: Waiter = { predicate, resolve, reject } + if (timeout !== undefined) + waiter.timer = setTimeout(() => { + this.remove(waiter) + resolve(false) + }, parseDuration(timeout)) + this.waiters.push(waiter) + this.tick() + this.ensureTicker() + }), + setSignalHandler: (name, handler) => this.signalHandlers.set(name, handler), + setUpdateHandler: (name, handler) => this.updateHandlers.set(name, handler), + runContinuation: (input) => drains.drain(input, this.abort.signal), + runTurnStep: (input) => drains.stepDrain(input, this.abort.signal), + cancelCurrentScope: () => this.cancel(), + isCancellation: (error) => error instanceof LocalCancellation, + } + this.done = run(rt).finally(() => { + this.completed = true + this.stopTicker() + onDone() + }) + } + + signal(name: "wake" | "interrupt") { + this.signalHandlers.get(name)?.() + this.tick() + } + + update(name: "resume"): Promise { + const handler = this.updateHandlers.get(name) + if (!handler) return Promise.reject(new Error(`Update handler not registered: ${name}`)) + const result = handler() + // Nudge parked conditions when the update settles; the poll covers everything in between. + result.finally(() => this.tick()).catch(() => {}) + return result + } + + private cancel() { + this.cancelled = true + // The drain rethrows the signal's reason on cancellation, so the supervisor observes the same + // LocalCancellation from a cancelled drain as from a rejected condition. + this.abort.abort(new LocalCancellation("session interrupted")) + for (const waiter of this.waiters.splice(0)) { + if (waiter.timer) clearTimeout(waiter.timer) + waiter.reject(new LocalCancellation()) + } + } + + private remove(waiter: Waiter) { + this.waiters = this.waiters.filter((entry) => entry !== waiter) + } + + private tick() { + for (const waiter of [...this.waiters]) { + if (!waiter.predicate()) continue + this.remove(waiter) + if (waiter.timer) clearTimeout(waiter.timer) + waiter.resolve(true) + } + if (this.waiters.length === 0) this.stopTicker() + } + + private ensureTicker() { + if (this.ticker || this.waiters.length === 0) return + this.ticker = setInterval(() => this.tick(), 25) + } + + private stopTicker() { + if (!this.ticker) return + clearInterval(this.ticker) + this.ticker = undefined + } +} + +/** + * An in-process SessionExecution running the shared supervisor with no Temporal anywhere: + * - wake -> deliver the wake signal (starting a driver if the session has none) + * - resume -> run the resume update and await it, surfacing the exact RunError + * - interrupt -> deliver the interrupt signal (cancels the drain and parked waits) + * - active -> the live drivers + */ +const layer = Layer.effect( + SessionExecution.Service, + Effect.gen(function* () { + const store = yield* SessionStore.Service + const locations = yield* LocationServiceMap.Service + const ctx = yield* Effect.context() + const drains = makeDrains({ store, locations, ctx }) + const drivers = new Map() + // Read at layer build (not module load) so tests can set these before constructing the layer. + // local-driver = whole-turn drains; local-driver-turn = one drain call per step, mirroring the + // temporal / temporal-turn split so both drivers exercise both supervisors. The idle override + // shortens the supervisor's 5-minute self-termination. + const PER_STEP = process.env.OPENCODE_SESSION_EXECUTION === "local-driver-turn" + const IDLE_TIMEOUT = process.env.OPENCODE_SESSION_IDLE_TIMEOUT + + const ensure = (id: SessionSchema.ID): SessionDriver => { + const existing = drivers.get(id) + if (existing && !existing.completed) return existing + const driver = new SessionDriver( + (rt) => { + const workflows = makeWorkflows(rt, IDLE_TIMEOUT ? { idleTimeout: IDLE_TIMEOUT } : undefined) + return (PER_STEP ? workflows.sessionTurn : workflows.sessionExecution)(id) + }, + drains, + () => { + if (drivers.get(id) === driver) drivers.delete(id) + }, + ) + drivers.set(id, driver) + // wake tolerates supervisor failures (they are already recorded in the session log); an + // unhandled rejection here would crash the process instead. + driver.done.catch(() => {}) + return driver + } + + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + for (const driver of drivers.values()) driver.signal("interrupt") + await Promise.allSettled([...drivers.values()].map((driver) => driver.done)) + }), + ) + + yield* Effect.logInfo("SessionExecutionLocalDriver ready").pipe( + Effect.annotateLogs({ supervisor: PER_STEP ? "sessionTurn" : "sessionExecution" }), + ) + + return SessionExecution.Service.of({ + active: Effect.sync(() => new Set(drivers.keys())), + wake: (id) => + Effect.sync(() => { + ensure(id).signal("wake") + }), + resume: (id) => + Effect.tryPromise({ + try: () => ensure(id).update("resume"), + catch: (e) => toRunError(id, e), + }), + interrupt: (id) => + Effect.sync(() => { + drivers.get(id)?.signal("interrupt") + }), + }) + }), +) + +export const node = makeGlobalNode({ + service: SessionExecution.Service, + layer, + deps: [SessionStore.node, LocationServiceMap.node], +}) diff --git a/packages/core/src/session/execution/run-error-codec.ts b/packages/core/src/session/execution/run-error-codec.ts index efe7d8178c71..261289de0986 100644 --- a/packages/core/src/session/execution/run-error-codec.ts +++ b/packages/core/src/session/execution/run-error-codec.ts @@ -7,6 +7,7 @@ import { LLMError } from "@opencode-ai/llm" import { Integration } from "../../integration" import { SystemContext } from "../../system-context/index" import { ToolOutputStore } from "../../tool-output-store" +import type { SessionSchema } from "../schema" import { ContextSnapshotDecodeError, MessageDecodeError } from "../error" import { ModelNotSelectedError, @@ -50,3 +51,23 @@ export function decodeRunError(payload: unknown): SessionRunner.RunError | undef return undefined } } + +// Walk a failure chain (WorkflowUpdateFailedError -> ActivityFailure -> ApplicationFailure, or a +// bare ApplicationFailure from the in-process driver) to the encoded run error the drain attached, +// and reconstruct the exact tagged error. Falls back to a ContextSnapshotDecodeError carrying the +// text, since the RunError union has no generic member. +export function toRunError(sessionID: SessionSchema.ID, e: unknown): SessionRunner.RunError { + let node = e as { details?: unknown; cause?: unknown; message?: string } | undefined + for (let depth = 0; node && depth < 6; depth++) { + if (Array.isArray(node.details) && node.details.length > 0) { + const decoded = decodeRunError(node.details[0]) + if (decoded) return decoded + return new ContextSnapshotDecodeError({ sessionID, details: `session run failed: ${node.message}` }) + } + node = node.cause as typeof node + } + return new ContextSnapshotDecodeError({ + sessionID, + details: `session run failed: ${(e as { message?: string })?.message ?? String(e)}`, + }) +} diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index 95793e6ef6ea..100f40124082 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -1,13 +1,11 @@ -// The durable equivalent of SessionRunCoordinator, as a Temporal workflow (one per session). +// The Temporal driver for the session supervisor. The supervisor itself lives in workflow-core.ts +// and is written once; this file adapts the real SDK's primitives (condition, signal/update +// handlers, activity proxies, cancellation) to the WorkflowRuntime interface and exports the two +// workflow functions the worker registers. The in-process driver (local-driver.ts) runs the SAME +// supervisor with plain promises. // -// MUST stay pure: Temporal bundles this in an isolated sandbox, so no `effect`, no -// `@opencode-ai/core`, no Node builtins. It only ever sees the sessionID string. All real work -// (SessionRunner.run against the durable event log) happens in the runContinuation activity. -// -// Semantics mirror the coordinator (run-coordinator.ts): drains are serialized (one at a time), -// a `wake` drives a drain and is tolerant of errors, and `resume` (an Update) drives a forced -// drain and returns its result to the caller (throwing the run's error). The workflow stays alive -// to serve later wakes/resumes and terminates after an idle period. +// MUST stay sandbox-safe: Temporal bundles this in an isolated context, so no `effect`, no +// `@opencode-ai/core` runtime imports, no Node builtins. import { proxyActivities, @@ -18,7 +16,8 @@ import { CancellationScope, isCancellation, } from "@temporalio/workflow" -import type { Activities, StepActivities, StepDrainResult } from "./temporal-activities" +import type { Activities, StepActivities } from "./temporal-activities" +import { makeWorkflows, type WorkflowRuntime } from "./workflow-core" const activityOptions = { // The heartbeat is the liveness bound (it stops within seconds of a worker death and Temporal @@ -38,143 +37,31 @@ const { runTurnStep } = proxyActivities(activityOptions) export const wake = defineSignal("wake") export const interrupt = defineSignal("interrupt") export const resume = defineUpdate("resume") +const signals = { wake, interrupt } as const -const IDLE_TIMEOUT = "5 minutes" - -export async function sessionExecution(sessionID: string): Promise { - let pendingWake = true // started via signalWithStart -> there is work to drain - let stopping = false - let draining = false - let handlers = 0 - - // Serialize drains, like the coordinator (one owner fiber per session at a time). Re-check after - // every wakeup: two waiters parked on the same condition can both observe `!draining` in one - // activation, and without the loop both would start a drain. - const drainOnce = async (force: boolean) => { - for (;;) { - await condition(() => !draining || stopping) - if (stopping) return - if (!draining) break - } - draining = true - try { - await runContinuation({ sessionID, force }) - } finally { - draining = false +const runtime: WorkflowRuntime = { + condition: async (predicate, timeout) => { + if (timeout === undefined) { + await condition(predicate) + return true } - } + // The runtime interface uses plain strings; the SDK's Duration is a branded string template. + return condition(predicate, timeout as never) + }, + setSignalHandler: (name, handler) => setHandler(signals[name], handler), + setUpdateHandler: (_name, handler) => setHandler(resume, handler), + runContinuation, + runTurnStep, + cancelCurrentScope: () => CancellationScope.current().cancel(), + isCancellation, +} - setHandler(wake, () => { - pendingWake = true - }) - setHandler(interrupt, () => { - stopping = true - CancellationScope.current().cancel() - }) - // resume = coordinator.run: force one drain and surface its result (a run error rejects the - // Update, so the caller observes it). - setHandler(resume, async () => { - handlers++ - try { - await drainOnce(true) - } finally { - handlers-- - } - }) +const workflows = makeWorkflows(runtime) - // interrupt cancels the workflow's root scope, so a cancellation can surface at the idle wait - // itself, not just inside a drain; treat it as a normal stop rather than a workflow failure. - try { - for (;;) { - const gotWork = await condition(() => pendingWake || stopping, IDLE_TIMEOUT) - if (stopping) return - if (!gotWork) { - // Idle: terminate only when nothing is in flight. A later wake/resume starts a fresh run. - if (!draining && handlers === 0) return - continue - } - pendingWake = false - try { - await drainOnce(false) - } catch (e) { - // wake tolerates run errors (the coordinator logs and moves on); only cancellation stops us. - if (isCancellation(e)) return - } - } - } catch (e) { - if (isCancellation(e)) return - throw e - } +export async function sessionExecution(sessionID: string): Promise { + return workflows.sessionExecution(sessionID) } -// Per-step variant (OPENCODE_SESSION_EXECUTION=temporal-turn): identical lifecycle, but a turn is -// driven one step at a time -- each step (one provider attempt + its tools) is its own -// runTurnStep activity, and the step loop is workflow control flow. The loop state (step / -// promotion / first) mirrors SessionRunner.run's loop and lives in the (deterministic) workflow. export async function sessionTurn(sessionID: string): Promise { - let pendingWake = true - let stopping = false - let draining = false - let handlers = 0 - - const drainTurn = async (force: boolean) => { - // Same re-check loop as drainOnce: a single wakeup must admit a single drain. - for (;;) { - await condition(() => !draining || stopping) - if (stopping) return - if (!draining) break - } - draining = true - try { - let step = 1 - let promotion: string | null = null - let first = true - for (;;) { - const r: StepDrainResult = await runTurnStep({ sessionID, step, promotion, first, force }) - if (!r.continue) break - step = r.step - promotion = r.promotion - first = false - } - } finally { - draining = false - } - } - - setHandler(wake, () => { - pendingWake = true - }) - setHandler(interrupt, () => { - stopping = true - CancellationScope.current().cancel() - }) - setHandler(resume, async () => { - handlers++ - try { - await drainTurn(true) - } finally { - handlers-- - } - }) - - // Same as sessionExecution: a root-scope cancellation surfacing at the idle wait is a stop. - try { - for (;;) { - const gotWork = await condition(() => pendingWake || stopping, IDLE_TIMEOUT) - if (stopping) return - if (!gotWork) { - if (!draining && handlers === 0) return - continue - } - pendingWake = false - try { - await drainTurn(false) - } catch (e) { - if (isCancellation(e)) return - } - } - } catch (e) { - if (isCancellation(e)) return - throw e - } + return workflows.sessionTurn(sessionID) } diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 0e96d6354308..0b2ac3042a3c 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -1,27 +1,18 @@ export * as SessionExecutionTemporal from "./temporal" import { fileURLToPath } from "node:url" -import { Cause, Effect, Exit, Layer } from "effect" +import { Effect, Layer } from "effect" import { Client, Connection, WithStartWorkflowOperation } from "@temporalio/client" -import { ApplicationFailure } from "@temporalio/activity" import { NativeConnection, Worker } from "@temporalio/worker" import { LocationServiceMap } from "../../location-service-map" import { makeGlobalNode } from "../../effect/app-node" -import { SessionRunner } from "../runner" import { SessionSchema } from "../schema" import { SessionStore } from "../store" import { SessionExecution } from "../execution" -import { ContextSnapshotDecodeError } from "../error" -import { - makeActivities, - makeStepActivities, - type DrainInput, - type StepDrainInput, - type StepDrainResult, -} from "./temporal-activities" -import type { SessionInput } from "../input" -import { encodeRunError, decodeRunError } from "./run-error-codec" +import { makeActivities, makeStepActivities } from "./temporal-activities" +import { makeDrains } from "./drain" +import { toRunError } from "./run-error-codec" import * as WF from "./temporal-workflow" const ADDRESS = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" @@ -43,24 +34,6 @@ const ROLE = process.env.OPENCODE_TEMPORAL_ROLE ?? "both" const HOST_WORKER = ROLE !== "client" const HOST_CLIENT = ROLE !== "worker" -// The v2 RunError union has no generic member, so a run failure surfaced across the durable -// boundary is carried as a ContextSnapshotDecodeError with the original text in `details`. Faithful -// per-member reconstruction (Schema round-trip of the exact tagged error) is a further follow-up. -const toRunError = (sessionID: SessionSchema.ID, e: unknown): SessionRunner.RunError => { - // Walk the failure chain (WorkflowUpdateFailedError -> ActivityFailure -> ApplicationFailure) to - // the encoded run error the activity attached, and reconstruct the exact tagged error. - let node: any = e - for (let depth = 0; node && depth < 6; depth++) { - if (Array.isArray(node.details) && node.details.length > 0) { - const decoded = decodeRunError(node.details[0]) - if (decoded) return decoded - return new ContextSnapshotDecodeError({ sessionID, details: `session run failed: ${node.message}` }) - } - node = node.cause - } - return new ContextSnapshotDecodeError({ sessionID, details: `session run failed: ${(e as any)?.message ?? String(e)}` }) -} - /** * A Temporal-backed SessionExecution. It makes each session a durable workflow: * - wake -> signalWithStart(wake) (start while idle, or coalesce into the running run) @@ -81,87 +54,9 @@ const layer = Layer.effect( // SessionRunner and all of its dependencies. const ctx = yield* Effect.context() - const drain = async (input: DrainInput, signal: AbortSignal): Promise => { - const exit = await Effect.runPromiseExit( - Effect.gen(function* () { - const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) - if (!session) return - yield* SessionRunner.Service.use((runner) => - runner.run({ sessionID: session.id, force: input.force }), - ).pipe(Effect.provide(locations.get(session.location))) - }).pipe(Effect.provide(ctx), Effect.scoped), - { signal }, - ) - if (Exit.isSuccess(exit)) return - const cause = exit.cause - if (Cause.hasInterruptsOnly(cause)) { - // Two interrupt sources: Temporal cancellation (the AbortSignal fired -- rethrow its reason - // so the attempt records Cancelled, not Failed) and an internal halt like a user declining a - // permission (the signal did NOT fire). The latter must be non-retryable, or the workflow - // re-drives a turn the user explicitly stopped. - if (signal.aborted) - throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") - throw ApplicationFailure.create({ - message: "session run halted (user declined)", - type: "SessionRunDeclined", - nonRetryable: true, - }) - } - // A genuine run error is thrown non-retryable so Temporal surfaces it (to resume) rather than - // retrying; only crashes / task timeouts (never thrown here) go through the retry policy. The - // error is encoded faithfully in `details` so the caller can reconstruct the exact RunError. - const squashed = Cause.squash(cause) as { _tag?: string; message?: string } - const encoded = encodeRunError(squashed) - throw ApplicationFailure.create({ - message: squashed?.message ?? Cause.pretty(cause), - type: squashed?._tag ?? "SessionRunError", - nonRetryable: true, - details: encoded === undefined ? undefined : [encoded], - }) - } - - // Per-step drain: run exactly one step of the turn (used by temporal-turn mode). Same context - // and error encoding as the whole-turn drain; returns the next loop state to the workflow. - const stepDrain = async (input: StepDrainInput, signal: AbortSignal): Promise => { - const exit = await Effect.runPromiseExit( - Effect.gen(function* () { - const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) - if (!session) return { ran: false, continue: false, step: input.step, promotion: null } - const r = yield* SessionRunner.Service.use((runner) => - runner.runStep({ - sessionID: session.id, - step: input.step, - promotion: (input.promotion ?? undefined) as SessionInput.Delivery | undefined, - first: input.first, - force: input.force, - }), - ).pipe(Effect.provide(locations.get(session.location))) - return { ran: r.ran, continue: r.continue, step: r.step, promotion: r.promotion ?? null } - }).pipe(Effect.provide(ctx), Effect.scoped), - { signal }, - ) - if (Exit.isSuccess(exit)) return exit.value - const cause = exit.cause - if (Cause.hasInterruptsOnly(cause)) { - // Same split as the whole-turn drain: cancellation rethrows its reason (records Cancelled), - // an internal user-decline halt is non-retryable. - if (signal.aborted) - throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") - throw ApplicationFailure.create({ - message: "session run halted (user declined)", - type: "SessionRunDeclined", - nonRetryable: true, - }) - } - const squashed = Cause.squash(cause) as { _tag?: string; message?: string } - const encoded = encodeRunError(squashed) - throw ApplicationFailure.create({ - message: squashed?.message ?? Cause.pretty(cause), - type: squashed?._tag ?? "SessionRunError", - nonRetryable: true, - details: encoded === undefined ? undefined : [encoded], - }) - } + // The drain bodies are shared with the in-process micro-driver (drain.ts), so turn semantics + // and error encoding cannot differ between drivers. + const { drain, stepDrain } = makeDrains({ store, locations, ctx }) // Worker connection (native) hosts the runContinuation activity + the workflow. Skipped in // client-only role so serve can run without an embedded worker. diff --git a/packages/core/src/session/execution/workflow-core.ts b/packages/core/src/session/execution/workflow-core.ts new file mode 100644 index 000000000000..343fb1af5e54 --- /dev/null +++ b/packages/core/src/session/execution/workflow-core.ts @@ -0,0 +1,178 @@ +// The session supervisor, written ONCE, over a six-primitive runtime interface. Two drivers +// execute it: the Temporal workflow adapter (temporal-workflow.ts; the real SDK provides the +// primitives, activities carry the drains) and the in-process micro-driver (local-driver.ts; plain +// promises provide the primitives, the drains run directly). The supervisor cannot drift between +// modes because there is only one of it. +// +// MUST stay pure: the Temporal driver bundles this into the workflow sandbox, so no `effect`, no +// `@opencode-ai/core` runtime imports, no Node builtins. Type-only imports are erased and safe. +// +// Semantics mirror the local coordinator (run-coordinator.ts): drains are serialized (one at a +// time), a `wake` drives a drain and is tolerant of errors, and `resume` (an update) drives a +// forced drain and returns its result to the caller (throwing the run's error). The supervisor +// stays alive to serve later wakes/resumes and terminates after an idle period. + +import type { DrainInput, StepDrainInput, StepDrainResult } from "./temporal-activities" + +/** What a driver must provide. Six primitives; everything else is supervisor logic. */ +export interface WorkflowRuntime { + /** Wait until the predicate is true. With a timeout, resolve false when it expires first. */ + readonly condition: (predicate: () => boolean, timeout?: string) => Promise + readonly setSignalHandler: (name: "wake" | "interrupt", handler: () => void) => void + readonly setUpdateHandler: (name: "resume", handler: () => Promise) => void + /** One whole-turn drain (SessionRunner.run). The Temporal driver runs it as an activity. */ + readonly runContinuation: (input: DrainInput) => Promise + /** One step of a turn (SessionRunner.runStep), for the per-step variant. */ + readonly runTurnStep: (input: StepDrainInput) => Promise + /** Cancel the in-flight drain and any parked condition (interrupt semantics). */ + readonly cancelCurrentScope: () => void + /** Whether an error is the driver's cancellation (a normal stop, not a failure). */ + readonly isCancellation: (error: unknown) => boolean +} + +export interface WorkflowOptions { + /** How long to stay alive with no work before self-terminating. */ + readonly idleTimeout?: string +} + +export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => { + const IDLE_TIMEOUT = options?.idleTimeout ?? "5 minutes" + + async function sessionExecution(sessionID: string): Promise { + let pendingWake = true // started by a wake -> there is work to drain + let stopping = false + let draining = false + let handlers = 0 + + // Serialize drains, like the coordinator (one owner fiber per session at a time). Re-check + // after every wakeup: two waiters parked on the same condition can both observe `!draining` + // in one activation, and without the loop both would start a drain. + const drainOnce = async (force: boolean) => { + for (;;) { + await rt.condition(() => !draining || stopping) + if (stopping) return + if (!draining) break + } + draining = true + try { + await rt.runContinuation({ sessionID, force }) + } finally { + draining = false + } + } + + rt.setSignalHandler("wake", () => { + pendingWake = true + }) + rt.setSignalHandler("interrupt", () => { + stopping = true + rt.cancelCurrentScope() + }) + // resume = coordinator.run: force one drain and surface its result (a run error rejects the + // update, so the caller observes it). + rt.setUpdateHandler("resume", async () => { + handlers++ + try { + await drainOnce(true) + } finally { + handlers-- + } + }) + + // interrupt cancels the whole scope, so a cancellation can surface at the idle wait itself, + // not just inside a drain; treat it as a normal stop rather than a failure. + try { + for (;;) { + const gotWork = await rt.condition(() => pendingWake || stopping, IDLE_TIMEOUT) + if (stopping) return + if (!gotWork) { + // Idle: terminate only when nothing is in flight. A later wake/resume starts a fresh run. + if (!draining && handlers === 0) return + continue + } + pendingWake = false + try { + await drainOnce(false) + } catch (e) { + // wake tolerates run errors (the coordinator logs and moves on); only cancellation stops us. + if (rt.isCancellation(e)) return + } + } + } catch (e) { + if (rt.isCancellation(e)) return + throw e + } + } + + // Per-step variant: identical lifecycle, but a turn is driven one step at a time -- each step + // (one provider attempt + its tools) is its own drain call, and the step loop is supervisor + // control flow. The loop state (step / promotion / first) mirrors SessionRunner.run's loop. + async function sessionTurn(sessionID: string): Promise { + let pendingWake = true + let stopping = false + let draining = false + let handlers = 0 + + const drainTurn = async (force: boolean) => { + // Same re-check loop as drainOnce: a single wakeup must admit a single drain. + for (;;) { + await rt.condition(() => !draining || stopping) + if (stopping) return + if (!draining) break + } + draining = true + try { + let step = 1 + let promotion: string | null = null + let first = true + for (;;) { + const r: StepDrainResult = await rt.runTurnStep({ sessionID, step, promotion, first, force }) + if (!r.continue) break + step = r.step + promotion = r.promotion + first = false + } + } finally { + draining = false + } + } + + rt.setSignalHandler("wake", () => { + pendingWake = true + }) + rt.setSignalHandler("interrupt", () => { + stopping = true + rt.cancelCurrentScope() + }) + rt.setUpdateHandler("resume", async () => { + handlers++ + try { + await drainTurn(true) + } finally { + handlers-- + } + }) + + try { + for (;;) { + const gotWork = await rt.condition(() => pendingWake || stopping, IDLE_TIMEOUT) + if (stopping) return + if (!gotWork) { + if (!draining && handlers === 0) return + continue + } + pendingWake = false + try { + await drainTurn(false) + } catch (e) { + if (rt.isCancellation(e)) return + } + } + } catch (e) { + if (rt.isCancellation(e)) return + throw e + } + } + + return { sessionExecution, sessionTurn } +} diff --git a/packages/core/test/session-execution-local-driver.test.ts b/packages/core/test/session-execution-local-driver.test.ts new file mode 100644 index 000000000000..3c0b7ebf1f98 --- /dev/null +++ b/packages/core/test/session-execution-local-driver.test.ts @@ -0,0 +1,206 @@ +// Contract tests for the in-process micro-driver: the SAME supervisor function the Temporal +// workflow runs (workflow-core.ts), driven with plain promises and no server. The contract is the +// SessionExecution interface: wake drives a turn to settlement, resume surfaces the exact tagged +// RunError (through the same encode/decode path the Temporal boundary uses), interrupt cancels an +// in-flight turn, and an idle supervisor retires itself. +import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" +import { LLMEvent } from "@opencode-ai/llm" +import { Database } from "@opencode-ai/core/database/database" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { EventV2 } from "@opencode-ai/core/event" +import { PermissionV2 } from "@opencode-ai/core/permission" +import { Config } from "@opencode-ai/core/config" +import { Project } from "@opencode-ai/core/project" +import { ProjectTable } from "@opencode-ai/core/project/sql" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { SessionV2 } from "@opencode-ai/core/session" +import { Snapshot } from "@opencode-ai/core/snapshot" +import { SessionEvent } from "@opencode-ai/core/session/event" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import { SessionExecution } from "@opencode-ai/core/session/execution" +import { SessionExecutionLocalDriver } from "@opencode-ai/core/session/execution/local-driver" +import { SessionRunnerModel, ModelNotSelectedError } from "@opencode-ai/core/session/runner/model" +import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionStore } from "@opencode-ai/core/session/store" +import { SessionMessage } from "@opencode-ai/core/session/message" +import { Prompt } from "@opencode-ai/core/session/prompt" +import { Location } from "@opencode-ai/core/location" +import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" +import { SystemContext } from "@opencode-ai/core/system-context" +import { SkillGuidance } from "@opencode-ai/core/skill/guidance" +import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" +import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" +import { Auth } from "@opencode-ai/llm/route" +import { describe, expect } from "bun:test" +import { realpathSync } from "node:fs" +import { tmpdir } from "node:os" +import { Cause, Context, DateTime, Effect, Exit, Layer, Stream } from "effect" +import { testEffect } from "./lib/effect" + +// The per-location service build resolves the session directory on disk, so it must exist. +const WORKSPACE = AbsolutePath.make(realpathSync(tmpdir())) + +const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) + .model({ id: "gpt-4o-mini" }) +const okModels = SessionRunnerModel.layerWith(() => Effect.succeed(model)) +const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) +const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) +const permission = Layer.mock(PermissionV2.Service, {}) + +const mockClient = (stream: LLMClientShape["stream"]) => + Layer.succeed( + LLMClient.Service, + LLMClient.Service.of({ + prepare: () => Effect.die("unused"), + generate: () => Effect.die("unused"), + stream, + }), + ) + +const countingModel = () => { + const requests: number[] = [] + const stream: LLMClientShape["stream"] = () => { + requests.push(1) + return Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })]) + } + return { requests, stream } +} + +// The executor under test, built as its own graph over the shared database file (the same way the +// serve process builds it), with the model/LLM mocked. +const makeExecution = (stream: LLMClientShape["stream"], models = okModels) => + AppNodeBuilder.build(SessionExecutionLocalDriver.node, [ + [LayerNodePlatform.llmClient, mockClient(stream)], + [PermissionV2.node, permission], + [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [SessionRunnerModel.node, models], + [SystemContextRegistry.node, systemContext], + [Location.node, Location.boundNode({ directory: WORKSPACE })], + [SkillGuidance.node, skillGuidance], + [ReferenceGuidance.node, referenceGuidance], + [Config.node, config], + [Snapshot.node, Snapshot.noopLayer], + ]) + +// Reads and seeds go through a separate graph sharing the same database file. +const it = testEffect( + AppNodeBuilder.build(LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node])), +) + +const seedSession = (sessionID: SessionV2.ID) => + Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: WORKSPACE, sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "t", + directory: WORKSPACE, + title: "t", + version: "t", + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + }) + +const seedPrompt = (sessionID: SessionV2.ID) => + Effect.gen(function* () { + const events = yield* EventV2.Service + yield* events.publish(SessionEvent.Prompted, { + sessionID, + timestamp: yield* DateTime.now, + messageID: SessionMessage.ID.create(), + prompt: Prompt.make({ text: "do the thing" }), + delivery: "queue", + }) + }) + +const until = (read: Effect.Effect, predicate: (value: A) => boolean, timeoutMs = 8000) => + Effect.gen(function* () { + const deadline = Date.now() + timeoutMs + for (;;) { + const value = yield* read + if (predicate(value)) return value + if (Date.now() > deadline) throw new Error("condition not reached in time") + yield* Effect.sleep(50) + } + }) + +describe("SessionExecution local micro-driver", () => { + { + const { requests, stream } = countingModel() + const sessionID = SessionV2.ID.make("ses_driver_wake") + it.live("wake drives a turn to settlement, then the idle supervisor retires", () => + Effect.gen(function* () { + process.env.OPENCODE_SESSION_IDLE_TIMEOUT = "2 seconds" + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + const exec = Context.get(yield* Layer.build(makeExecution(stream)), SessionExecution.Service) + yield* exec.wake(sessionID) + const store = yield* SessionStore.Service + yield* until(store.context(sessionID), (context) => { + const assistant = context.findLast((message) => message.type === "assistant") + return assistant?.type === "assistant" && Boolean(assistant.time.completed) + }) + expect(requests).toHaveLength(1) + expect((yield* exec.active).has(sessionID)).toBe(true) + // Idle self-termination: the driver retires without an interrupt. + yield* until(exec.active, (active) => !active.has(sessionID)) + }), + ) + } + + { + const sessionID = SessionV2.ID.make("ses_driver_error") + const failingModels = SessionRunnerModel.layerWith(() => + Effect.fail(new ModelNotSelectedError({ sessionID })), + ) + it.live("resume surfaces the exact tagged RunError through the shared codec", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + const { stream } = countingModel() + const exec = Context.get(yield* Layer.build(makeExecution(stream, failingModels)), SessionExecution.Service) + const exit = yield* exec.resume(sessionID).pipe(Effect.exit) + expect(Exit.isFailure(exit)).toBe(true) + const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined + // The same encode -> details -> decode path the Temporal boundary uses, so the caller gets + // the identical tagged instance in both modes. + expect(error).toBeInstanceOf(ModelNotSelectedError) + }), + ) + } + + { + const sessionID = SessionV2.ID.make("ses_driver_interrupt") + it.live("interrupt cancels an in-flight turn and the driver retires", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + // A model that never answers: the turn hangs until interrupted. + const exec = Context.get( + yield* Layer.build(makeExecution(() => Stream.never)), + SessionExecution.Service, + ) + yield* exec.wake(sessionID) + yield* Effect.sleep(200) + expect((yield* exec.active).has(sessionID)).toBe(true) + yield* exec.interrupt(sessionID) + yield* until(exec.active, (active) => !active.has(sessionID), 4000) + }), + ) + } +}) diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index 3d12c38f5463..9cff7142e9eb 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -10,6 +10,7 @@ import { SessionV2 } from "@opencode-ai/core/session" import { SessionExecution } from "@opencode-ai/core/session/execution" import { LocationServiceMap } from "@opencode-ai/core/location-service-map" import { SessionExecutionLocal } from "@opencode-ai/core/session/execution/local" +import { SessionExecutionLocalDriver } from "@opencode-ai/core/session/execution/local-driver" import { SessionExecutionTemporal } from "@opencode-ai/core/session/execution/temporal" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { HttpRouter, HttpServer } from "effect/unstable/http" @@ -53,11 +54,17 @@ export function createEmbeddedRoutes() { // Shared by the HTTP routes and the standalone worker entrypoint (src/worker.ts) so both build the // exact same context. export function createServiceLayer() { - // Opt in to Temporal-backed durable execution: "temporal" runs one activity per turn, - // "temporal-turn" runs one activity per step. Otherwise the stock in-process coordinator is used. + // The factory: one binding selects how sessions execute. "temporal" runs one activity per turn, + // "temporal-turn" one per step; "local-driver"/"local-driver-turn" run the SAME supervisor + // in-process with no server (workflow-core.ts + local-driver.ts). Otherwise the stock + // in-process coordinator is used. const exec = process.env.OPENCODE_SESSION_EXECUTION const executionNode = - exec === "temporal" || exec === "temporal-turn" ? SessionExecutionTemporal.node : SessionExecutionLocal.node + exec === "temporal" || exec === "temporal-turn" + ? SessionExecutionTemporal.node + : exec === "local-driver" || exec === "local-driver-turn" + ? SessionExecutionLocalDriver.node + : SessionExecutionLocal.node return AppNodeBuilder.build(applicationServices, [[SessionExecution.node, executionNode]]) } From bc9202ae561c09e873151b245611e3674bb7fd0f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 12:46:27 -0700 Subject: [PATCH 039/103] Documented the one-supervisor-two-drivers demonstration and the SDK-surface line. The micro-driver closes the duplicated-supervisor objection; the assessment states where whole-SDK coverage is mechanical (live semantics) and where it becomes embedded Temporal (replay-grade durability of workflow-local state). --- packages/temporal/docs/ai399-local-options.md | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index bf6ec08f3c07..0e3b2303fa41 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -179,6 +179,57 @@ definition, tools, approval policy, event stream) the swap seam with a non-Tempo locally (in-process bus, identical schemas). The ADK integration's `in_workflow()` mechanism is the shipped precedent for how the harness's Temporal-aware pieces could degrade in-process. +## One supervisor, two drivers (demonstrated on this branch) + +The two-implementations objection to the plugin pattern is real: the loop is written once, but the +SUPERVISOR contract (drains serialize, wakes coalesce, interrupt cancels, resume surfaces the +error) existed twice, once as the local coordinator and once as the workflow, and the independent +review found bugs precisely in the duplicated copy. This branch now closes that gap: + +- `workflow-core.ts`: the supervisor, written ONCE, over a six-primitive `WorkflowRuntime` + interface (`condition`, signal handlers, update handlers, the drain calls, cancellation, + `isCancellation`). Temporal's sandbox had already forced it to be pure, which is what makes it + executable anywhere. +- `temporal-workflow.ts`: the Temporal driver; the real SDK provides the six primitives, the + drains run as activities. +- `local-driver.ts` (`OPENCODE_SESSION_EXECUTION=local-driver` / `local-driver-turn`): the + in-process driver; plain promises provide the primitives (a polled `condition`, method-call + signals, an `AbortController` for cancellation), the drains run directly. No server, no worker, + no ports. +- `drain.ts` and the error codec are shared modules, so turn semantics and typed errors are + byte-identical in both modes; contract tests drive the shared supervisor in-process (a turn + settles, the exact tagged `RunError` crosses the same encode/decode path, interrupt cancels, an + idle supervisor retires), and the worker smoke proves the same file still bundles in the + Temporal sandbox. + +The factory still hides the choice: one binding selects the driver. This is the factory-shaped +answer to the ADK integration's ambient check, demonstrated. + +## Can the whole SDK surface be covered this way? + +The micro-driver needed six primitives because the supervisor uses six. Extending it across +`@temporalio/workflow`'s API splits cleanly: + +- **Mechanical for live execution:** `sleep`/timers (`setTimeout`), `workflow.now` (`Date.now`), + `random`/`uuid4` (plain random: determinism only matters for replay, which a local driver never + does), queries (read a handler map), `patched`/`deprecatePatch` (constant true / no-op), + `sideEffect` (run the function), search attributes and memo (a local map), logging sinks + (console), child workflows and external handles (spawn sibling drivers, route signals through an + in-process registry), continue-as-new (re-invoke the function with the new arguments). +- **The fundamental line is replay.** Temporal recovers workflow-VARIABLE state after a crash by + replaying history; a local driver has no history, so in-flight workflow variables and pending + timers die with the process. Covering that is not a shim, it is the embedded-service payload of + path B. + +Two consequences. First, the design rule for dual-mode apps: keep durable truth in an app-owned +log and treat workflow variables as ephemeral. This branch already obeys it (supervisor state is +reconstructible; turn state is in the event store; re-drives are log-based), which is why the +local driver loses nothing that matters on a desktop crash, and it is the rule to hand any +customer taking this path. Second, the honest scope statement: a whole-SDK local runtime is +achievable for LIVE semantics as a bounded engineering effort, but replay-grade durability of +workflow-local state is exactly where "cover the SDK surface" becomes "build embedded Temporal", +and should be decided as that (path B), not approached incrementally by accident. + ## What the local mode actually has to support: the control surface Measured from opencode's protocol (the session group plus human-in-the-loop groups), the session From d44de7002c346528be4881692839da468bb0a9a3 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 12:55:05 -0700 Subject: [PATCH 040/103] Collapsed the factory to two modes: local and temporal. With one supervisor and shared drains, per-turn vs per-step stopped being a user-facing choice and the stock coordinator stopped being the local story. `OPENCODE_SESSION_EXECUTION=temporal` runs the supervisor on a Temporal worker; anything else runs the same supervisor in-process with the micro-driver. Both drive the turn per step. The whole-turn workflow stays exported for executions already running, and the stock coordinator stays in the tree as upstream code. --- .../src/session/execution/local-driver.ts | 11 +++----- .../session/execution/temporal-activities.ts | 2 +- .../core/src/session/execution/temporal.ts | 9 +++---- .../core/test/session-runner-resume.test.ts | 2 +- packages/server/src/routes.ts | 21 ++++++---------- packages/server/src/worker.ts | 2 +- packages/temporal/README.md | 25 +++++++++++-------- packages/temporal/docs/ai399-local-options.md | 18 +++++++------ .../scripts/standalone-worker-smoke.sh | 2 +- 9 files changed, 43 insertions(+), 49 deletions(-) diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts index db2a23cba621..260e25fbc593 100644 --- a/packages/core/src/session/execution/local-driver.ts +++ b/packages/core/src/session/execution/local-driver.ts @@ -156,11 +156,8 @@ const layer = Layer.effect( const ctx = yield* Effect.context() const drains = makeDrains({ store, locations, ctx }) const drivers = new Map() - // Read at layer build (not module load) so tests can set these before constructing the layer. - // local-driver = whole-turn drains; local-driver-turn = one drain call per step, mirroring the - // temporal / temporal-turn split so both drivers exercise both supervisors. The idle override - // shortens the supervisor's 5-minute self-termination. - const PER_STEP = process.env.OPENCODE_SESSION_EXECUTION === "local-driver-turn" + // Read at layer build (not module load) so tests can set it before constructing the layer. + // The idle override shortens the supervisor's 5-minute self-termination. const IDLE_TIMEOUT = process.env.OPENCODE_SESSION_IDLE_TIMEOUT const ensure = (id: SessionSchema.ID): SessionDriver => { @@ -169,7 +166,7 @@ const layer = Layer.effect( const driver = new SessionDriver( (rt) => { const workflows = makeWorkflows(rt, IDLE_TIMEOUT ? { idleTimeout: IDLE_TIMEOUT } : undefined) - return (PER_STEP ? workflows.sessionTurn : workflows.sessionExecution)(id) + return workflows.sessionTurn(id) }, drains, () => { @@ -191,7 +188,7 @@ const layer = Layer.effect( ) yield* Effect.logInfo("SessionExecutionLocalDriver ready").pipe( - Effect.annotateLogs({ supervisor: PER_STEP ? "sessionTurn" : "sessionExecution" }), + Effect.annotateLogs({ supervisor: "sessionTurn" }), ) return SessionExecution.Service.of({ diff --git a/packages/core/src/session/execution/temporal-activities.ts b/packages/core/src/session/execution/temporal-activities.ts index cd7e81470a50..060ea6620591 100644 --- a/packages/core/src/session/execution/temporal-activities.ts +++ b/packages/core/src/session/execution/temporal-activities.ts @@ -35,7 +35,7 @@ export function makeActivities( } } -// Per-step variant (OPENCODE_SESSION_EXECUTION=temporal-turn): one runTurnStep activity = one step +// Per-step activities: one runTurnStep activity = one step // (one provider attempt + its tools). The workflow loops it, so each step is its own activity with // its own retry/timeout/visibility. `promotion` is null (not undefined) so it serializes cleanly. export interface StepDrainInput { diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 0b2ac3042a3c..de600b7617d2 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -20,11 +20,10 @@ const NAMESPACE = process.env.TEMPORAL_NAMESPACE ?? "default" const TASK_QUEUE = process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? "opencode-session-exec" const workflowId = (id: string) => `session-exec-${id}` -// temporal = one activity per turn; temporal-turn = one activity per step (the model call + its -// tools), with the step loop as workflow control flow. -const PER_STEP = process.env.OPENCODE_SESSION_EXECUTION === "temporal-turn" -const WORKFLOW = PER_STEP ? WF.sessionTurn : WF.sessionExecution -const WORKFLOW_TYPE = PER_STEP ? "sessionTurn" : "sessionExecution" +// One activity per step (the model call + its tools), with the step loop as workflow control +// flow. The whole-turn sessionExecution workflow stays exported for workflows already running. +const WORKFLOW = WF.sessionTurn +const WORKFLOW_TYPE = "sessionTurn" // Role split so the worker fleet can run separately from the HTTP server. `both` (default) hosts the // activity worker AND the workflow client in one process (the serve process). `client` makes serve diff --git a/packages/core/test/session-runner-resume.test.ts b/packages/core/test/session-runner-resume.test.ts index ac98196278f6..dfe258afdcf8 100644 --- a/packages/core/test/session-runner-resume.test.ts +++ b/packages/core/test/session-runner-resume.test.ts @@ -1,4 +1,4 @@ -// Resumability of a per-step turn (temporal-turn): a Temporal step retry re-invokes runStep on the +// Resumability of a per-step turn (temporal mode): a Temporal step retry re-invokes runStep on the // same durable log. These tests seed a crashed in-flight step (Step.Started + tool events, no // Step.Ended) and drive runStep to check the two recovery behaviors: // - Slice 1: a dangling tool left by an interrupted attempt is closed on every step entry, not diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index 9cff7142e9eb..d905f9e2d4ec 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -9,7 +9,6 @@ import { PtyTicket } from "@opencode-ai/core/pty/ticket" import { SessionV2 } from "@opencode-ai/core/session" import { SessionExecution } from "@opencode-ai/core/session/execution" import { LocationServiceMap } from "@opencode-ai/core/location-service-map" -import { SessionExecutionLocal } from "@opencode-ai/core/session/execution/local" import { SessionExecutionLocalDriver } from "@opencode-ai/core/session/execution/local-driver" import { SessionExecutionTemporal } from "@opencode-ai/core/session/execution/temporal" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" @@ -54,17 +53,13 @@ export function createEmbeddedRoutes() { // Shared by the HTTP routes and the standalone worker entrypoint (src/worker.ts) so both build the // exact same context. export function createServiceLayer() { - // The factory: one binding selects how sessions execute. "temporal" runs one activity per turn, - // "temporal-turn" one per step; "local-driver"/"local-driver-turn" run the SAME supervisor - // in-process with no server (workflow-core.ts + local-driver.ts). Otherwise the stock - // in-process coordinator is used. - const exec = process.env.OPENCODE_SESSION_EXECUTION + // The factory: two modes, one supervisor (workflow-core.ts). "temporal" runs it on a Temporal + // worker (one activity per step); anything else runs it in-process with no server + // (local-driver.ts). The loop, the drains, and the error codec are the same code either way. const executionNode = - exec === "temporal" || exec === "temporal-turn" + process.env.OPENCODE_SESSION_EXECUTION === "temporal" ? SessionExecutionTemporal.node - : exec === "local-driver" || exec === "local-driver-turn" - ? SessionExecutionLocalDriver.node - : SessionExecutionLocal.node + : SessionExecutionLocalDriver.node return AppNodeBuilder.build(applicationServices, [[SessionExecution.node, executionNode]]) } @@ -72,11 +67,9 @@ export function createServiceLayer() { // SessionExecution as a built member so constructing the layer eagerly starts the Temporal worker // (with OPENCODE_TEMPORAL_ROLE=worker there is no HTTP handler to pull it in lazily). export function createWorkerLayer() { - const exec = process.env.OPENCODE_SESSION_EXECUTION - const executionNode = - exec === "temporal" || exec === "temporal-turn" ? SessionExecutionTemporal.node : SessionExecutionLocal.node + // A standalone worker only makes sense in temporal mode. return AppNodeBuilder.build(LayerNode.group([applicationServices, SessionExecution.node]), [ - [SessionExecution.node, executionNode], + [SessionExecution.node, SessionExecutionTemporal.node], ]) } diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts index 7cc8ef048e37..fd61b200cfe2 100644 --- a/packages/server/src/worker.ts +++ b/packages/server/src/worker.ts @@ -3,7 +3,7 @@ // forcing SessionExecution constructs the Temporal worker, which then polls the task queue and runs // session continuations. Run one or many of these next to (or instead of) an embedded worker. // -// OPENCODE_TEMPORAL_ROLE=worker OPENCODE_SESSION_EXECUTION=temporal-turn \ +// OPENCODE_TEMPORAL_ROLE=worker OPENCODE_SESSION_EXECUTION=temporal \ // TEMPORAL_ADDRESS=127.0.0.1:7237 OPENCODE_DB_URL=... \ // bun run packages/server/src/worker.ts // diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 0d3548380c82..823dbd0e115f 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -107,15 +107,18 @@ session runs as a Temporal workflow `session-exec-`. embedded worker) mid-turn, then restarting, still completes the turn. Temporal re-drives `runContinuation` (attempt 2), the run continues from the event log, and the workflow completes. -### Per-step turns (`temporal-turn`) - -`OPENCODE_SESSION_EXECUTION=temporal` runs a whole turn in one `runContinuation` activity. -`OPENCODE_SESSION_EXECUTION=temporal-turn` instead drives the turn one **step** at a time: the -`sessionTurn` workflow loops a `runTurnStep` activity, so each step (one provider attempt + its -tools) is its own activity with its own retry/timeout/visibility, and the step loop is workflow -control flow. It reuses `SessionRunner.runStep` (one iteration of `run`'s loop), so the turn -semantics are unchanged. Verified: a create-then-read-then-reply turn recorded three `runTurnStep` -activities under a `sessionTurn` workflow and completed. +### Two modes, one supervisor + +The factory has exactly two modes. `OPENCODE_SESSION_EXECUTION=temporal` runs the session +supervisor on a Temporal worker; anything else (the default) runs the SAME supervisor in-process +with the micro-driver (`workflow-core.ts` + `local-driver.ts`): no server, no worker, no ports, +durability from the event log. Both modes drive the turn one **step** at a time: the `sessionTurn` +supervisor loops a `runTurnStep` drain, so in temporal mode each step (one provider attempt + its +tools) is its own activity with its own retry/timeout/visibility. It reuses `SessionRunner.runStep` +(one iteration of `run`'s loop), so the turn semantics are unchanged. Verified: a +create-then-read-then-reply turn recorded three `runTurnStep` activities under a `sessionTurn` +workflow and completed. (Earlier whole-turn-per-activity and stock-coordinator modes were folded +away; the whole-turn workflow stays exported for executions already running.) A per-step re-drive resumes from the durable event log rather than re-running work. `runStep` closes any tool left dangling by an interrupted attempt on every entry, not just the first. Without that, a @@ -160,10 +163,10 @@ workers and point serve at client-only: ```bash # serve drives workflows, hosts no worker -OPENCODE_TEMPORAL_ROLE=client OPENCODE_SESSION_EXECUTION=temporal-turn ... serve --port 4601 +OPENCODE_TEMPORAL_ROLE=client OPENCODE_SESSION_EXECUTION=temporal ... serve --port 4601 # one or more standalone activity workers (no HTTP surface) -OPENCODE_TEMPORAL_ROLE=worker OPENCODE_SESSION_EXECUTION=temporal-turn \ +OPENCODE_TEMPORAL_ROLE=worker OPENCODE_SESSION_EXECUTION=temporal \ TEMPORAL_ADDRESS=127.0.0.1:7237 OPENCODE_DB_URL=... \ bun run packages/server/src/worker.ts ``` diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index 0e3b2303fa41..440c7186b33f 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -77,7 +77,7 @@ substrate, a local runner is another. **Evidence 1: this fork.** opencode's v2 engine event-sources every session to a store and exposes a substitutable `SessionExecution` interface (`active`/`wake`/`resume`/`interrupt`, four methods). The stock implementation is an in-process coordinator; this fork adds a Temporal-backed one -(`OPENCODE_SESSION_EXECUTION=temporal` per turn, `temporal-turn` per step) plus a shared store so +(`OPENCODE_SESSION_EXECUTION=temporal`, one activity per step) plus a shared store so any worker resumes any session. Because durability lives in the engine's event log, the LOCAL mode is already crash-recoverable without Temporal; Temporal adds supervised retries, worker distribution, restart-surviving visibility, and cross-process interrupt/resume. The swap point is @@ -152,7 +152,7 @@ machine's dev build): | License | MIT (verified in release tarball and via GitHub API) | Marginal cost measured on THIS app (same engine, same machine): stock local serve 297 MB RSS; -`temporal-turn` serve 494 MB (embedded worker) + dev server 123 MB = ~617 MB across two processes, +temporal-mode serve 494 MB (embedded worker) + dev server 123 MB = ~617 MB across two processes, plus the binary on disk, for full Temporal semantics on identical code. Measured to quantify the alternative, not to propose it here: this fork's intended desktop shape is the in-process local mode (path C), which is what we would suggest to the desktop customer. A bundled server is the @@ -192,18 +192,20 @@ review found bugs precisely in the duplicated copy. This branch now closes that executable anywhere. - `temporal-workflow.ts`: the Temporal driver; the real SDK provides the six primitives, the drains run as activities. -- `local-driver.ts` (`OPENCODE_SESSION_EXECUTION=local-driver` / `local-driver-turn`): the - in-process driver; plain promises provide the primitives (a polled `condition`, method-call - signals, an `AbortController` for cancellation), the drains run directly. No server, no worker, - no ports. +- `local-driver.ts` (the default mode; `OPENCODE_SESSION_EXECUTION=temporal` selects the Temporal + driver instead): the in-process driver; plain promises provide the primitives (a polled + `condition`, method-call signals, an `AbortController` for cancellation), the drains run + directly. No server, no worker, no ports. - `drain.ts` and the error codec are shared modules, so turn semantics and typed errors are byte-identical in both modes; contract tests drive the shared supervisor in-process (a turn settles, the exact tagged `RunError` crosses the same encode/decode path, interrupt cancels, an idle supervisor retires), and the worker smoke proves the same file still bundles in the Temporal sandbox. -The factory still hides the choice: one binding selects the driver. This is the factory-shaped -answer to the ADK integration's ambient check, demonstrated. +The factory still hides the choice, and it is now exactly two modes: the in-process driver by +default, `temporal` for the server-backed one. The earlier whole-turn and stock-coordinator modes +were folded away once the shared supervisor made them redundant. This is the factory-shaped answer +to the ADK integration's ambient check, demonstrated. ## Can the whole SDK surface be covered this way? diff --git a/packages/temporal/scripts/standalone-worker-smoke.sh b/packages/temporal/scripts/standalone-worker-smoke.sh index 1d178d774da1..3148504b29f8 100755 --- a/packages/temporal/scripts/standalone-worker-smoke.sh +++ b/packages/temporal/scripts/standalone-worker-smoke.sh @@ -34,7 +34,7 @@ done echo "starting standalone worker (role=worker, no serve)" OPENCODE_TEMPORAL_ROLE=worker \ - OPENCODE_SESSION_EXECUTION=temporal-turn \ + OPENCODE_SESSION_EXECUTION=temporal \ TEMPORAL_ADDRESS="127.0.0.1:$PORT" \ OPENCODE_DB="$DB" \ bun run "$REPO/packages/server/src/worker.ts" >"$WLOG" 2>&1 & From e0e586e1194a33d551aa4e98bff7582cb0dd5205 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 13:00:14 -0700 Subject: [PATCH 041/103] Dropped the bundled-server pairing from the measurements. Nobody ships temporal mode plus a dev server on a desktop; the pairing existed only to close that question. The desktop number stays (297 MB, one process); the standalone dev-server figures stay as the appliance numbers. --- packages/temporal/docs/ai399-local-options.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index 440c7186b33f..722d25ae3f13 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -151,12 +151,10 @@ machine's dev build): | Embedding-friendly flags | `--headless`, `--db-filename`, `--port`, `--ip`, repeatable `--namespace`, `--sqlite-pragma`, `--dynamic-config-value`; http/metrics ports default to random free | | License | MIT (verified in release tarball and via GitHub API) | -Marginal cost measured on THIS app (same engine, same machine): stock local serve 297 MB RSS; -temporal-mode serve 494 MB (embedded worker) + dev server 123 MB = ~617 MB across two processes, -plus the binary on disk, for full Temporal semantics on identical code. Measured to quantify the -alternative, not to propose it here: this fork's intended desktop shape is the in-process local -mode (path C), which is what we would suggest to the desktop customer. A bundled server is the -appliance answer, not the desktop one. +On this app the desktop number is 297 MB RSS, one process, local mode. A bundled server was +measured once to close the desktop question and is not a configuration anyone ships: desktop is +local mode, and temporal mode's server is Cloud or a fleet. The standalone figures above are the +appliance-archetype numbers. - **Fidelity is the differentiator: it is the real server against SQLite, not an emulator.** Research verified multi-namespace and Nexus endpoints work and persist. Signals, updates, @@ -285,7 +283,7 @@ The ask is real, old, and largely unanswered publicly: | | A: language shim | B: rust-core shim | C: plugin pattern (incl. ADK-style fallback) | D: local dev server | |---|---|---|---|---| | Durability (crash mid-turn) | partial: what the shim persists; faithful replay = reimplementing Temporal | full IF built (it IS an embedded service) | partial: checkpoint-file coarse; event-sourced local store near-full for one machine | full (with `--db-filename`; default is in-memory) | -| Resource needs | lightest (in-process) | in-process; core carries matching+history | lightest (in-process; store is a file) | ~102-139 MB RSS second process, ~128 MiB disk; measured here ~617 MB total for this app | +| Resource needs | lightest (in-process) | in-process; core carries matching+history | lightest (in-process; store is a file) | ~102-139 MB RSS second process, ~128 MiB disk (appliance archetype; not a desktop configuration) | | Signal/Update support | re-implemented, drift-prone | full IF built | the app's control surface (~a dozen verbs), honored identically by both modes | full | | Complexity / maintainability | high, permanent (chases SDK surface per language); bounded if scoped to the control surface | highest; a second Temporal service implementation (Java test server: years, still short of parity) | lowest sustained; up-front loop design, or shipped by Temporal (ADK) | near-zero code; packaging+lifecycle burden | | Feature fidelity | subset, hand-built | full IF built | app semantics, not Temporal's | full: the real server (multi-namespace, Nexus verified) | From 2f10a04391f2aeb5e44247b7ee2d9a8d230fe416 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 16:11:08 -0700 Subject: [PATCH 042/103] Replaced raw control bytes in source with escape sequences. loop-guard.ts carried literal 0x00/0x01 separator bytes, so git flagged the file as binary: no diffs, no blame. permission.ts had a literal NUL inside the ask-id join. Same runtime behavior via unicode string escapes; both files are plain text again. --- packages/core/src/permission.ts | 2 +- .../core/src/session/runner/loop-guard.ts | Bin 2842 -> 2852 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/permission.ts b/packages/core/src/permission.ts index cec6feb9ace4..6f1f62528db2 100644 --- a/packages/core/src/permission.ts +++ b/packages/core/src/permission.ts @@ -216,7 +216,7 @@ const layer = Layer.effect( function deterministicID(input: AssertInput) { if (!input.source?.callID) return undefined const digest = createHash("sha256") - .update([input.sessionID, input.source.callID, input.action, ...[...input.resources].sort()].join("")) + .update([input.sessionID, input.source.callID, input.action, ...[...input.resources].sort()].join("\u0000")) .digest("hex") return ID.create(`per_${digest.slice(0, 26)}`) } diff --git a/packages/core/src/session/runner/loop-guard.ts b/packages/core/src/session/runner/loop-guard.ts index 1df0edc2640b02a8f22c04879f07f61b2abc30b4..56c777c1be274e14fd4dde9629291ec6beabc11b 100644 GIT binary patch delta 30 gcmbOwwnS{hE;hE9QUf5Eyqm2W!Zh4$&i7E;dGn$p_e~85uV_vOi`907+y9lK=n! From daa74d5c0ce691078a124bb647305381f53a9d9e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 16:13:40 -0700 Subject: [PATCH 043/103] Guarded permission status transitions; adopted same-process retries. Status writes were unconditional, so a graceful shutdown racing a remote approval flipped an approved row to expired and failed the waiter with a decline the user never made. Transitions are now compare-and-set on the expected status, and the shutdown finalizer reads the row first and honors a landed outcome. A retry that lands in the same process while the prior attempt's waiter is still parked now shares that waiter; it used to die on the duplicate id, which the drain turned into a non-retryable failure. --- packages/core/src/permission.ts | 43 ++++++++++----- packages/core/test/permission-durable.test.ts | 55 ++++++++++++++++++- 2 files changed, 83 insertions(+), 15 deletions(-) diff --git a/packages/core/src/permission.ts b/packages/core/src/permission.ts index 6f1f62528db2..5b8b4a47e75c 100644 --- a/packages/core/src/permission.ts +++ b/packages/core/src/permission.ts @@ -146,11 +146,13 @@ const layer = Layer.effect( ) .all() .pipe(EffectRuntime.orDie) - const updateRow = (id: string, status: string, message?: string) => + // Status moves are compare-and-set on the stated `from`, so a reply that lost a race, or a + // shutdown racing an approval, cannot overwrite a landed outcome. + const transitionRow = (id: string, from: string, to: string, message?: string) => db .update(PermissionRequestTable) - .set({ status, message: message ?? null }) - .where(eq(PermissionRequestTable.id, id)) + .set({ status: to, message: message ?? null }) + .where(and(eq(PermissionRequestTable.id, id), eq(PermissionRequestTable.status, from))) .run() .pipe(EffectRuntime.orDie) const decodeRow = (row: { payload: string }) => @@ -160,12 +162,22 @@ const layer = Layer.effect( EffectRuntime.forEach( pending.values(), (item) => - // Graceful shutdown: unblock the local waiter and retire the row. The waiting fiber dies - // with this process either way, so a later reply would have nothing to resume. - Deferred.fail(item.deferred, new DeclinedError()).pipe( - EffectRuntime.andThen(updateRow(item.request.id, "expired")), - EffectRuntime.catch(() => EffectRuntime.void), - ), + // Graceful shutdown: a reply may have landed on the row before the local poll saw it. + // Honor it, or the shutdown records a decline the user never made. Only a row that is + // still pending gets retired. + EffectRuntime.gen(function* () { + const row = yield* readRow(item.request.id) + if (row?.status === "approved") { + yield* Deferred.succeed(item.deferred, undefined) + return + } + if (row?.status === "corrected") { + yield* Deferred.fail(item.deferred, new CorrectedError({ feedback: row.message ?? "" })) + return + } + yield* transitionRow(item.request.id, "pending", "expired") + yield* Deferred.fail(item.deferred, new DeclinedError()) + }).pipe(EffectRuntime.catch(() => EffectRuntime.void)), { discard: true }, ).pipe( EffectRuntime.ensuring( @@ -236,9 +248,12 @@ const layer = Layer.effect( const create = (request: Request, agent?: AgentV2.ID) => EffectRuntime.uninterruptible( EffectRuntime.gen(function* () { + // A retry can land in this process while the prior attempt's waiter is still parked + // (deterministic ids make them the same ask). Share the waiter instead of dying. + const parked = pending.get(request.id) + if (parked) return parked const deferred = yield* Deferred.make() const item = { request, agent, deferred } - if (pending.has(request.id)) return yield* EffectRuntime.die(`Duplicate pending permission ID: ${request.id}`) pending.set(request.id, item) yield* db .insert(PermissionRequestTable) @@ -355,7 +370,7 @@ const layer = Layer.effect( }) if (input.reply === "reject") { - yield* updateRow(existing.id, input.message ? "corrected" : "declined", input.message) + yield* transitionRow(existing.id, "pending", input.message ? "corrected" : "declined", input.message) yield* settleLocal(existing.id, (deferred) => Deferred.fail( deferred, @@ -370,7 +385,7 @@ const layer = Layer.effect( requestID: ID.make(other.id), reply: "reject", }) - yield* updateRow(other.id, "declined") + yield* transitionRow(other.id, "pending", "declined") yield* settleLocal(ID.make(other.id), (deferred) => Deferred.fail(deferred, new DeclinedError())) } return @@ -383,7 +398,7 @@ const layer = Layer.effect( resources: existing.save, }) } - yield* updateRow(existing.id, "approved") + yield* transitionRow(existing.id, "pending", "approved") yield* settleLocal(existing.id, (deferred) => Deferred.succeed(deferred, undefined)) if (input.reply !== "always" || !existing.save?.length) return @@ -406,7 +421,7 @@ const layer = Layer.effect( requestID: other.id, reply: "always", }) - yield* updateRow(other.id, "approved") + yield* transitionRow(other.id, "pending", "approved") yield* settleLocal(other.id, (deferred) => Deferred.succeed(deferred, undefined)) } }), diff --git a/packages/core/test/permission-durable.test.ts b/packages/core/test/permission-durable.test.ts index cf4f84da4d7f..d106dee1b08e 100644 --- a/packages/core/test/permission-durable.test.ts +++ b/packages/core/test/permission-durable.test.ts @@ -4,7 +4,7 @@ // independent service stacks share one DB file to simulate the two processes. import { describe, expect } from "bun:test" import path from "path" -import { Cause, Context, Effect, Exit, Fiber, Layer } from "effect" +import { Cause, Context, Effect, Exit, Fiber, Layer, Scope } from "effect" import { AgentV2 } from "@opencode-ai/core/agent" import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" import { Database } from "@opencode-ai/core/database/database" @@ -184,6 +184,59 @@ describe("PermissionV2 durable asks", () => { }), ) + it.live("shutdown honors an approval that landed before the poll saw it", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + yield* seed(file) + const B = yield* Layer.build(stack(file)) + const permB = Context.get(B, PermissionV2.Service) + // A lives in its own scope so the test can shut it down while the waiter is parked. + const scope = yield* Scope.make() + const A = yield* Layer.build(stack(file)).pipe(Effect.provideService(Scope.Scope, scope)) + const permA = Context.get(A, PermissionV2.Service) + const blocked = yield* permA + .assert({ sessionID, action: "bash", resources: ["echo hi"], agent }) + .pipe(Effect.forkChild) + const ask = yield* awaitAsk(permB) + yield* permB.reply({ requestID: ask.id, reply: "once" }) + // Shut A down inside the poll window: the finalizer must honor the approval on the row, + // not record a decline the user never made. + yield* Scope.close(scope, Exit.void) + const exit = yield* Fiber.await(blocked) + expect(Exit.isSuccess(exit)).toBe(true) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + it.live("a same-process retry shares the parked waiter instead of dying", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + yield* seed(file) + const A = yield* Layer.build(stack(file)) + const B = yield* Layer.build(stack(file)) + const permA = Context.get(A, PermissionV2.Service) + const permB = Context.get(B, PermissionV2.Service) + const input = { + sessionID, + action: "bash", + resources: ["make it"], + agent, + source: { type: "tool" as const, messageID: "msg_3", callID: "call_shared" }, + } + const first = yield* permA.assert(input).pipe(Effect.forkChild) + yield* Effect.sleep(100) + const second = yield* permA.assert(input).pipe(Effect.forkChild) + const ask = yield* awaitAsk(permB) + expect(yield* permB.list()).toHaveLength(1) + yield* permB.reply({ requestID: ask.id, reply: "once" }) + const exits = [yield* Fiber.await(first), yield* Fiber.await(second)] + expect(exits.every(Exit.isSuccess)).toBe(true) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + it.live("expires locally-pending asks on shutdown so they do not linger as pending rows", () => Effect.gen(function* () { const tmp = yield* Effect.promise(() => tmpdir()) From 0edd1a579ae816bc7fb41f63f360a9f3afbe9909 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 16:15:17 -0700 Subject: [PATCH 044/103] Retried busy errors on the shared store; let reads skip the permit. The remote path has no busy_timeout, so a cross-process write-lock conflict (serve inserting while a worker holds a turn transaction) died the caller through orDie. Auto-commit statements and the write-transaction open now retry with bounded backoff on busy-classified errors. Reads bypass the process-wide permit: WAL readers do not take the write lock, and a permission poll or an active() read must not queue behind an open turn transaction. --- packages/core/src/database/sqlite.libsql.ts | 43 +++++++++++++++++---- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/packages/core/src/database/sqlite.libsql.ts b/packages/core/src/database/sqlite.libsql.ts index a4543131b77a..38173de6ee2a 100644 --- a/packages/core/src/database/sqlite.libsql.ts +++ b/packages/core/src/database/sqlite.libsql.ts @@ -56,16 +56,38 @@ const make = (options: LibsqlConfig) => reason: classifySqliteError(cause, { message: "Failed to execute statement", operation: "execute" }), }) + // sqld surfaces a cross-process write-lock conflict as a busy error instead of queueing, and + // the remote path has no `busy_timeout` pragma. Without a bounded retry, a routine collision + // (serve writing while a worker holds a turn transaction) dies the caller. + const isBusy = (cause: unknown) => + /SQLITE_BUSY|database is locked|database table is locked/i.test(String(cause)) + const withBusyRetry = async (attempt: () => Promise): Promise => { + for (let tries = 0; ; tries++) { + try { + return await attempt() + } catch (cause) { + if (tries >= 5 || !isBusy(cause)) throw cause + const delay = Math.min(50 * 2 ** tries, 500) * (0.5 + Math.random()) + await new Promise((resolve) => setTimeout(resolve, delay)) + } + } + } + const toRows = (r: ResultSet) => r.rows as unknown as Array> const toValues = (r: ResultSet) => r.rows.map((row) => r.columns.map((c) => (row as Record)[c])) as Array // Auto-commit connection: each statement is its own request. Used outside transactions. const run = (query: string, params: ReadonlyArray = []) => - Effect.tryPromise({ try: () => native.execute({ sql: query, args: params as never[] }).then(toRows), catch: fail }) + Effect.tryPromise({ + try: () => + withBusyRetry(() => native.execute({ sql: query, args: params as never[] })).then(toRows), + catch: fail, + }) const runValues = (query: string, params: ReadonlyArray = []) => Effect.tryPromise({ - try: () => native.execute({ sql: query, args: params as never[] }).then(toValues), + try: () => + withBusyRetry(() => native.execute({ sql: query, args: params as never[] })).then(toValues), catch: fail, }) @@ -95,7 +117,8 @@ const make = (options: LibsqlConfig) => let tx: Transaction | null = null const exec = async (query: string, params: ReadonlyArray): Promise => { if (BEGIN.test(query)) { - tx = await native.transaction("write") + // Opening the write transaction is where the cross-process lock conflict lands. + tx = await withBusyRetry(() => native.transaction("write")) return EMPTY } if (COMMIT.test(query)) { @@ -158,13 +181,19 @@ const make = (options: LibsqlConfig) => // auto-commit write racing an open pinned transaction would get SQLITE_BUSY from the server // (the remote path has no busy_timeout) and die the caller. Statements inside a transaction // scope route to the transaction's own connection, so this never self-deadlocks. + // Reads never take the write lock (WAL), so they skip the permit: a permission poll or an + // `active` read must not queue behind an open turn transaction. Busy retry is the backstop if + // a server still reports a conflict. + const isRead = (query: string) => /^\s*(select|pragma)\b/i.test(query) + const guard = (query: string, effect: Effect.Effect) => + isRead(query) ? effect : semaphore.withPermits(1)(effect) const guarded = identity({ execute: (query, params, transformRows) => - semaphore.withPermits(1)(connection.execute(query, params, transformRows)), - executeRaw: (query, params) => semaphore.withPermits(1)(connection.executeRaw(query, params)), - executeValues: (query, params) => semaphore.withPermits(1)(connection.executeValues(query, params)), + guard(query, connection.execute(query, params, transformRows)), + executeRaw: (query, params) => guard(query, connection.executeRaw(query, params)), + executeValues: (query, params) => guard(query, connection.executeValues(query, params)), executeUnprepared: (query, params, transformRows) => - semaphore.withPermits(1)(connection.executeUnprepared(query, params, transformRows)), + guard(query, connection.executeUnprepared(query, params, transformRows)), executeStream() { return Stream.die("executeStream not implemented") }, From 52c43722b93341b3751bd4e38bcba434fc3b3117 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 16:17:17 -0700 Subject: [PATCH 045/103] Bounded workflow history and closed two idle-boundary wake losses. A continuously busy session never hits the idle return, so its history grew until Temporal's 50k-event termination. The supervisor now continues-as-new after a bounded number of drains when the driver keeps history; a fresh run starts with a pending wake, so no work is lost. Two lost-wake windows at the idle boundary are closed: the loop re-checks pendingWake when the idle timer fires, and the local driver retries a wake that lands on a supervisor that returned but has not finished retiring. --- .../src/session/execution/local-driver.ts | 8 +++++++- .../session/execution/temporal-workflow.ts | 2 ++ .../src/session/execution/workflow-core.ts | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts index 260e25fbc593..194a574191c4 100644 --- a/packages/core/src/session/execution/local-driver.ts +++ b/packages/core/src/session/execution/local-driver.ts @@ -195,7 +195,13 @@ const layer = Layer.effect( active: Effect.sync(() => new Set(drivers.keys())), wake: (id) => Effect.sync(() => { - ensure(id).signal("wake") + // A wake can land between the supervisor's return and its finally; retry onto a fresh + // driver so the prompt is not stranded until the next wake. + for (let tries = 0; tries < 3; tries++) { + const driver = ensure(id) + driver.signal("wake") + if (!driver.completed) return + } }), resume: (id) => Effect.tryPromise({ diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index 100f40124082..7746c80b13a8 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -13,6 +13,7 @@ import { defineUpdate, setHandler, condition, + continueAsNew, CancellationScope, isCancellation, } from "@temporalio/workflow" @@ -54,6 +55,7 @@ const runtime: WorkflowRuntime = { runTurnStep, cancelCurrentScope: () => CancellationScope.current().cancel(), isCancellation, + continueAsNew: (sessionID) => continueAsNew<(id: string) => Promise>(sessionID), } const workflows = makeWorkflows(runtime) diff --git a/packages/core/src/session/execution/workflow-core.ts b/packages/core/src/session/execution/workflow-core.ts index 343fb1af5e54..c19ba42a2e86 100644 --- a/packages/core/src/session/execution/workflow-core.ts +++ b/packages/core/src/session/execution/workflow-core.ts @@ -28,15 +28,23 @@ export interface WorkflowRuntime { readonly cancelCurrentScope: () => void /** Whether an error is the driver's cancellation (a normal stop, not a failure). */ readonly isCancellation: (error: unknown) => boolean + /** Restart the run with fresh history. Only meaningful for drivers that keep history. */ + readonly continueAsNew?: (sessionID: string) => Promise } export interface WorkflowOptions { /** How long to stay alive with no work before self-terminating. */ readonly idleTimeout?: string + /** Drains per run before continue-as-new, when the driver supports it. */ + readonly maxDrainsPerRun?: number } export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => { const IDLE_TIMEOUT = options?.idleTimeout ?? "5 minutes" + // A continuously busy session never hits the idle return, so without a bound its history grows + // until Temporal terminates the workflow. A fresh run starts with a pending wake, so no work is + // lost across the boundary. + const MAX_DRAINS_PER_RUN = options?.maxDrainsPerRun ?? 30 async function sessionExecution(sessionID: string): Promise { let pendingWake = true // started by a wake -> there is work to drain @@ -81,11 +89,14 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => // interrupt cancels the whole scope, so a cancellation can surface at the idle wait itself, // not just inside a drain; treat it as a normal stop rather than a failure. + let drains = 0 try { for (;;) { const gotWork = await rt.condition(() => pendingWake || stopping, IDLE_TIMEOUT) if (stopping) return if (!gotWork) { + // A wake can race the idle timer; without this re-check it would be dropped. + if (pendingWake) continue // Idle: terminate only when nothing is in flight. A later wake/resume starts a fresh run. if (!draining && handlers === 0) return continue @@ -97,6 +108,9 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => // wake tolerates run errors (the coordinator logs and moves on); only cancellation stops us. if (rt.isCancellation(e)) return } + drains++ + if (rt.continueAsNew && drains >= MAX_DRAINS_PER_RUN && handlers === 0) + await rt.continueAsNew(sessionID) } } catch (e) { if (rt.isCancellation(e)) return @@ -153,11 +167,13 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => } }) + let drains = 0 try { for (;;) { const gotWork = await rt.condition(() => pendingWake || stopping, IDLE_TIMEOUT) if (stopping) return if (!gotWork) { + if (pendingWake) continue if (!draining && handlers === 0) return continue } @@ -167,6 +183,9 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => } catch (e) { if (rt.isCancellation(e)) return } + drains++ + if (rt.continueAsNew && drains >= MAX_DRAINS_PER_RUN && handlers === 0) + await rt.continueAsNew(sessionID) } } catch (e) { if (rt.isCancellation(e)) return From 658755c041ad1109c744450010016d0ae5c9c5e7 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 16:19:15 -0700 Subject: [PATCH 046/103] Served the step-entry checks from one projected-history load. Every step loaded the projection three times before the turn (crash-resume check, recovery gate, interrupted-tool close), and each load runs through the store permit. Nothing mutates the projection between those checks, so one load now serves all three; paths that mutate first still reload. --- packages/core/src/session/runner/llm.ts | 41 +++++++++++++++++-------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index ea196181a958..ca8305318f9c 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -118,10 +118,13 @@ const layer = Layer.effect( const getContext = Effect.fn("SessionRunner.getContext")(function* (sessionID: SessionSchema.ID) { return yield* store.context(sessionID) }) + // `preloaded` lets one projected-history read serve the step-entry checks; callers that just + // mutated the projection must not pass it, or they act on a stale view. const failInterruptedTools = Effect.fn("SessionRunner.failInterruptedTools")(function* ( sessionID: SessionSchema.ID, + preloaded?: ReadonlyArray, ) { - for (const message of yield* getContext(sessionID)) { + for (const message of preloaded ?? (yield* getContext(sessionID))) { if (message.type !== "assistant") continue for (const tool of message.content) { if (tool.type !== "tool" || (tool.state.status !== "pending" && tool.state.status !== "running")) continue @@ -149,8 +152,11 @@ const layer = Layer.effect( // conversational message is a user prompt with no assistant reply, or the latest assistant is // still in flight. An interrupted turn stays excluded because its cleanup completes the // assistant via Step.Failed. - const hasRecoverableWork = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) { - const context = yield* getContext(sessionID) + const hasRecoverableWork = Effect.fnUntraced(function* ( + sessionID: SessionSchema.ID, + preloaded?: ReadonlyArray, + ) { + const context = preloaded ?? (yield* getContext(sessionID)) for (let index = context.length - 1; index >= 0; index--) { const message = context[index] if (message?.type === "assistant") return !message.time.completed @@ -412,9 +418,12 @@ const layer = Layer.effect( }) { const hasSteer = yield* SessionInput.hasPending(db, input.sessionID, "steer") const hasQueue = hasSteer ? false : yield* SessionInput.hasPending(db, input.sessionID, "queue") - const recover = !input.force && !hasSteer && !hasQueue && (yield* hasRecoverableWork(input.sessionID)) + const entryContext = yield* getContext(input.sessionID) + const recover = + !input.force && !hasSteer && !hasQueue && + (yield* hasRecoverableWork(input.sessionID, entryContext)) if (!input.force && !hasSteer && !hasQueue && !recover) return - yield* failInterruptedTools(input.sessionID) + yield* failInterruptedTools(input.sessionID, entryContext) let promotion: SessionInput.Delivery | undefined = hasSteer ? "steer" : hasQueue ? "queue" : undefined let shouldRun = input.force || hasSteer || hasQueue || recover while (shouldRun) { @@ -442,11 +451,14 @@ const layer = Layer.effect( // fresh step, or a partial with no dispatched tools, which is safe to re-stream). Token/cost // metering is 0 for the resumed step only; faithful metering would need a durable step-sealed // marker carrying the provider usage. - const resumeCrashedStep = Effect.fn("SessionRunner.resumeCrashedStep")(function* (input: { - readonly sessionID: SessionSchema.ID - readonly step: number - }) { - const context = yield* getContext(input.sessionID) + const resumeCrashedStep = Effect.fn("SessionRunner.resumeCrashedStep")(function* ( + input: { + readonly sessionID: SessionSchema.ID + readonly step: number + }, + preloaded?: ReadonlyArray, + ) { + const context = preloaded ?? (yield* getContext(input.sessionID)) // At most one assistant is in flight (the projector supersedes older ones); it only exists at // step entry on a re-drive, never on a fresh step. const inFlight = context.findLast( @@ -533,9 +545,12 @@ const layer = Layer.effect( readonly first: boolean readonly force: boolean }) { + // One projected-history load serves all the entry checks; nothing mutates the projection + // between them. The turn itself reloads after the first mutation. + const entryContext = yield* getContext(input.sessionID) // Re-drive of a crashed step: finalize it from the log rather than re-calling the model and // re-running its already-dispatched tools. - const resumed = yield* resumeCrashedStep(input) + const resumed = yield* resumeCrashedStep(input, entryContext) if (resumed) return resumed let promotion = input.promotion if (input.first) { @@ -548,7 +563,7 @@ const layer = Layer.effect( !input.force && !hasSteer && !hasQueue && - !(yield* hasRecoverableWork(input.sessionID)) + !(yield* hasRecoverableWork(input.sessionID, entryContext)) ) return { ran: false, continue: false, step: input.step, promotion: undefined } promotion = hasSteer ? "steer" : hasQueue ? "queue" : undefined @@ -557,7 +572,7 @@ const layer = Layer.effect( // first. A mid-turn re-drive (first=false, from a Temporal step retry) would otherwise // re-stream a request with a dangling tool_use and no tool_result, which the provider rejects // -- a retry poison loop. This is a no-op on a healthy step (the prior step settled its tools). - yield* failInterruptedTools(input.sessionID) + yield* failInterruptedTools(input.sessionID, entryContext) const result = yield* runTurn(input.sessionID, promotion, input.step) let needsContinuation = result.needsContinuation if (!needsContinuation) needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer") From 981f941ac303349240fcb9ef56d3515aa0320be6 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 16:20:21 -0700 Subject: [PATCH 047/103] Applied foreign keys best-effort on the shared backend; small hygiene. SQLite defaults foreign keys off, and the shared backend skipped the pragma entirely, so the two backends diverged in integrity behavior. The pragma now runs best-effort (an embedded file honors it; a remote server applies it per stream at most), and the README states that remote deletes rely on the application-side cascades. The driver test restores the idle-timeout env var it sets, and the README notes the fresh-remote-store migration transaction and the zombie-fence follow-up design. --- packages/core/src/database/database.ts | 4 ++++ .../test/session-execution-local-driver.test.ts | 4 ++++ packages/temporal/README.md | 13 +++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/core/src/database/database.ts b/packages/core/src/database/database.ts index 669b4831d487..e3e625f8f254 100644 --- a/packages/core/src/database/database.ts +++ b/packages/core/src/database/database.ts @@ -35,6 +35,10 @@ function makeServiceLayer(localPragmas: boolean) { yield* db.run("PRAGMA cache_size = -64000") yield* db.run("PRAGMA foreign_keys = ON") yield* db.run("PRAGMA wal_checkpoint(PASSIVE)") + } else { + // Best effort on the shared backend: an embedded file honors it, a remote server applies + // it per stream at most. Deletes on a remote store rely on the application-side cascades. + yield* db.run("PRAGMA foreign_keys = ON").pipe(Effect.exit) } yield* DatabaseMigration.apply(db) diff --git a/packages/core/test/session-execution-local-driver.test.ts b/packages/core/test/session-execution-local-driver.test.ts index 3c0b7ebf1f98..e9307b9481e8 100644 --- a/packages/core/test/session-execution-local-driver.test.ts +++ b/packages/core/test/session-execution-local-driver.test.ts @@ -146,6 +146,7 @@ describe("SessionExecution local micro-driver", () => { const sessionID = SessionV2.ID.make("ses_driver_wake") it.live("wake drives a turn to settlement, then the idle supervisor retires", () => Effect.gen(function* () { + const previousIdle = process.env.OPENCODE_SESSION_IDLE_TIMEOUT process.env.OPENCODE_SESSION_IDLE_TIMEOUT = "2 seconds" yield* seedSession(sessionID) yield* seedPrompt(sessionID) @@ -160,6 +161,9 @@ describe("SessionExecution local micro-driver", () => { expect((yield* exec.active).has(sessionID)).toBe(true) // Idle self-termination: the driver retires without an interrupt. yield* until(exec.active, (active) => !active.has(sessionID)) + // Restore so later layer builds in this process get the real default. + if (previousIdle === undefined) delete process.env.OPENCODE_SESSION_IDLE_TIMEOUT + else process.env.OPENCODE_SESSION_IDLE_TIMEOUT = previousIdle }), ) } diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 823dbd0e115f..4c108ee4ec6e 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -150,7 +150,11 @@ later. Verified by `packages/core/test/session-runner-resume.test.ts`. Known limit: when an attempt is retried while the previous one is still alive (a network partition, or the backstop firing), the old attempt keeps publishing for a few seconds until its heartbeat is rejected and the AbortSignal interrupts it; the projector's status guards make - duplicate settlements no-ops in projection, but the overlap window is not fully fenced. + duplicate settlements no-ops in projection, but the overlap window is not fully fenced. The + fence design exists as a follow-up: `event_sequence.owner_id` and `claim()` are already in the + event store; claim the aggregate per attempt and reject stale-owner appends inside the per-event + transaction. It threads an owner through every publish (including compaction's), so it deserves + its own change. `scripts/resume-check.ts` verifies resume: it resolves on a healthy session and rejects on a failing one with the original tagged error (`LLM.Error`) reconstructed across the boundary. @@ -221,7 +225,12 @@ distinct worker identities. - Cold-start migrations serialize across processes (one `BEGIN IMMEDIATE` transaction wraps check-and-apply, so concurrent starts wait and then no-op). Running migrations once as a deploy step is still good practice for large fleets, and on a remote store it keeps cold starts from - contending for the write lock. + contending for the write lock. A fresh remote store also builds the whole schema inside one + write transaction; a slow link or a server-side transaction timeout can kill that, one more + reason to migrate before starting workers. +- Foreign keys are enforced on the local backend and best-effort on the shared one (SQLite + defaults them off; a remote server applies the pragma per stream at most). Deletes on the + shared store rely on the application-side cascades. - The PRAGMAs (`journal_mode` / `synchronous` / `busy_timeout` / `cache_size` / `wal_checkpoint`) are local-file semantics and are skipped for the shared/libSQL backend, which manages journaling itself. From 96b49c875466421451af59e14d1e522249cf4036 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 17:00:26 -0700 Subject: [PATCH 048/103] Read the active-session store checks concurrently. active() did one sequential store.get per running workflow. The reads bypass the permit, so bounded concurrency removes the serial N+1; an IN query is the upgrade if the set ever grows to hundreds. --- packages/core/src/session/execution/temporal.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index de600b7617d2..081b6df9499c 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -141,9 +141,14 @@ const layer = Layer.effect( } return ids }) - const ids = new Set() - for (const id of found) if (yield* store.get(id)) ids.add(id) - return ids + // Point reads, but concurrent: they bypass the store permit, and the running-workflow set + // is small. A single IN query is the upgrade if this ever grows to hundreds. + const known = yield* Effect.forEach( + found, + (id) => Effect.map(store.get(id), (session) => (session ? id : undefined)), + { concurrency: 8 }, + ) + return new Set(known.filter((id): id is SessionSchema.ID => id !== undefined)) }), wake: (id) => drive(id).pipe(Effect.asVoid), // resume = coordinator.run: drive a forced run via an Update-with-Start and AWAIT its result, From 598a3c8ccae311a88ed8a9083f38f107d1780f60 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 17:01:57 -0700 Subject: [PATCH 049/103] Typed the user-decline halt across the durable boundary. A decline crossed the boundary as a bare ApplicationFailure and decoded into the ContextSnapshotDecodeError carrier, so a resume caller could not tell a deliberate stop from a decode fault. SessionRunDeclinedError is now a RunError member, rides the same codec details, and decodes back to itself. --- packages/core/src/session/error.ts | 13 +++++++++++++ packages/core/src/session/execution/drain.ts | 9 +++++++++ .../core/src/session/execution/run-error-codec.ts | 3 ++- packages/core/src/session/runner/index.ts | 3 ++- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/core/src/session/error.ts b/packages/core/src/session/error.ts index 158e46dd073e..096a0c9049c2 100644 --- a/packages/core/src/session/error.ts +++ b/packages/core/src/session/error.ts @@ -22,3 +22,16 @@ export class ContextSnapshotDecodeError extends Schema.TaggedErrorClass()( + "Session.SessionRunDeclinedError", + { + sessionID: SessionSchema.ID, + }, +) { + override get message() { + return `Session run halted by the user: ${this.sessionID}` + } +} diff --git a/packages/core/src/session/execution/drain.ts b/packages/core/src/session/execution/drain.ts index db4aeb39ccf4..2c2f0ee683a1 100644 --- a/packages/core/src/session/execution/drain.ts +++ b/packages/core/src/session/execution/drain.ts @@ -10,6 +10,7 @@ import type { LocationError, LocationServices } from "../../location-services" import { SessionRunner } from "../runner" import { SessionSchema } from "../schema" import { SessionStore } from "../store" +import { SessionRunDeclinedError } from "../error" import type { SessionInput } from "../input" import { encodeRunError } from "./run-error-codec" import type { DrainInput, StepDrainInput, StepDrainResult } from "./temporal-activities" @@ -43,10 +44,14 @@ export const makeDrains = ({ store, locations, ctx }: DrainDeps) => { // re-drives a turn the user explicitly stopped. if (signal.aborted) throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") + const declined = encodeRunError( + new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make(input.sessionID) }), + ) throw ApplicationFailure.create({ message: "session run halted (user declined)", type: "SessionRunDeclined", nonRetryable: true, + details: declined === undefined ? undefined : [declined], }) } // A genuine run error is thrown non-retryable so Temporal surfaces it (to resume) rather than @@ -89,10 +94,14 @@ export const makeDrains = ({ store, locations, ctx }: DrainDeps) => { // an internal user-decline halt is non-retryable. if (signal.aborted) throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") + const declined = encodeRunError( + new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make(input.sessionID) }), + ) throw ApplicationFailure.create({ message: "session run halted (user declined)", type: "SessionRunDeclined", nonRetryable: true, + details: declined === undefined ? undefined : [declined], }) } const squashed = Cause.squash(cause) as { _tag?: string; message?: string } diff --git a/packages/core/src/session/execution/run-error-codec.ts b/packages/core/src/session/execution/run-error-codec.ts index 261289de0986..472a0eba90ba 100644 --- a/packages/core/src/session/execution/run-error-codec.ts +++ b/packages/core/src/session/execution/run-error-codec.ts @@ -8,7 +8,7 @@ import { Integration } from "../../integration" import { SystemContext } from "../../system-context/index" import { ToolOutputStore } from "../../tool-output-store" import type { SessionSchema } from "../schema" -import { ContextSnapshotDecodeError, MessageDecodeError } from "../error" +import { ContextSnapshotDecodeError, MessageDecodeError, SessionRunDeclinedError } from "../error" import { ModelNotSelectedError, ModelUnavailableError, @@ -26,6 +26,7 @@ const RunErrorSchema = Schema.Union([ Integration.AuthorizationError, MessageDecodeError, ContextSnapshotDecodeError, + SessionRunDeclinedError, SystemContext.InitializationBlocked, ToolOutputStore.StorageError, ]) diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index c842be7dca61..e3ef80b67d07 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -3,7 +3,7 @@ export * as SessionRunner from "./index" import type { LLMError } from "@opencode-ai/llm" import { Context, Effect } from "effect" import { SessionSchema } from "../schema" -import type { ContextSnapshotDecodeError, MessageDecodeError } from "../error" +import type { ContextSnapshotDecodeError, MessageDecodeError, SessionRunDeclinedError } from "../error" import type { SessionInput } from "../input" import { SessionRunnerModel } from "./model" import type { SystemContext } from "../../system-context/index" @@ -14,6 +14,7 @@ export type RunError = | SessionRunnerModel.Error | MessageDecodeError | ContextSnapshotDecodeError + | SessionRunDeclinedError | SystemContext.InitializationBlocked | ToolOutputStore.Error From 4146c27e922fd6ae1fe0d2ae2f13b7620249d95f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 17:03:25 -0700 Subject: [PATCH 050/103] Gave the local driver the same 12h drain backstop as Temporal mode. A hung tool in local mode held draining=true forever, so no later wake could run. A per-drain timer aborts through the driver's own signal, so a timed-out drain unwinds like any interrupt. --- .../src/session/execution/local-driver.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts index 194a574191c4..4a01919771c5 100644 --- a/packages/core/src/session/execution/local-driver.ts +++ b/packages/core/src/session/execution/local-driver.ts @@ -40,6 +40,8 @@ const parseDuration = (value: string): number => { class LocalCancellation extends Error {} +const BACKSTOP_MS = 12 * 60 * 60 * 1000 + interface Waiter { readonly predicate: () => boolean readonly resolve: (value: boolean) => void @@ -78,8 +80,10 @@ class SessionDriver { }), setSignalHandler: (name, handler) => this.signalHandlers.set(name, handler), setUpdateHandler: (name, handler) => this.updateHandlers.set(name, handler), - runContinuation: (input) => drains.drain(input, this.abort.signal), - runTurnStep: (input) => drains.stepDrain(input, this.abort.signal), + // Same 12 h backstop as the Temporal activity: a hung tool must not hold `draining` forever. + // The abort reason is a LocalCancellation, so a timed-out drain looks like any other cancel. + runContinuation: (input) => this.withBackstop((signal) => drains.drain(input, signal)), + runTurnStep: (input) => this.withBackstop((signal) => drains.stepDrain(input, signal)), cancelCurrentScope: () => this.cancel(), isCancellation: (error) => error instanceof LocalCancellation, } @@ -104,6 +108,17 @@ class SessionDriver { return result } + // The drain shares the driver's abort signal so an interrupt still cancels it; the timer only + // adds an upper bound. + private async withBackstop(run: (signal: AbortSignal) => Promise): Promise { + const timer = setTimeout(() => this.abort.abort(new LocalCancellation("drain backstop")), BACKSTOP_MS) + try { + return await run(this.abort.signal) + } finally { + clearTimeout(timer) + } + } + private cancel() { this.cancelled = true // The drain rethrows the signal's reason on cancellation, so the supervisor observes the same From 8d15027a3d3d43f0660ce1672562929410a9e305 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 17:07:11 -0700 Subject: [PATCH 051/103] Swept abandoned pending permission rows; deflaked concurrent open. An interrupted turn's asks stayed pending forever, so they kept showing in list()/forSession() and the decline cascade. Reads now lazily expire rows untouched past a TTL, with a status index behind the no-session scan. Separately, the bun backend set busy_timeout after switching to WAL, so N cold workers racing the same shared file could SQLITE_BUSY at open; the timeout now comes first. --- packages/core/schema.json | 18 ++++++++- packages/core/src/database/migration.gen.ts | 1 + ...812000509_permission_request_status_idx.ts | 11 ++++++ packages/core/src/database/schema.gen.ts | 1 + packages/core/src/database/sqlite.bun.ts | 4 ++ packages/core/src/permission.ts | 37 ++++++++++++++----- packages/core/src/permission/sql.ts | 6 ++- packages/core/test/permission-durable.test.ts | 27 ++++++++++++++ 8 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 packages/core/src/database/migration/20260812000509_permission_request_status_idx.ts diff --git a/packages/core/schema.json b/packages/core/schema.json index f9825126f5ec..f03443e8c216 100644 --- a/packages/core/schema.json +++ b/packages/core/schema.json @@ -1,9 +1,9 @@ { "version": "7", "dialect": "sqlite", - "id": "ef05c3dc-ecd1-4def-9573-6b872e58366e", + "id": "69462d47-737f-4005-83ff-1f43cf647276", "prevIds": [ - "f14a9b18-8207-487e-a3d3-227e629ba9ad" + "ef05c3dc-ecd1-4def-9573-6b872e58366e" ], "ddl": [ { @@ -2002,6 +2002,20 @@ "entityType": "indexes", "table": "permission_request" }, + { + "columns": [ + { + "value": "status", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "permission_request_status_idx", + "entityType": "indexes", + "table": "permission_request" + }, { "columns": [ { diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index 9ff18f333253..6c6c79d16089 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -41,5 +41,6 @@ export const migrations = ( import("./migration/20260622170816_reset_v2_session_state"), import("./migration/20260622202450_simplify_session_input"), import("./migration/20260810092511_permission_request"), + import("./migration/20260812000509_permission_request_status_idx"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20260812000509_permission_request_status_idx.ts b/packages/core/src/database/migration/20260812000509_permission_request_status_idx.ts new file mode 100644 index 000000000000..9f1a921df73e --- /dev/null +++ b/packages/core/src/database/migration/20260812000509_permission_request_status_idx.ts @@ -0,0 +1,11 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812000509_permission_request_status_idx", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(`CREATE INDEX \`permission_request_status_idx\` ON \`permission_request\` (\`status\`);`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/schema.gen.ts b/packages/core/src/database/schema.gen.ts index a0fbf49536ff..78c4351cfd0e 100644 --- a/packages/core/src/database/schema.gen.ts +++ b/packages/core/src/database/schema.gen.ts @@ -253,6 +253,7 @@ export default { yield* tx.run( `CREATE INDEX \`permission_request_session_status_idx\` ON \`permission_request\` (\`session_id\`,\`status\`);`, ) + yield* tx.run(`CREATE INDEX \`permission_request_status_idx\` ON \`permission_request\` (\`status\`);`) yield* tx.run( `CREATE UNIQUE INDEX \`permission_project_action_resource_idx\` ON \`permission\` (\`project_id\`,\`action\`,\`resource\`);`, ) diff --git a/packages/core/src/database/sqlite.bun.ts b/packages/core/src/database/sqlite.bun.ts index e15f4c117e46..1f217331995c 100644 --- a/packages/core/src/database/sqlite.bun.ts +++ b/packages/core/src/database/sqlite.bun.ts @@ -161,6 +161,10 @@ const nativeLayer = (config: Config) => create: config.create ?? true, }) yield* Effect.addFinalizer(() => Effect.sync(() => native.close())) + // busy_timeout must come first: switching to WAL takes a write lock, and N cold workers + // opening the same shared file race on it. Without the timeout the losers get SQLITE_BUSY at + // open instead of waiting. + native.run("PRAGMA busy_timeout = 5000;") if (config.disableWAL !== true) native.run("PRAGMA journal_mode = WAL;") return native }), diff --git a/packages/core/src/permission.ts b/packages/core/src/permission.ts index 5b8b4a47e75c..068149e04608 100644 --- a/packages/core/src/permission.ts +++ b/packages/core/src/permission.ts @@ -18,6 +18,10 @@ import { PermissionRequestTable } from "./permission/sql" export { Effect, Rule, Ruleset } from "@opencode-ai/schema/permission" const missingAgentPermissions: Permission.Ruleset = [{ action: "*", resource: "*", effect: "deny" }] +// A pending ask older than this is treated as abandoned (the turn that raised it was interrupted or +// crashed, so nothing will answer it). Long enough that a human deliberating never trips it. +const PENDING_TTL_MS = 24 * 60 * 60 * 1000 + export const ID = Permission.ID export type ID = typeof ID.Type @@ -135,17 +139,30 @@ const layer = Layer.effect( EffectRuntime.orDie, EffectRuntime.map((rows) => rows[0]), ) + // An interrupted turn leaves its asks pending with nothing to answer them; without a bound they + // linger forever in list()/forSession() and the decline cascade. A row untouched past the TTL + // is treated as abandoned. Reads sweep them lazily (opportunistic prune), so no scheduler and + // no cross-process coordination. const pendingRows = (sessionID?: SessionV2.ID) => - db - .select() - .from(PermissionRequestTable) - .where( - sessionID - ? and(eq(PermissionRequestTable.session_id, sessionID), eq(PermissionRequestTable.status, "pending")) - : eq(PermissionRequestTable.status, "pending"), - ) - .all() - .pipe(EffectRuntime.orDie) + EffectRuntime.gen(function* () { + const cutoff = Date.now() - PENDING_TTL_MS + const rows = yield* db + .select() + .from(PermissionRequestTable) + .where( + sessionID + ? and(eq(PermissionRequestTable.session_id, sessionID), eq(PermissionRequestTable.status, "pending")) + : eq(PermissionRequestTable.status, "pending"), + ) + .all() + .pipe(EffectRuntime.orDie) + const fresh: typeof rows = [] + for (const row of rows) { + if (row.time_updated < cutoff) yield* transitionRow(row.id, "pending", "expired") + else fresh.push(row) + } + return fresh + }) // Status moves are compare-and-set on the stated `from`, so a reply that lost a race, or a // shutdown racing an approval, cannot overwrite a landed outcome. const transitionRow = (id: string, from: string, to: string, message?: string) => diff --git a/packages/core/src/permission/sql.ts b/packages/core/src/permission/sql.ts index 000257d9351b..d29c737583a9 100644 --- a/packages/core/src/permission/sql.ts +++ b/packages/core/src/permission/sql.ts @@ -34,5 +34,9 @@ export const PermissionRequestTable = sqliteTable( message: text(), ...Timestamps, }, - (table) => [index("permission_request_session_status_idx").on(table.session_id, table.status)], + (table) => [ + index("permission_request_session_status_idx").on(table.session_id, table.status), + // The no-session list() and the TTL sweep both filter by status alone. + index("permission_request_status_idx").on(table.status), + ], ) diff --git a/packages/core/test/permission-durable.test.ts b/packages/core/test/permission-durable.test.ts index d106dee1b08e..4970fe914fb4 100644 --- a/packages/core/test/permission-durable.test.ts +++ b/packages/core/test/permission-durable.test.ts @@ -4,6 +4,7 @@ // independent service stacks share one DB file to simulate the two processes. import { describe, expect } from "bun:test" import path from "path" +import { createClient } from "@libsql/client" import { Cause, Context, Effect, Exit, Fiber, Layer, Scope } from "effect" import { AgentV2 } from "@opencode-ai/core/agent" import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" @@ -237,6 +238,32 @@ describe("PermissionV2 durable asks", () => { }), ) + it.live("sweeps an abandoned pending ask out of the list on read", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + yield* seed(file) + const A = yield* Layer.build(stack(file)) + const permA = Context.get(A, PermissionV2.Service) + const blocked = yield* permA + .assert({ sessionID, action: "bash", resources: ["echo hi"], agent }) + .pipe(Effect.forkChild) + const ask = yield* awaitAsk(permA) + // Backdate the row past the TTL: the turn that raised it is gone. + yield* Effect.promise(async () => { + const raw = createClient({ url: `file:${file}` }) + await raw.execute({ + sql: "UPDATE permission_request SET time_updated = ? WHERE id = ?", + args: [Date.now() - 25 * 60 * 60 * 1000, ask.id], + }) + raw.close() + }) + expect(yield* permA.list()).toEqual([]) + yield* Fiber.interrupt(blocked) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + it.live("expires locally-pending asks on shutdown so they do not linger as pending rows", () => Effect.gen(function* () { const tmp = yield* Effect.promise(() => tmpdir()) From bb3ec94ffed15e0f369f4818c4bc7c1de5f5b241 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 17:33:49 -0700 Subject: [PATCH 052/103] Retried the cold WAL switch instead of trusting busy_timeout. A journal_mode change does not honor busy_timeout, so cold workers racing the first WAL switch on a fresh file still hit SQLITE_BUSY. The switch now retries until one wins and the rest find the file already in WAL. --- packages/core/src/database/sqlite.bun.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/core/src/database/sqlite.bun.ts b/packages/core/src/database/sqlite.bun.ts index 1f217331995c..41d659400c46 100644 --- a/packages/core/src/database/sqlite.bun.ts +++ b/packages/core/src/database/sqlite.bun.ts @@ -161,11 +161,22 @@ const nativeLayer = (config: Config) => create: config.create ?? true, }) yield* Effect.addFinalizer(() => Effect.sync(() => native.close())) - // busy_timeout must come first: switching to WAL takes a write lock, and N cold workers - // opening the same shared file race on it. Without the timeout the losers get SQLITE_BUSY at - // open instead of waiting. + // Set busy_timeout first so later writers wait on a held lock instead of erroring. native.run("PRAGMA busy_timeout = 5000;") - if (config.disableWAL !== true) native.run("PRAGMA journal_mode = WAL;") + // A journal_mode change does NOT honor busy_timeout, so N cold workers racing the first WAL + // switch on a fresh file can get SQLITE_BUSY. Retry until one wins; the rest then find the + // file already in WAL and the pragma returns at once. + if (config.disableWAL !== true) { + for (let attempt = 0; ; attempt++) { + try { + native.run("PRAGMA journal_mode = WAL;") + break + } catch (cause) { + if (attempt >= 50 || !/SQLITE_BUSY|database is locked/i.test(String(cause))) throw cause + Bun.sleepSync(20) + } + } + } return native }), ) From 5227d92b0780181112e301f23ee76a7261b2780e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 17:33:50 -0700 Subject: [PATCH 053/103] Fenced the event log behind the running attempt. A superseded Temporal attempt or a retired local driver could keep appending events under a session another attempt had taken over, corrupting the log. Each drain now claims the log with an attempt token, and a live durable append dies if a newer attempt has since claimed it. --- packages/core/src/event.ts | 28 ++++++++++- packages/core/src/session/execution/drain.ts | 13 +++-- .../src/session/execution/local-driver.ts | 16 +++++-- .../session/execution/temporal-activities.ts | 17 ++++++- .../core/src/session/execution/temporal.ts | 6 ++- packages/core/test/event.test.ts | 48 +++++++++++++++++++ 6 files changed, 115 insertions(+), 13 deletions(-) diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index c92ac0ac2ce3..97a003bdf132 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -115,6 +115,15 @@ export class SubscriberOverflowError extends Schema.TaggedErrorClass("@opencode/Event/Owner", { + defaultValue: () => undefined, +}) + export interface PublishOptions { readonly id?: ID readonly metadata?: Record @@ -234,6 +243,7 @@ export const layerWith = (options?: LayerOptions) => ) } const list = projectors.get(event.type) ?? [] + const owner = yield* EventOwner return yield* Effect.uninterruptible( Effect.gen(function* () { const committed = yield* db @@ -259,6 +269,18 @@ export const layerWith = (options?: LayerOptions) => }), ) } + // Fence a live append behind the current owner. A superseded attempt + // still holding the loop would otherwise keep writing events under a + // session a newer attempt has claimed. `input` is the replay path, which + // owns its check above. + if (!input && owner !== undefined && row?.ownerID != null && row.ownerID !== owner) { + yield* Effect.die( + new InvalidDurableEventError({ + type: event.type, + message: `Owner fence for aggregate ${aggregateID}: held by ${row.ownerID}, publisher ${owner}`, + }), + ) + } if (input && input.seq <= latest) { const stored = yield* db .select() @@ -323,12 +345,14 @@ export const layerWith = (options?: LayerOptions) => if (commit) yield* commit(seq) yield* db .insert(EventSequenceTable) - .values([{ aggregate_id: aggregateID, seq, owner_id: input?.ownerID }]) + .values([{ aggregate_id: aggregateID, seq, owner_id: input?.ownerID ?? owner }]) .onConflictDoUpdate({ target: EventSequenceTable.aggregate_id, set: { seq, - ...(input?.ownerID && row?.ownerID == null ? { owner_id: input.ownerID } : {}), + ...((input?.ownerID ?? owner) && row?.ownerID == null + ? { owner_id: input?.ownerID ?? owner } + : {}), }, }) .run() diff --git a/packages/core/src/session/execution/drain.ts b/packages/core/src/session/execution/drain.ts index 2c2f0ee683a1..06fa68aa984b 100644 --- a/packages/core/src/session/execution/drain.ts +++ b/packages/core/src/session/execution/drain.ts @@ -7,6 +7,7 @@ import { ApplicationFailure } from "@temporalio/activity" import type { LocationServiceMap } from "../../location-service-map" import type { Location } from "../../location" import type { LocationError, LocationServices } from "../../location-services" +import { EventV2 } from "../../event" import { SessionRunner } from "../runner" import { SessionSchema } from "../schema" import { SessionStore } from "../store" @@ -21,18 +22,22 @@ export interface DrainDeps { /** The app context the drain runs in; providing it plus the per-location layer supplies * SessionRunner and all of its dependencies. */ readonly ctx: Context.Context + /** Used to claim the event log for the running attempt so a superseded one is fenced. */ + readonly events: EventV2.Interface } -export const makeDrains = ({ store, locations, ctx }: DrainDeps) => { +export const makeDrains = ({ store, locations, ctx, events }: DrainDeps) => { const drain = async (input: DrainInput, signal: AbortSignal): Promise => { const exit = await Effect.runPromiseExit( Effect.gen(function* () { const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) if (!session) return + // Take the event log before running so a superseded attempt's later appends are fenced. + if (input.owner) yield* events.claim(session.id, input.owner) yield* SessionRunner.Service.use((runner) => runner.run({ sessionID: session.id, force: input.force }), ).pipe(Effect.provide(locations.get(session.location))) - }).pipe(Effect.provide(ctx), Effect.scoped), + }).pipe(Effect.provideService(EventV2.EventOwner, input.owner), Effect.provide(ctx), Effect.scoped), { signal }, ) if (Exit.isSuccess(exit)) return @@ -74,6 +79,8 @@ export const makeDrains = ({ store, locations, ctx }: DrainDeps) => { Effect.gen(function* () { const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) if (!session) return { ran: false, continue: false, step: input.step, promotion: null } + // Take the event log before running so a superseded attempt's later appends are fenced. + if (input.owner) yield* events.claim(session.id, input.owner) const r = yield* SessionRunner.Service.use((runner) => runner.runStep({ sessionID: session.id, @@ -84,7 +91,7 @@ export const makeDrains = ({ store, locations, ctx }: DrainDeps) => { }), ).pipe(Effect.provide(locations.get(session.location))) return { ran: r.ran, continue: r.continue, step: r.step, promotion: r.promotion ?? null } - }).pipe(Effect.provide(ctx), Effect.scoped), + }).pipe(Effect.provideService(EventV2.EventOwner, input.owner), Effect.provide(ctx), Effect.scoped), { signal }, ) if (Exit.isSuccess(exit)) return exit.value diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts index 4a01919771c5..e8fc57ff6958 100644 --- a/packages/core/src/session/execution/local-driver.ts +++ b/packages/core/src/session/execution/local-driver.ts @@ -9,7 +9,9 @@ export * as SessionExecutionLocalDriver from "./local-driver" // drivers" shape: the factory picks the driver, the supervisor is written once. import { Effect, Layer } from "effect" +import { randomUUID } from "node:crypto" import { LocationServiceMap } from "../../location-service-map" +import { EventV2 } from "../../event" import { makeGlobalNode } from "../../effect/app-node" import { SessionSchema } from "../schema" import { SessionStore } from "../store" @@ -62,6 +64,9 @@ class SessionDriver { private ticker: ReturnType | undefined private cancelled = false private readonly abort = new AbortController() + // One token per driver instance. A wake that lands after a driver finished starts a fresh driver + // (a new token), so the retired one's late appends are fenced, mirroring the Temporal attempt. + private readonly owner = randomUUID() constructor(run: (rt: WorkflowRuntime) => Promise, drains: Drains, onDone: () => void) { const rt: WorkflowRuntime = { @@ -82,8 +87,10 @@ class SessionDriver { setUpdateHandler: (name, handler) => this.updateHandlers.set(name, handler), // Same 12 h backstop as the Temporal activity: a hung tool must not hold `draining` forever. // The abort reason is a LocalCancellation, so a timed-out drain looks like any other cancel. - runContinuation: (input) => this.withBackstop((signal) => drains.drain(input, signal)), - runTurnStep: (input) => this.withBackstop((signal) => drains.stepDrain(input, signal)), + runContinuation: (input) => + this.withBackstop((signal) => drains.drain({ ...input, owner: this.owner }, signal)), + runTurnStep: (input) => + this.withBackstop((signal) => drains.stepDrain({ ...input, owner: this.owner }, signal)), cancelCurrentScope: () => this.cancel(), isCancellation: (error) => error instanceof LocalCancellation, } @@ -169,7 +176,8 @@ const layer = Layer.effect( const store = yield* SessionStore.Service const locations = yield* LocationServiceMap.Service const ctx = yield* Effect.context() - const drains = makeDrains({ store, locations, ctx }) + const events = yield* EventV2.Service + const drains = makeDrains({ store, locations, ctx, events }) const drivers = new Map() // Read at layer build (not module load) so tests can set it before constructing the layer. // The idle override shortens the supervisor's 5-minute self-termination. @@ -234,5 +242,5 @@ const layer = Layer.effect( export const node = makeGlobalNode({ service: SessionExecution.Service, layer, - deps: [SessionStore.node, LocationServiceMap.node], + deps: [SessionStore.node, LocationServiceMap.node, EventV2.node], }) diff --git a/packages/core/src/session/execution/temporal-activities.ts b/packages/core/src/session/execution/temporal-activities.ts index 060ea6620591..74879b08099d 100644 --- a/packages/core/src/session/execution/temporal-activities.ts +++ b/packages/core/src/session/execution/temporal-activities.ts @@ -5,9 +5,20 @@ import { heartbeat, Context } from "@temporalio/activity" +// The event-log owner for this attempt: the run id plus the attempt number. A Temporal retry gets a +// fresh attempt, so once the retry claims the log, the previous attempt (if it is still running) is +// fenced out of writing. +function ownerToken(): string { + const info = Context.current().info + return `${info.workflowExecution?.runId ?? info.workflowType}#${info.attempt}` +} + export interface DrainInput { sessionID: string force: boolean + // The attempt that owns the event log while this drain runs. Set activity-side from the run id + // and attempt so it stays out of the workflow's deterministic input. + owner?: string } export type Activities = { @@ -27,7 +38,7 @@ export function makeActivities( } }, 3000) try { - await drain(input, Context.current().cancellationSignal) + await drain({ ...input, owner: ownerToken() }, Context.current().cancellationSignal) } finally { clearInterval(beat) } @@ -44,6 +55,8 @@ export interface StepDrainInput { promotion: string | null first: boolean force: boolean + // Set activity-side (see ownerToken), not part of the workflow's deterministic input. + owner?: string } export interface StepDrainResult { @@ -68,7 +81,7 @@ export function makeStepActivities( } catch {} }, 3000) try { - return await stepDrain(input, Context.current().cancellationSignal) + return await stepDrain({ ...input, owner: ownerToken() }, Context.current().cancellationSignal) } finally { clearInterval(beat) } diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 081b6df9499c..dc2fef01723b 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -6,6 +6,7 @@ import { Client, Connection, WithStartWorkflowOperation } from "@temporalio/clie import { NativeConnection, Worker } from "@temporalio/worker" import { LocationServiceMap } from "../../location-service-map" +import { EventV2 } from "../../event" import { makeGlobalNode } from "../../effect/app-node" import { SessionSchema } from "../schema" import { SessionStore } from "../store" @@ -52,10 +53,11 @@ const layer = Layer.effect( // The app context the local drain runs in: providing it, then the per-location layer, supplies // SessionRunner and all of its dependencies. const ctx = yield* Effect.context() + const events = yield* EventV2.Service // The drain bodies are shared with the in-process micro-driver (drain.ts), so turn semantics // and error encoding cannot differ between drivers. - const { drain, stepDrain } = makeDrains({ store, locations, ctx }) + const { drain, stepDrain } = makeDrains({ store, locations, ctx, events }) // Worker connection (native) hosts the runContinuation activity + the workflow. Skipped in // client-only role so serve can run without an embedded worker. @@ -203,5 +205,5 @@ const layer = Layer.effect( export const node = makeGlobalNode({ service: SessionExecution.Service, layer, - deps: [SessionStore.node, LocationServiceMap.node], + deps: [SessionStore.node, LocationServiceMap.node, EventV2.node], }) diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index e4329a2dde98..e5e71c9a81ad 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -1097,6 +1097,54 @@ describe("EventV2", () => { }), ) + it.effect("fences a live publish from a superseded owner and admits the current one", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const aggregateID = Session.ID.create() + + yield* events + .publish(DurableMessage, durableData(aggregateID, "seed")) + .pipe(Effect.provideService(EventV2.EventOwner, "owner-a")) + // A newer attempt takes the log. + yield* events.claim(aggregateID, "owner-b") + + const fenced = yield* events + .publish(DurableMessage, durableData(aggregateID, "stale")) + .pipe(Effect.provideService(EventV2.EventOwner, "owner-a"), Effect.exit) + yield* events + .publish(DurableMessage, durableData(aggregateID, "fresh")) + .pipe(Effect.provideService(EventV2.EventOwner, "owner-b")) + const rows = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, aggregateID)) + .all() + .pipe(Effect.orDie) + + expect(String(fenced)).toContain("Owner fence") + expect(rows.map((row) => row.seq)).toEqual([0, 1]) + expect(rows.map((row) => (row.data as { messageID: string }).messageID)).toEqual([ + durableData(aggregateID, "seed").messageID, + durableData(aggregateID, "fresh").messageID, + ]) + }), + ) + + it.effect("never fences a publish made outside a drain", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const aggregateID = Session.ID.create() + yield* events.publish(DurableMessage, durableData(aggregateID, "seed")) + yield* events.claim(aggregateID, "owner-b") + + // No owner in context (the default), so ownership is not enforced for ordinary callers. + const event = yield* events.publish(DurableMessage, durableData(aggregateID, "unfenced")) + + expect(event.durable?.seq).toBe(1) + }), + ) + it.effect("remove clears durable event sequence", () => Effect.gen(function* () { const events = yield* EventV2.Service From 3c4fbc4067b9328f4cc2746b6ced47f2d79ef50d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 17:41:25 -0700 Subject: [PATCH 054/103] Corrected the remote throughput limit and the fenced-overlap note. The delta-per-transaction limit was wrong: streaming deltas are live-only and write no row, so the remote cost is the bounded set of per-step boundary and tool events, not one transaction per token. Coalescing them would regress the crash-resume granularity, so per-event durability stays. Also updated the retry-overlap note now that each attempt fences the event log. --- packages/temporal/README.md | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 4c108ee4ec6e..baa524dc2cb6 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -147,14 +147,14 @@ later. Verified by `packages/core/test/session-runner-resume.test.ts`. `OPENCODE_SESSION_EXECUTION=temporal`. - Activity bounds: the 10s heartbeat is the liveness bound (worker death re-drives within seconds); `startToCloseTimeout` is a 12-hour backstop for a drain that hangs while its process stays alive. - Known limit: when an attempt is retried while the previous one is still alive (a network - partition, or the backstop firing), the old attempt keeps publishing for a few seconds until its - heartbeat is rejected and the AbortSignal interrupts it; the projector's status guards make - duplicate settlements no-ops in projection, but the overlap window is not fully fenced. The - fence design exists as a follow-up: `event_sequence.owner_id` and `claim()` are already in the - event store; claim the aggregate per attempt and reject stale-owner appends inside the per-event - transaction. It threads an owner through every publish (including compaction's), so it deserves - its own change. + When an attempt is retried while the previous one is still alive (a network partition, or the + backstop firing), the old attempt could briefly keep publishing until its heartbeat is rejected + and the AbortSignal interrupts it. That overlap is now fenced: each drain claims the event log + with an attempt token (`event_sequence.owner_id` via `claim()`), and a live durable append dies + if a newer attempt has since claimed the log (the check is in `event.ts`, gated by the + `EventOwner` context the drain provides). The owner is set activity-side from the run id and + attempt, so it stays out of the workflow's deterministic input; the local driver uses a + per-instance token. The projector's status guards still make any duplicate settlement a no-op. `scripts/resume-check.ts` verifies resume: it resolves on a healthy session and rejects on a failing one with the original tagged error (`LLM.Error`) reconstructed across the boundary. @@ -244,10 +244,16 @@ distinct worker identities. The remote suite needs a server, so it runs only when `OPENCODE_LIBSQL_TEST_URL` is set (e.g. `turso dev --port 8899`, then `OPENCODE_LIBSQL_TEST_URL=http://127.0.0.1:8899`); it skips otherwise. -- Known limit: every durable event is its own transaction, and the engine records text/reasoning - deltas as events, so a streaming turn against a REMOTE store pays one interactive transaction per - delta, serialized per process. Fine for a shared local file; expect reduced streaming throughput - over a network URL until delta events are batched for the remote backend. +- Every durable event is its own interactive transaction, but streaming text and reasoning deltas + are live-only: they broadcast in memory and write no row (verified by `session-runner.test.ts` + "broadcasts provider ... deltas without storing projection rewrites", which streams 32 chunks and + asserts zero delta rows while the context still rebuilds from the log). So a streaming turn does + NOT pay a transaction per token. The durable cost of one step is the handful of boundary and tool + events (`Step.Started`, a `Text.Ended` per block, the `Tool.*` pair per call, `Step.Ended`), each + recorded as it settles. Over a remote store that is a small, bounded number of round trips per + step, not one per token. Coalescing them into a single commit at step end would cut round trips + further, but a mid-step crash would then lose the completed-tool records the resume path reuses, + so per-event durability is kept on purpose. ### What resumes cross-host, and what does not From 28aacd1997bf0e31e9f3e71699d897ac9cdd928a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 11 Aug 2026 23:56:48 -0700 Subject: [PATCH 055/103] Moved the serve-wrapper increment to its own branch. The HTTP wrapper over shipping opencode serve now lives on 2026/08/opencode-temporal-http as its own PR, so this change carries only the v2 engine integration. The package keeps the verification scripts and docs; resume-check still needs the Temporal client. --- bun.lock | 6 -- packages/temporal/README.md | 72 +++-------------------- packages/temporal/package.json | 15 +---- packages/temporal/src/activities.ts | 54 ----------------- packages/temporal/src/crash-demo.ts | 89 ----------------------------- packages/temporal/src/demo.ts | 40 ------------- packages/temporal/src/opencode.ts | 66 --------------------- packages/temporal/src/worker.ts | 28 --------- packages/temporal/src/workflows.ts | 69 ---------------------- 9 files changed, 9 insertions(+), 430 deletions(-) delete mode 100644 packages/temporal/src/activities.ts delete mode 100644 packages/temporal/src/crash-demo.ts delete mode 100644 packages/temporal/src/demo.ts delete mode 100644 packages/temporal/src/opencode.ts delete mode 100644 packages/temporal/src/worker.ts delete mode 100644 packages/temporal/src/workflows.ts diff --git a/bun.lock b/bun.lock index c6df1094397f..bd1736f8be3b 100644 --- a/bun.lock +++ b/bun.lock @@ -951,13 +951,7 @@ "name": "@opencode-ai/temporal", "version": "0.0.0", "dependencies": { - "@temporalio/activity": "^1.11.0", "@temporalio/client": "^1.11.0", - "@temporalio/worker": "^1.11.0", - "@temporalio/workflow": "^1.11.0", - }, - "devDependencies": { - "tsx": "^4.19.0", }, }, "packages/tui": { diff --git a/packages/temporal/README.md b/packages/temporal/README.md index baa524dc2cb6..ade087a29898 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -5,75 +5,17 @@ Temporal workflow, so a coding session survives worker loss, can run detached or background, and can be driven from anywhere by signal. It is a drop-in layer: opencode's loop, tools, model, storage, and HTTP API are untouched. -There are two phases. - -## Phase 1 (this directory): wrap the shipping `opencode serve` - -A workflow owns one session. It creates the session, then drains a queue of prompts, running each -turn as an activity that drives the shipping opencode server over its HTTP API. The conversation, -the prompt queue, and which turns have completed all live in workflow state, so the session is -durable and resumable. The turn itself runs inside opencode (`prompt_async` forks it server-side), -so it keeps running even while the Temporal worker is down; recovery re-attaches by reading the -recorded messages. - -- `src/opencode.ts` — a thin `fetch` client for the opencode HTTP API. -- `src/activities.ts` — `createSession`, `runTurn` (idempotent on the user-message count, so a - retry never double-sends), `abortTurn`. -- `src/workflows.ts` — `durableSession`: create session, drain the prompt queue, one turn per - activity; signals `submitPrompt` / `abortSession` / `closeSession`; query `getState`. -- `src/worker.ts` — the Temporal worker (runs on Node via `tsx`). -- `src/demo.ts` — drive a two-turn session end to end. -- `src/crash-demo.ts` — kill the worker mid-turn and show the session still completes. +A lighter increment exists as its own change: the `2026/08/opencode-temporal-http` branch wraps +the **shipping** `opencode serve` over its HTTP API, for the agent-as-black-box case. It makes the +orchestration durable but cannot recover a partial turn. This change is the deeper one: durability +inside the engine, so a crashed turn resumes mid-step instead of being re-attached to. -### Run it - -```bash -# 1. a Temporal dev server -temporal server start-dev --port 7237 - -# 2. opencode serve with a provider key (any provider opencode supports) -OPENAI_API_KEY=... bun run --cwd packages/opencode --conditions=browser src/index.ts serve --port 4599 - -# 3. the durable-session worker -TEMPORAL_ADDRESS=127.0.0.1:7237 OPENCODE_BASE_URL=http://127.0.0.1:4599 bun run --cwd packages/temporal worker - -# 4. drive a session -TEMPORAL_ADDRESS=127.0.0.1:7237 bun run --cwd packages/temporal demo -``` - -Env: `TEMPORAL_ADDRESS` (default `127.0.0.1:7237`), `TEMPORAL_TASK_QUEUE` (`opencode-durable`), -`OPENCODE_BASE_URL` (`http://127.0.0.1:4599`), `OPENCODE_PROVIDER` (`openai`), `OPENCODE_MODEL` -(`gpt-5-mini`). - -### What it proves - -`bun run --cwd packages/temporal crash-demo` starts a turn, `SIGKILL`s the worker mid-turn, starts -a fresh worker, and checks the outcome. A passing run shows: - -``` -reply: "RECOVERED" -runTurn attempts (max): 2 | started-event attempts: [ 1, 2 ] -user messages in session: 1 -CRASH-RECOVERY: PASS -``` - -That is: the turn completed after the crash, Temporal re-drove the activity on a new worker -(attempt 2), and idempotency kept it to a single prompt (no double-send). - -### Honest limits of Phase 1 - -- It makes the **orchestration** durable (the session, the queue, turn re-drive), not opencode's - in-process turn. If the opencode **server** dies mid-turn, the turn's host side effects can be - partial; re-driving re-attaches to whatever the server recorded. -- The worker talks to the opencode server over unauthenticated HTTP by default. Run them together - or set `OPENCODE_SERVER_PASSWORD` and pass the header. - -## Phase 2 (built): a durable `SessionExecution` on the v2 engine +## A durable `SessionExecution` on the v2 engine opencode's v2 engine (`packages/core` + `packages/server`) is already event-sourced per session and exposes a substitutable `SessionExecution` service (`active` / `resume` / `wake` / `interrupt`) -whose local impl comments "Future remote placement belongs here." Phase 2 provides a Temporal-backed -`SessionExecution` in `packages/core/src/session/execution/`: +whose local impl comments "Future remote placement belongs here." This change provides a +Temporal-backed `SessionExecution` in `packages/core/src/session/execution/`: - `temporal-workflow.ts` — the pure per-session workflow (the Temporal equivalent of `SessionRunCoordinator`: `wake`/`force` drive one drain, wakes coalesce, quiescent runs end). diff --git a/packages/temporal/package.json b/packages/temporal/package.json index 5f478bad923e..ab061b976c87 100644 --- a/packages/temporal/package.json +++ b/packages/temporal/package.json @@ -3,19 +3,8 @@ "version": "0.0.0", "private": true, "type": "module", - "description": "A Temporal durable-execution layer that drives an opencode session as a durable workflow.", - "scripts": { - "worker": "tsx src/worker.ts", - "demo": "tsx src/demo.ts", - "crash-demo": "tsx src/crash-demo.ts" - }, + "description": "Docs and verification scripts for the Temporal-backed v2 SessionExecution.", "dependencies": { - "@temporalio/activity": "^1.11.0", - "@temporalio/client": "^1.11.0", - "@temporalio/worker": "^1.11.0", - "@temporalio/workflow": "^1.11.0" - }, - "devDependencies": { - "tsx": "^4.19.0" + "@temporalio/client": "^1.11.0" } } diff --git a/packages/temporal/src/activities.ts b/packages/temporal/src/activities.ts deleted file mode 100644 index b1c36819dd88..000000000000 --- a/packages/temporal/src/activities.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { heartbeat } from "@temporalio/activity" -import * as oc from "./opencode" - -const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) - -export async function createSession(title?: string): Promise { - return oc.createSession(title) -} - -/** - * Run one turn idempotently and return the assistant's text. - * - * `turnIndex` is the 0-based position of this turn's user message in the session. The workflow - * drives turns strictly in order, so the count of user messages already recorded is the - * idempotency key: post the prompt only when this turn's user message does not exist yet. A - * Temporal retry (e.g. after a worker crash) therefore never double-sends; it just resumes - * polling. Because the turn runs server-side (prompt_async), it keeps going while the worker is - * down, and recovery re-attaches by reading the recorded messages. - */ -export async function runTurn(input: { sessionID: string; turnIndex: number; text: string }): Promise { - const { sessionID, turnIndex, text } = input - - const userCount = (await oc.listMessages(sessionID)).filter((m) => m.info.role === "user").length - if (userCount <= turnIndex) { - await oc.promptAsync(sessionID, text) - } - - // A turn produces several assistant messages (one per step). It is done when the session is idle - // AND this turn's user message has at least one completed assistant message after it; the final - // answer is the last such message's text. - for (;;) { - heartbeat() - const msgs = await oc.listMessages(sessionID) - const users = msgs.filter((m) => m.info.role === "user") - if (users.length > turnIndex && (await oc.isIdle(sessionID))) { - const userCreated = users[turnIndex].info.time?.created ?? 0 - const replies = msgs.filter( - (m) => - m.info.role === "assistant" && - (m.info.time?.created ?? 0) >= userCreated && - m.info.time?.completed, - ) - if (replies.length > 0) { - const last = replies[replies.length - 1] - return oc.assistantText(last) || replies.map(oc.assistantText).filter(Boolean).join("\n") - } - } - await wait(1500) - } -} - -export async function abortTurn(sessionID: string): Promise { - await oc.abort(sessionID) -} diff --git a/packages/temporal/src/crash-demo.ts b/packages/temporal/src/crash-demo.ts deleted file mode 100644 index 6581b3c20dc0..000000000000 --- a/packages/temporal/src/crash-demo.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { spawn, type ChildProcess } from "node:child_process" -import { Client, Connection } from "@temporalio/client" -import { durableSession, submitPrompt, closeSession, getState } from "./workflows" -import * as oc from "./opencode" - -// Self-contained crash-recovery proof. It owns the worker lifecycle: start a turn, KILL the worker -// mid-turn, restart it, and show the workflow still completes, the runTurn activity was re-driven -// (attempt > 1), and no duplicate prompt was sent (exactly one user message). The turn itself keeps -// running in the opencode server while the worker is down; recovery re-attaches to it. - -const ADDR = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" -const QUEUE = process.env.TEMPORAL_TASK_QUEUE ?? "opencode-durable-crash" -const OPENCODE = process.env.OPENCODE_BASE_URL ?? "http://127.0.0.1:4599" - -const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) - -function startWorker(): ChildProcess { - return spawn("bun", ["run", "worker"], { - cwd: process.cwd(), - env: { ...process.env, TEMPORAL_ADDRESS: ADDR, TEMPORAL_TASK_QUEUE: QUEUE, OPENCODE_BASE_URL: OPENCODE }, - stdio: "ignore", - detached: true, // own process group, so we can kill the whole tree (bun -> tsx -> node) - }) -} - -function killWorker(w: ChildProcess): void { - try { - process.kill(-(w.pid as number), "SIGKILL") // negative pid = the group - } catch { - w.kill("SIGKILL") - } -} - -async function main() { - const connection = await Connection.connect({ address: ADDR }) - const client = new Client({ connection }) - - console.log("[1] starting worker A") - let worker = startWorker() - await wait(10_000) // let it bundle + connect - - const workflowId = `crash-${Date.now()}` - const handle = await client.workflow.start(durableSession, { - taskQueue: QUEUE, - workflowId, - args: [{ title: "crash demo" }], - }) - await handle.signal(submitPrompt, "Create a file crash.txt containing exactly RECOVERED, then read it back and reply with only its contents.") - await handle.signal(closeSession) - console.log(`[2] started workflow ${workflowId}, turn in progress`) - - await wait(4_000) // let the turn get going (prompt posted, polling) - console.log("[3] KILLING worker A mid-turn") - killWorker(worker) - - await wait(4_000) // worker down; the turn keeps running server-side - console.log("[4] starting worker B (recovery)") - worker = startWorker() - - console.log("[5] awaiting workflow completion ...") - await handle.result() - const state = await handle.query(getState) - const reply = state.turns[0]?.reply ?? "" - - // Evidence from history: how many attempts did runTurn take? - const events = (await handle.fetchHistory()).events ?? [] - const startedAttempts = events - .filter((e: any) => e.activityTaskStartedEventAttributes) - .map((e: any) => Number(e.activityTaskStartedEventAttributes.attempt ?? 1)) - const maxAttempt = startedAttempts.length ? Math.max(...startedAttempts) : 1 - - // Idempotency: exactly one user message for the one prompt (no double-send on retry). - const userMsgs = (await oc.listMessages(state.sessionID!)).filter((m) => m.info.role === "user").length - - console.log("\n=== RESULT ===") - console.log("reply:", JSON.stringify(reply)) - console.log("runTurn attempts (max):", maxAttempt, "| started-event attempts:", startedAttempts) - console.log("user messages in session:", userMsgs) - const ok = reply.includes("RECOVERED") && maxAttempt >= 2 && userMsgs === 1 - console.log("CRASH-RECOVERY:", ok ? "PASS" : "FAIL") - - killWorker(worker) - process.exit(ok ? 0 : 1) -} - -main().catch((err) => { - console.error(err) - process.exit(1) -}) diff --git a/packages/temporal/src/demo.ts b/packages/temporal/src/demo.ts deleted file mode 100644 index e4d694184731..000000000000 --- a/packages/temporal/src/demo.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Client, Connection } from "@temporalio/client" -import { durableSession, submitPrompt, closeSession, getState } from "./workflows" - -// Drives one durable session: start the workflow, queue prompts by signal, close it, wait, and -// print the recorded conversation. Nothing here holds the turn open; the workflow owns it. -async function main() { - const address = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" - const taskQueue = process.env.TEMPORAL_TASK_QUEUE ?? "opencode-durable" - - const connection = await Connection.connect({ address }) - const client = new Client({ connection }) - - const workflowId = process.env.WORKFLOW_ID ?? `oc-session-${Date.now()}` - const prompts = process.argv.slice(2) - if (prompts.length === 0) { - prompts.push( - "Create a file called note.txt containing exactly the token DURABLE_OK, then confirm.", - "Run `cat note.txt` and reply with only its contents.", - ) - } - - const handle = await client.workflow.start(durableSession, { - taskQueue, - workflowId, - args: [{ title: "durable demo" }], - }) - console.log(`started workflow ${workflowId}`) - - for (const p of prompts) await handle.signal(submitPrompt, p) - await handle.signal(closeSession) - - await handle.result() - const state = await handle.query(getState) - console.log(JSON.stringify(state, null, 2)) -} - -main().catch((err) => { - console.error(err) - process.exit(1) -}) diff --git a/packages/temporal/src/opencode.ts b/packages/temporal/src/opencode.ts deleted file mode 100644 index 991b1239fa29..000000000000 --- a/packages/temporal/src/opencode.ts +++ /dev/null @@ -1,66 +0,0 @@ -// A thin HTTP client for the shipping `opencode serve` API. Used by activities (Node context), -// never from workflow code. The durability layer treats opencode as a black-box server it drives. - -const BASE = () => process.env.OPENCODE_BASE_URL ?? "http://127.0.0.1:4599" -const MODEL = () => ({ - providerID: process.env.OPENCODE_PROVIDER ?? "openai", - modelID: process.env.OPENCODE_MODEL ?? "gpt-5-mini", -}) - -export interface OcMessage { - info: { id: string; role: string; time?: { created?: number; completed?: number } } - parts: Array<{ type: string; text?: string }> -} - -async function api(path: string, init?: RequestInit): Promise { - const res = await fetch(BASE() + path, init) - if (!res.ok) { - const body = await res.text().catch(() => "") - throw new Error(`opencode ${init?.method ?? "GET"} ${path} -> ${res.status} ${body.slice(0, 300)}`) - } - return res -} - -export async function createSession(title?: string): Promise { - const res = await api("/session", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(title ? { title } : {}), - }) - return (await res.json()).id as string -} - -export async function listMessages(sessionID: string): Promise { - const res = await api(`/session/${sessionID}/message`) - return (await res.json()) as OcMessage[] -} - -// Fire-and-return: the server forks the turn and answers 204 immediately, so the turn keeps -// running even if the caller (our worker) dies. Recovery re-attaches by polling listMessages. -export async function promptAsync(sessionID: string, text: string): Promise { - await api(`/session/${sessionID}/prompt_async`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ model: MODEL(), parts: [{ type: "text", text }] }), - }) -} - -export async function abort(sessionID: string): Promise { - await api(`/session/${sessionID}/abort`, { method: "POST" }).catch(() => {}) -} - -// The session is busy for the whole turn (all steps) and idle when it is fully done. Verified: the -// status does not blip idle between steps, so this is a reliable end-of-turn signal. -export async function isIdle(sessionID: string): Promise { - const res = await api("/session/status") - const map = (await res.json()) as Record - return (map[sessionID]?.type ?? "idle") === "idle" -} - -export function assistantText(m: OcMessage): string { - return m.parts - .filter((p) => p.type === "text" && p.text) - .map((p) => p.text) - .join("") - .trim() -} diff --git a/packages/temporal/src/worker.ts b/packages/temporal/src/worker.ts deleted file mode 100644 index 932b09e2ec22..000000000000 --- a/packages/temporal/src/worker.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { fileURLToPath } from "node:url" -import { NativeConnection, Worker } from "@temporalio/worker" -import * as activities from "./activities" - -async function main() { - const address = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" - const namespace = process.env.TEMPORAL_NAMESPACE ?? "default" - const taskQueue = process.env.TEMPORAL_TASK_QUEUE ?? "opencode-durable" - - const connection = await NativeConnection.connect({ address }) - const worker = await Worker.create({ - connection, - namespace, - taskQueue, - workflowsPath: fileURLToPath(new URL("./workflows.ts", import.meta.url)), - activities, - }) - console.log( - `opencode-temporal worker ready: queue=${taskQueue} temporal=${address} ` + - `opencode=${process.env.OPENCODE_BASE_URL ?? "http://127.0.0.1:4599"}`, - ) - await worker.run() -} - -main().catch((err) => { - console.error(err) - process.exit(1) -}) diff --git a/packages/temporal/src/workflows.ts b/packages/temporal/src/workflows.ts deleted file mode 100644 index d719beaa266c..000000000000 --- a/packages/temporal/src/workflows.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { proxyActivities, defineSignal, defineQuery, setHandler, condition } from "@temporalio/workflow" -import type * as activities from "./activities" - -// Unlimited retries with heartbeat: this is what makes a turn survive a worker crash. The turn -// runs server-side, so a re-run just re-attaches (idempotent on the user-message count). -const { runTurn } = proxyActivities({ - startToCloseTimeout: "15 minutes", - // Short heartbeat so a dead worker's in-flight turn is detected and re-driven quickly. The turn - // keeps running server-side meanwhile, so the retry just re-attaches. - heartbeatTimeout: "8 seconds", -}) -// These calls never heartbeat, so they must not carry a heartbeat timeout (any call slower than it -// would fail spuriously). createSession is a non-idempotent POST: a retry after an ambiguous -// failure can orphan a server-side session, so retries are bounded. -const { createSession, abortTurn } = proxyActivities({ - startToCloseTimeout: "1 minute", - retry: { maximumAttempts: 3 }, -}) - -export const submitPrompt = defineSignal<[string]>("submitPrompt") -export const abortSession = defineSignal("abortSession") -export const closeSession = defineSignal("closeSession") -export const getState = defineQuery("getState") - -export interface DurableSessionInput { - title?: string -} - -export interface DurableSessionState { - sessionID?: string - turns: Array<{ text: string; reply: string }> - pending: number -} - -/** - * One durable opencode session. The conversation, the queue of prompts, and which turns have - * completed all live in workflow state, so the session survives worker loss, can run detached or - * in the background, and can be driven from anywhere by signal. The turns still execute in the - * opencode server; this workflow is the durable brain that drives and remembers them. - */ -export async function durableSession(input: DurableSessionInput = {}): Promise { - const queue: string[] = [] - const turns: Array<{ text: string; reply: string }> = [] - let closed = false - - const sessionID = await createSession(input.title) - - setHandler(submitPrompt, (text) => { - queue.push(text) - }) - setHandler(closeSession, () => { - closed = true - }) - setHandler(abortSession, () => { - // Best-effort: tell the server to abort the active turn; the running activity then returns the - // (aborted) assistant message and the loop moves on. A floating rejection would fail the - // workflow task, so it is swallowed. - abortTurn(sessionID).catch(() => {}) - }) - setHandler(getState, () => ({ sessionID, turns, pending: queue.length })) - - for (;;) { - await condition(() => queue.length > 0 || closed) - if (queue.length === 0 && closed) break - const text = queue.shift()! - const reply = await runTurn({ sessionID, turnIndex: turns.length, text }) - turns.push({ text, reply }) - } -} From 35d847ab83989e40b99386bb3d6dfdb91177c402 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 12 Aug 2026 00:20:40 -0700 Subject: [PATCH 056/103] Rebuilt missing worktrees from shared-store snapshot packs. The conversation resumed on any worker, but the project tree stayed host-local, so a turn that edits files could only resume where the tree lived. The runner now ships each captured snapshot tree to the shared store as an incremental git pack, and a drain on a worker without the tree rebuilds it (uncommitted edits and untracked files included) before the run. Ignored files and dependencies still need the project's own bootstrap. --- packages/core/schema.json | 133 +++++++++++++++- packages/core/src/database/migration.gen.ts | 1 + .../migration/20260812070622_snapshot_pack.ts | 26 ++++ packages/core/src/database/schema.gen.ts | 16 ++ packages/core/src/location-services.ts | 2 + packages/core/src/session/execution/drain.ts | 9 +- .../src/session/execution/local-driver.ts | 6 +- .../core/src/session/execution/temporal.ts | 6 +- .../core/src/session/execution/worktree.ts | 146 ++++++++++++++++++ packages/core/src/session/runner/llm.ts | 8 + packages/core/src/snapshot-sync.ts | 130 ++++++++++++++++ packages/core/src/snapshot/sql.ts | 24 +++ .../core/test/worktree-materialize.test.ts | 128 +++++++++++++++ packages/temporal/README.md | 16 +- .../temporal/docs/worktree-portability.md | 33 ++-- 15 files changed, 658 insertions(+), 26 deletions(-) create mode 100644 packages/core/src/database/migration/20260812070622_snapshot_pack.ts create mode 100644 packages/core/src/session/execution/worktree.ts create mode 100644 packages/core/src/snapshot-sync.ts create mode 100644 packages/core/src/snapshot/sql.ts create mode 100644 packages/core/test/worktree-materialize.test.ts diff --git a/packages/core/schema.json b/packages/core/schema.json index f03443e8c216..01aeeffb9540 100644 --- a/packages/core/schema.json +++ b/packages/core/schema.json @@ -1,9 +1,9 @@ { "version": "7", "dialect": "sqlite", - "id": "69462d47-737f-4005-83ff-1f43cf647276", + "id": "c609b4b3-8061-4cef-a339-c44d54121462", "prevIds": [ - "ef05c3dc-ecd1-4def-9573-6b872e58366e" + "69462d47-737f-4005-83ff-1f43cf647276" ], "ddl": [ { @@ -86,6 +86,10 @@ "name": "session_share", "entityType": "tables" }, + { + "name": "snapshot_pack", + "entityType": "tables" + }, { "type": "text", "notNull": false, @@ -1566,6 +1570,86 @@ "entityType": "columns", "table": "session_share" }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "snapshot_pack" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "directory", + "entityType": "columns", + "table": "snapshot_pack" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "worktree", + "entityType": "columns", + "table": "snapshot_pack" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "tree", + "entityType": "columns", + "table": "snapshot_pack" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "base", + "entityType": "columns", + "table": "snapshot_pack" + }, + { + "type": "blob", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "pack", + "entityType": "columns", + "table": "snapshot_pack" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "snapshot_pack" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "snapshot_pack" + }, { "columns": [ "project_id" @@ -1944,6 +2028,15 @@ "table": "session_share", "entityType": "pks" }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "snapshot_pack_pk", + "table": "snapshot_pack", + "entityType": "pks" + }, { "columns": [ { @@ -2285,6 +2378,42 @@ "name": "todo_session_idx", "entityType": "indexes", "table": "todo" + }, + { + "columns": [ + { + "value": "directory", + "isExpression": false + }, + { + "value": "time_created", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "snapshot_pack_directory_idx", + "entityType": "indexes", + "table": "snapshot_pack" + }, + { + "columns": [ + { + "value": "worktree", + "isExpression": false + }, + { + "value": "time_created", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "snapshot_pack_worktree_idx", + "entityType": "indexes", + "table": "snapshot_pack" } ], "renames": [] diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index 6c6c79d16089..c1cfbf9a0c7e 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -42,5 +42,6 @@ export const migrations = ( import("./migration/20260622202450_simplify_session_input"), import("./migration/20260810092511_permission_request"), import("./migration/20260812000509_permission_request_status_idx"), + import("./migration/20260812070622_snapshot_pack"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20260812070622_snapshot_pack.ts b/packages/core/src/database/migration/20260812070622_snapshot_pack.ts new file mode 100644 index 000000000000..e04ccfe38e6a --- /dev/null +++ b/packages/core/src/database/migration/20260812070622_snapshot_pack.ts @@ -0,0 +1,26 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812070622_snapshot_pack", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`snapshot_pack\` ( + \`id\` text PRIMARY KEY, + \`directory\` text NOT NULL, + \`worktree\` text NOT NULL, + \`tree\` text NOT NULL, + \`base\` text, + \`pack\` blob NOT NULL, + \`time_created\` integer NOT NULL, + \`time_updated\` integer NOT NULL + ); + `) + yield* tx.run( + `CREATE INDEX \`snapshot_pack_directory_idx\` ON \`snapshot_pack\` (\`directory\`,\`time_created\`);`, + ) + yield* tx.run(`CREATE INDEX \`snapshot_pack_worktree_idx\` ON \`snapshot_pack\` (\`worktree\`,\`time_created\`);`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/schema.gen.ts b/packages/core/src/database/schema.gen.ts index 78c4351cfd0e..60a72dacac89 100644 --- a/packages/core/src/database/schema.gen.ts +++ b/packages/core/src/database/schema.gen.ts @@ -248,6 +248,18 @@ export default { CONSTRAINT \`fk_session_share_session_id_session_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session\`(\`id\`) ON DELETE CASCADE ); `) + yield* tx.run(` + CREATE TABLE \`snapshot_pack\` ( + \`id\` text PRIMARY KEY, + \`directory\` text NOT NULL, + \`worktree\` text NOT NULL, + \`tree\` text NOT NULL, + \`base\` text, + \`pack\` blob NOT NULL, + \`time_created\` integer NOT NULL, + \`time_updated\` integer NOT NULL + ); + `) yield* tx.run(`CREATE UNIQUE INDEX \`event_aggregate_seq_idx\` ON \`event\` (\`aggregate_id\`,\`seq\`);`) yield* tx.run(`CREATE INDEX \`event_aggregate_type_seq_idx\` ON \`event\` (\`aggregate_id\`,\`type\`,\`seq\`);`) yield* tx.run( @@ -285,6 +297,10 @@ export default { yield* tx.run(`CREATE INDEX \`session_workspace_idx\` ON \`session\` (\`workspace_id\`);`) yield* tx.run(`CREATE INDEX \`session_parent_idx\` ON \`session\` (\`parent_id\`);`) yield* tx.run(`CREATE INDEX \`todo_session_idx\` ON \`todo\` (\`session_id\`);`) + yield* tx.run( + `CREATE INDEX \`snapshot_pack_directory_idx\` ON \`snapshot_pack\` (\`directory\`,\`time_created\`);`, + ) + yield* tx.run(`CREATE INDEX \`snapshot_pack_worktree_idx\` ON \`snapshot_pack\` (\`worktree\`,\`time_created\`);`) }) }, } satisfies Omit diff --git a/packages/core/src/location-services.ts b/packages/core/src/location-services.ts index 7da67673c319..c3b43b7ddde2 100644 --- a/packages/core/src/location-services.ts +++ b/packages/core/src/location-services.ts @@ -30,6 +30,7 @@ import { SessionTodo } from "./session/todo" import { SkillV2 } from "./skill" import { SkillGuidance } from "./skill/guidance" import { Snapshot } from "./snapshot" +import { SnapshotSync } from "./snapshot-sync" import { SystemContextBuiltIns } from "./system-context/builtins" import { SystemContextRegistry } from "./system-context/registry" import { BuiltInTools } from "./tool/builtins" @@ -75,6 +76,7 @@ export const locationServices = LayerNode.group([ BuiltInTools.node, SessionRunnerModel.node, Snapshot.node, + SnapshotSync.node, SessionRunnerLLM.node, ]) diff --git a/packages/core/src/session/execution/drain.ts b/packages/core/src/session/execution/drain.ts index 06fa68aa984b..e6f60970552d 100644 --- a/packages/core/src/session/execution/drain.ts +++ b/packages/core/src/session/execution/drain.ts @@ -8,6 +8,7 @@ import type { LocationServiceMap } from "../../location-service-map" import type { Location } from "../../location" import type { LocationError, LocationServices } from "../../location-services" import { EventV2 } from "../../event" +import { WorktreeMaterializer } from "./worktree" import { SessionRunner } from "../runner" import { SessionSchema } from "../schema" import { SessionStore } from "../store" @@ -24,9 +25,11 @@ export interface DrainDeps { readonly ctx: Context.Context /** Used to claim the event log for the running attempt so a superseded one is fenced. */ readonly events: EventV2.Interface + /** Rebuilds a missing project worktree from stored snapshot packs before the run. */ + readonly worktrees: WorktreeMaterializer.Interface } -export const makeDrains = ({ store, locations, ctx, events }: DrainDeps) => { +export const makeDrains = ({ store, locations, ctx, events, worktrees }: DrainDeps) => { const drain = async (input: DrainInput, signal: AbortSignal): Promise => { const exit = await Effect.runPromiseExit( Effect.gen(function* () { @@ -34,6 +37,8 @@ export const makeDrains = ({ store, locations, ctx, events }: DrainDeps) => { if (!session) return // Take the event log before running so a superseded attempt's later appends are fenced. if (input.owner) yield* events.claim(session.id, input.owner) + // A worker resuming on a host without the project tree rebuilds it from snapshot packs. + yield* worktrees.ensure(session.location.directory) yield* SessionRunner.Service.use((runner) => runner.run({ sessionID: session.id, force: input.force }), ).pipe(Effect.provide(locations.get(session.location))) @@ -81,6 +86,8 @@ export const makeDrains = ({ store, locations, ctx, events }: DrainDeps) => { if (!session) return { ran: false, continue: false, step: input.step, promotion: null } // Take the event log before running so a superseded attempt's later appends are fenced. if (input.owner) yield* events.claim(session.id, input.owner) + // A worker resuming on a host without the project tree rebuilds it from snapshot packs. + yield* worktrees.ensure(session.location.directory) const r = yield* SessionRunner.Service.use((runner) => runner.runStep({ sessionID: session.id, diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts index e8fc57ff6958..26b2135503cd 100644 --- a/packages/core/src/session/execution/local-driver.ts +++ b/packages/core/src/session/execution/local-driver.ts @@ -17,6 +17,7 @@ import { SessionSchema } from "../schema" import { SessionStore } from "../store" import { SessionExecution } from "../execution" import { makeDrains } from "./drain" +import { WorktreeMaterializer } from "./worktree" import { makeWorkflows, type WorkflowRuntime } from "./workflow-core" import { toRunError } from "./run-error-codec" @@ -177,7 +178,8 @@ const layer = Layer.effect( const locations = yield* LocationServiceMap.Service const ctx = yield* Effect.context() const events = yield* EventV2.Service - const drains = makeDrains({ store, locations, ctx, events }) + const worktrees = yield* WorktreeMaterializer.Service + const drains = makeDrains({ store, locations, ctx, events, worktrees }) const drivers = new Map() // Read at layer build (not module load) so tests can set it before constructing the layer. // The idle override shortens the supervisor's 5-minute self-termination. @@ -242,5 +244,5 @@ const layer = Layer.effect( export const node = makeGlobalNode({ service: SessionExecution.Service, layer, - deps: [SessionStore.node, LocationServiceMap.node, EventV2.node], + deps: [SessionStore.node, LocationServiceMap.node, EventV2.node, WorktreeMaterializer.node], }) diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index dc2fef01723b..c79658e0fbbe 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -13,6 +13,7 @@ import { SessionStore } from "../store" import { SessionExecution } from "../execution" import { makeActivities, makeStepActivities } from "./temporal-activities" import { makeDrains } from "./drain" +import { WorktreeMaterializer } from "./worktree" import { toRunError } from "./run-error-codec" import * as WF from "./temporal-workflow" @@ -54,10 +55,11 @@ const layer = Layer.effect( // SessionRunner and all of its dependencies. const ctx = yield* Effect.context() const events = yield* EventV2.Service + const worktrees = yield* WorktreeMaterializer.Service // The drain bodies are shared with the in-process micro-driver (drain.ts), so turn semantics // and error encoding cannot differ between drivers. - const { drain, stepDrain } = makeDrains({ store, locations, ctx, events }) + const { drain, stepDrain } = makeDrains({ store, locations, ctx, events, worktrees }) // Worker connection (native) hosts the runContinuation activity + the workflow. Skipped in // client-only role so serve can run without an embedded worker. @@ -205,5 +207,5 @@ const layer = Layer.effect( export const node = makeGlobalNode({ service: SessionExecution.Service, layer, - deps: [SessionStore.node, LocationServiceMap.node, EventV2.node], + deps: [SessionStore.node, LocationServiceMap.node, EventV2.node, WorktreeMaterializer.node], }) diff --git a/packages/core/src/session/execution/worktree.ts b/packages/core/src/session/execution/worktree.ts new file mode 100644 index 000000000000..060af00577c7 --- /dev/null +++ b/packages/core/src/session/execution/worktree.ts @@ -0,0 +1,146 @@ +export * as WorktreeMaterializer from "./worktree" + +// Rebuilds a missing project worktree from the shared store before a drain runs. This closes the +// host-local gap in cross-host resume: file tools need the tree, and a fresh worker does not have +// it. The capture side (snapshot-sync.ts) ships each snapshot as an incremental git pack; this +// side indexes every pack for the worktree into a fresh repo and checks out the newest tree. +// Ignored files and dependencies are not captured, so a bootstrap step (install, build) stays the +// project's own concern. + +import { rm, writeFile } from "node:fs/promises" +import path from "path" +import { Cause, Context, Effect, Layer } from "effect" +import { ChildProcess } from "effect/unstable/process" +import { asc, desc, eq } from "drizzle-orm" +import { Database } from "../../database/database" +import { makeGlobalNode } from "../../effect/app-node" +import { KeyedMutex } from "../../effect/keyed-mutex" +import { FSUtil } from "../../fs-util" +import { Git } from "../../git" +import { AppProcess } from "../../process" +import { AbsolutePath } from "../../schema" +import { SnapshotPackTable } from "../../snapshot/sql" + +export interface Interface { + /** + * Make sure the session's directory exists, rebuilding its worktree from stored snapshot packs + * when it does not. A directory with no stored packs, or one whose worktree root already + * exists, is left alone. Never fails the caller. + */ + readonly ensure: (directory: string) => Effect.Effect +} + +export class Service extends Context.Service()( + "@opencode/v2/WorktreeMaterializer", +) {} + +const layer = Layer.effect( + Service, + Effect.gen(function* () { + const fs = yield* FSUtil.Service + const git = yield* Git.Service + const proc = yield* AppProcess.Service + const { db } = yield* Database.Service + const locks = KeyedMutex.makeUnsafe() + + const materialize = Effect.fnUntraced(function* (tip: typeof SnapshotPackTable.$inferSelect) { + const worktree = AbsolutePath.make(tip.worktree) + const repository = yield* git.repo + .create({ worktree, gitDirectory: AbsolutePath.make(path.join(worktree, ".git")) }) + .pipe(Effect.orDie) + // Index every pack shipped for this worktree; objects accumulate, the newest tree wins. + const rows = yield* db + .select() + .from(SnapshotPackTable) + .where(eq(SnapshotPackTable.worktree, tip.worktree)) + .orderBy(asc(SnapshotPackTable.time_created)) + .all() + .pipe(Effect.orDie) + const packDirectory = path.join(repository.gitDirectory, "objects", "pack") + yield* fs.ensureDir(packDirectory).pipe(Effect.orDie) + for (const row of rows) { + const packFile = path.join(packDirectory, `pack-${row.id}.pack`) + yield* Effect.promise(() => writeFile(packFile, row.pack)) + const indexed = yield* proc + .run( + ChildProcess.make("git", ["--git-dir", repository.gitDirectory, "index-pack", packFile], { + cwd: worktree, + extendEnv: true, + }), + ) + .pipe(Effect.orDie) + if (indexed.exitCode !== 0) + return yield* Effect.die(new Error(`index-pack: ${indexed.stderr.toString("utf8")}`)) + } + yield* git.tree.checkout({ repository, tree: Git.TreeID.make(tip.tree) }).pipe(Effect.orDie) + // Point HEAD at the sync commit so the rebuilt repo reads as a clean checkout, not an + // unborn branch over a full untracked tree. Cosmetic; the files above are what matter. + yield* proc + .run( + ChildProcess.make( + "git", + ["--git-dir", repository.gitDirectory, "update-ref", "refs/heads/opencode-restore", tip.id], + { cwd: worktree, extendEnv: true }, + ), + ) + .pipe(Effect.ignore) + yield* proc + .run( + ChildProcess.make( + "git", + ["--git-dir", repository.gitDirectory, "symbolic-ref", "HEAD", "refs/heads/opencode-restore"], + { cwd: worktree, extendEnv: true }, + ), + ) + .pipe(Effect.ignore) + yield* Effect.logInfo("materialized worktree from snapshot packs", { + worktree: tip.worktree, + packs: rows.length, + tree: tip.tree, + }) + }) + + const ensure = Effect.fn("WorktreeMaterializer.ensure")(function* (directory: string) { + if (yield* fs.existsSafe(directory)) return + // The newest capture whose session ran in this directory decides which worktree to rebuild. + const tip = yield* db + .select() + .from(SnapshotPackTable) + .where(eq(SnapshotPackTable.directory, directory)) + .orderBy(desc(SnapshotPackTable.time_created)) + .limit(1) + .get() + .pipe(Effect.orDie) + if (!tip) return + yield* locks.withLock(tip.worktree)( + Effect.gen(function* () { + // Re-check inside the lock (a concurrent drain may have rebuilt it), and never touch a + // worktree root that already exists: something else owns that tree. + if (yield* fs.existsSafe(tip.worktree)) return + yield* materialize(tip).pipe( + Effect.catchCauseIf( + (cause) => !Cause.hasInterrupts(cause), + (cause) => + Effect.gen(function* () { + // A half-built tree would pass the exists check forever; remove what we created. + yield* Effect.promise(() => rm(tip.worktree, { recursive: true, force: true })) + yield* Effect.logWarning("failed to materialize worktree", { + worktree: tip.worktree, + cause, + }) + }), + ), + ) + }), + ) + }) + + return Service.of({ ensure }) + }), +) + +export const node = makeGlobalNode({ + service: Service, + layer, + deps: [Database.node, FSUtil.node, Git.node, AppProcess.node], +}) diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index ca8305318f9c..f49205e2b8ef 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -39,6 +39,7 @@ import { toLLMMessages } from "./to-llm-message" import { MAX_STEPS_PROMPT } from "./max-steps" import { DEFAULT_MAX_STEPS, REPEAT_LIMIT, REPEATED_CALLS_PROMPT, trailingIdenticalToolSteps } from "./loop-guard" import { Snapshot } from "../../snapshot" +import { SnapshotSync } from "../../snapshot-sync" import { makeLocationNode } from "../../effect/app-node" import { llmClient } from "../../effect/app-node-platform" @@ -107,6 +108,7 @@ const layer = Layer.effect( const referenceGuidance = yield* ReferenceGuidance.Service const config = yield* Config.Service const snapshots = yield* Snapshot.Service + const snapshotSync = yield* SnapshotSync.Service const db = (yield* Database.Service).db const compaction = SessionCompaction.make({ events, llm, config: yield* config.entries() }) const getSession = Effect.fn("SessionRunner.getSession")(function* (sessionID: SessionSchema.ID) { @@ -247,6 +249,8 @@ const layer = Layer.effect( if (yield* compaction.compactIfNeeded({ sessionID: session.id, entries, model, request })) return yield* Effect.die(continueAfterCompaction(currentStep)) const startSnapshot = yield* snapshots.capture() + // Ship the pre-step tree so another host can rebuild the worktree; best-effort inside push. + if (startSnapshot) yield* snapshotSync.push(startSnapshot) const publisher = createLLMEventPublisher(events, { sessionID: session.id, agent: agent.id, @@ -348,6 +352,8 @@ const layer = Layer.effect( const stepSettlement = publisher.stepSettlement() if (stepSettlement && !publisher.hasProviderError()) { const endSnapshot = yield* snapshots.capture() + // Ship the post-step tree: this is the state a resumed step on another host needs. + if (endSnapshot) yield* snapshotSync.push(endSnapshot) const files = startSnapshot && endSnapshot ? yield* snapshots @@ -506,6 +512,7 @@ const layer = Layer.effect( yield* failInterruptedTools(input.sessionID) const startSnapshot = inFlight.snapshot?.start const endSnapshot = yield* snapshots.capture() + if (endSnapshot) yield* snapshotSync.push(endSnapshot) const files = startSnapshot && endSnapshot ? yield* snapshots @@ -606,6 +613,7 @@ export const node = makeLocationNode({ ReferenceGuidance.node, Config.node, Snapshot.node, + SnapshotSync.node, Database.node, ], }) diff --git a/packages/core/src/snapshot-sync.ts b/packages/core/src/snapshot-sync.ts new file mode 100644 index 000000000000..edab2106d124 --- /dev/null +++ b/packages/core/src/snapshot-sync.ts @@ -0,0 +1,130 @@ +export * as SnapshotSync from "./snapshot-sync" + +// Ships captured snapshot trees to the shared store as git packs, so a worker on another host can +// rebuild the project worktree before it drains a session (see session/execution/worktree.ts). +// Each push wraps the tree in a sync commit chained onto the previous push and packs only the +// delta. Best-effort by design: a failed push degrades portability, never the turn. + +import { readFile, rm } from "node:fs/promises" +import os from "node:os" +import path from "path" +import { Cause, Context, Effect, Layer } from "effect" +import { ChildProcess } from "effect/unstable/process" +import { desc, eq } from "drizzle-orm" +import { Database } from "./database/database" +import { makeLocationNode } from "./effect/app-node" +import { FSUtil } from "./fs-util" +import { Git } from "./git" +import { Global } from "./global" +import { Location } from "./location" +import { AppProcess } from "./process" +import { AbsolutePath } from "./schema" +import type { Snapshot } from "./snapshot" +import { SnapshotPackTable } from "./snapshot/sql" +import { Hash } from "./util/hash" + +export interface Interface { + /** Ship a captured tree to the shared store as an incremental pack. Never fails the caller. */ + readonly push: (tree: Snapshot.ID) => Effect.Effect +} + +export class Service extends Context.Service()("@opencode/v2/SnapshotSync") {} + +const layer = Layer.effect( + Service, + Effect.gen(function* () { + const fs = yield* FSUtil.Service + const git = yield* Git.Service + const global = yield* Global.Service + const location = yield* Location.Service + const proc = yield* AppProcess.Service + const { db } = yield* Database.Service + const source = yield* git.repo.discover(location.project.directory) + const worktree = source + ? AbsolutePath.make(yield* fs.realPath(source.worktree).pipe(Effect.orDie)) + : location.project.directory + // The same side git-dir Snapshot.capture writes trees into. + const gitDirectory = path.join(global.data, "snapshot", location.project.id, Hash.fast(worktree)) + + const run = (args: string[], stdin?: string) => + proc.run( + ChildProcess.make("git", ["--git-dir", gitDirectory, "--work-tree", worktree, ...args], { + cwd: worktree, + env: { + GIT_AUTHOR_NAME: "opencode", + GIT_AUTHOR_EMAIL: "opencode@sync", + GIT_COMMITTER_NAME: "opencode", + GIT_COMMITTER_EMAIL: "opencode@sync", + }, + extendEnv: true, + }), + { stdin }, + ) + + const push = Effect.fn("SnapshotSync.push")(function* (tree: Snapshot.ID) { + yield* Effect.gen(function* () { + if (!source) return + const latest = yield* db + .select() + .from(SnapshotPackTable) + .where(eq(SnapshotPackTable.worktree, worktree)) + .orderBy(desc(SnapshotPackTable.time_created)) + .limit(1) + .get() + .pipe(Effect.orDie) + // The newest shipped state already is this tree: nothing to pack. + if (latest?.tree === tree) return + // Chain onto the previous sync commit only when this host has it; a base absent locally + // would produce a delta pack the pack builder cannot compute. + const base = + latest && + (yield* run(["cat-file", "-e", `${latest.id}^{commit}`]).pipe(Effect.orDie)).exitCode === 0 + ? latest.id + : undefined + const committed = yield* run([ + "commit-tree", + tree, + ...(base ? ["-p", base] : []), + "-m", + "opencode snapshot sync", + ]).pipe(Effect.orDie) + if (committed.exitCode !== 0) + return yield* Effect.die(new Error(`commit-tree: ${committed.stderr.toString("utf8")}`)) + const commit = committed.stdout.toString("utf8").trim() + const prefix = path.join(os.tmpdir(), `opencode-snapshot-${commit.slice(0, 12)}`) + const packed = yield* run( + ["pack-objects", "--revs", "-q", prefix], + `${commit}\n${base ? `^${base}\n` : ""}`, + ).pipe(Effect.orDie) + if (packed.exitCode !== 0) + return yield* Effect.die(new Error(`pack-objects: ${packed.stderr.toString("utf8")}`)) + const packHash = packed.stdout.toString("utf8").trim() + const packFile = `${prefix}-${packHash}.pack` + const bytes = yield* Effect.promise(() => readFile(packFile)) + yield* Effect.promise(() => Promise.allSettled([rm(packFile), rm(`${prefix}-${packHash}.idx`)])) + yield* db + .insert(SnapshotPackTable) + .values([ + { id: commit, directory: location.directory, worktree, tree, base: base ?? null, pack: bytes }, + ]) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + }).pipe( + Effect.catchCauseIf( + (cause) => !Cause.hasInterrupts(cause), + (cause) => Effect.logWarning("failed to ship snapshot pack", { tree, cause }), + ), + Effect.asVoid, + ) + }) + + return Service.of({ push }) + }), +) + +export const node = makeLocationNode({ + service: Service, + layer, + deps: [Database.node, FSUtil.node, Git.node, Global.node, Location.node, AppProcess.node], +}) diff --git a/packages/core/src/snapshot/sql.ts b/packages/core/src/snapshot/sql.ts new file mode 100644 index 000000000000..541423a578f7 --- /dev/null +++ b/packages/core/src/snapshot/sql.ts @@ -0,0 +1,24 @@ +import { blob, index, sqliteTable, text } from "drizzle-orm/sqlite-core" +import { Timestamps } from "../database/schema.sql" + +// A captured snapshot tree shipped as a git pack, so a worker on another host can rebuild the +// project worktree from the shared store before it drains a session. `id` is the sync commit that +// wraps the tree; `base` is the sync commit the pack was built against (null means a full pack). +// `directory` is the session location the capture ran in (the materializer's lookup key); +// `worktree` is the project root the tree checks out into. +export const SnapshotPackTable = sqliteTable( + "snapshot_pack", + { + id: text().primaryKey(), + directory: text().notNull(), + worktree: text().notNull(), + tree: text().notNull(), + base: text(), + pack: blob({ mode: "buffer" }).notNull(), + ...Timestamps, + }, + (table) => [ + index("snapshot_pack_directory_idx").on(table.directory, table.time_created), + index("snapshot_pack_worktree_idx").on(table.worktree, table.time_created), + ], +) diff --git a/packages/core/test/worktree-materialize.test.ts b/packages/core/test/worktree-materialize.test.ts new file mode 100644 index 000000000000..55274f129acd --- /dev/null +++ b/packages/core/test/worktree-materialize.test.ts @@ -0,0 +1,128 @@ +// Cross-host worktree materialization: the capture side ships snapshot trees as git packs into +// the shared store; a worker on a host without the project tree rebuilds it before draining. One +// DB file, two independent stacks: "host A" captures and pushes, the worktree is deleted to +// simulate a fresh host, "host B" materializes it back from the store alone. +import { describe, expect } from "bun:test" +import { $ } from "bun" +import { realpathSync } from "node:fs" +import { mkdir, readFile, rm, writeFile } from "node:fs/promises" +import path from "path" +import { asc } from "drizzle-orm" +import { Effect, Layer } from "effect" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { Database } from "@opencode-ai/core/database/database" +import { Global } from "@opencode-ai/core/global" +import { Location } from "@opencode-ai/core/location" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { Snapshot } from "@opencode-ai/core/snapshot" +import { SnapshotSync } from "@opencode-ai/core/snapshot-sync" +import { SnapshotPackTable } from "@opencode-ai/core/snapshot/sql" +import { WorktreeMaterializer } from "@opencode-ai/core/session/execution/worktree" +import { testEffect } from "./lib/effect" +import { tmpdir } from "./fixture/tmpdir" + +const it = testEffect(Layer.empty) + +// The capturing host: real Location resolution against the git worktree, its own data dir for the +// side snapshot repo, the shared DB file. +const captureStack = (file: string, worktree: string, data: string) => + AppNodeBuilder.build(LayerNode.group([Snapshot.node, SnapshotSync.node]), [ + [Database.node, Database.layerFromPath(file)], + [Location.node, Location.boundNode({ directory: AbsolutePath.make(worktree) })], + [Global.node, Layer.succeed(Global.Service, Global.make({ data }))], + ]) + +// The resuming host: only the shared store, no location, no snapshot repo, no worktree. +const materializeStack = (file: string) => + AppNodeBuilder.build(WorktreeMaterializer.node, [[Database.node, Database.layerFromPath(file)]]) + +describe("WorktreeMaterializer", () => { + it.live("rebuilds a deleted worktree from shared-store packs, incremental chain included", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const root = realpathSync(tmp.path) + const worktree = path.join(root, "project") + const file = path.join(root, "shared.db") + yield* Effect.promise(async () => { + await mkdir(worktree, { recursive: true }) + await $`git init -q ${worktree}`.quiet() + await $`git -C ${worktree} config user.email t@t`.quiet() + await $`git -C ${worktree} config user.name t`.quiet() + await writeFile(path.join(worktree, "tracked.txt"), "v1\n") + await $`git -C ${worktree} add .`.quiet() + await $`git -C ${worktree} commit -qm seed`.quiet() + // The state to port: an uncommitted edit and a file git never saw. + await writeFile(path.join(worktree, "tracked.txt"), "v2\n") + await writeFile(path.join(worktree, "untracked.txt"), "notes\n") + }) + + const A = yield* Layer.build(captureStack(file, worktree, path.join(root, "host-a-data"))) + const first = yield* Snapshot.Service.use((s) => s.capture()).pipe(Effect.provide(A)) + if (!first) throw new Error("expected the first capture to produce a tree") + yield* SnapshotSync.Service.use((s) => s.push(first)).pipe(Effect.provide(A)) + + // A second increment on top, so materialization has to index a base pack plus a delta pack. + yield* Effect.sleep(10) + yield* Effect.promise(async () => { + await writeFile(path.join(worktree, "tracked.txt"), "v3\n") + await writeFile(path.join(worktree, "extra.txt"), "more\n") + }) + const second = yield* Snapshot.Service.use((s) => s.capture()).pipe(Effect.provide(A)) + if (!second) throw new Error("expected the second capture to produce a tree") + yield* SnapshotSync.Service.use((s) => s.push(second)).pipe(Effect.provide(A)) + + const rows = yield* Database.Service.use(({ db }) => + db.select().from(SnapshotPackTable).orderBy(asc(SnapshotPackTable.time_created)).all(), + ).pipe(Effect.orDie, Effect.provide(Database.layerFromPath(file)), Effect.scoped) + expect(rows).toHaveLength(2) + expect(rows[1]?.base).toBe(rows[0]!.id) + + // The fresh host: the tree is gone, only the shared store remains. + yield* Effect.promise(() => rm(worktree, { recursive: true, force: true })) + const B = yield* Layer.build(materializeStack(file)) + yield* WorktreeMaterializer.Service.use((w) => w.ensure(worktree)).pipe(Effect.provide(B)) + + const [tracked, untracked, extra] = yield* Effect.promise(() => + Promise.all([ + readFile(path.join(worktree, "tracked.txt"), "utf8"), + readFile(path.join(worktree, "untracked.txt"), "utf8"), + readFile(path.join(worktree, "extra.txt"), "utf8"), + ]), + ) + expect(tracked).toBe("v3\n") + expect(untracked).toBe("notes\n") + expect(extra).toBe("more\n") + + // A second ensure on an existing tree is a no-op, not a rebuild. + yield* WorktreeMaterializer.Service.use((w) => w.ensure(worktree)).pipe(Effect.provide(B)) + expect(yield* Effect.promise(() => readFile(path.join(worktree, "tracked.txt"), "utf8"))).toBe( + "v3\n", + ) + + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + // The shared-store deployment uses the libsql backend, so the pack blob has to survive that + // driver's parameter path too, not only bun's. + it.live("round-trips a pack blob through the libsql backend", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(realpathSync(tmp.path), "libsql.db") + const bytes = Buffer.from([0x50, 0x41, 0x43, 0x4b, 0x00, 0x01, 0xff, 0xfe, 0x00, 0x7f]) + const layer = Database.layerFromLibsql(`file:${file}`) + yield* Database.Service.use(({ db }) => + db + .insert(SnapshotPackTable) + .values([{ id: "c".repeat(40), directory: "/w", worktree: "/w", tree: "t".repeat(40), pack: bytes }]) + .run(), + ).pipe(Effect.orDie, Effect.provide(layer), Effect.scoped) + const row = yield* Database.Service.use(({ db }) => + db.select().from(SnapshotPackTable).get(), + ).pipe(Effect.orDie, Effect.provide(layer), Effect.scoped) + expect(Buffer.from(row!.pack).equals(bytes)).toBeTrue() + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) +}) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index ade087a29898..3a12a4beac0c 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -205,15 +205,17 @@ on any worker: messages, tool results (the bounded preview and structured output prompt attachments (stored inline as `data:` URIs in the prompt), and credentials (`CredentialTable`) all ride the shared store. +**The project working tree** now rides the store too. After each step capture the runner ships +the snapshot tree as an incremental git pack (`snapshot-sync.ts`, `snapshot_pack` table). Before a +drain runs, a worker missing the session's directory rebuilds the worktree from those packs +(`session/execution/worktree.ts`): uncommitted edits and untracked files included, checked out at +the same absolute path it was captured at (a uniform fleet layout). Ignored files and dependencies +are not captured, so a rebuilt tree may need an install step before `bash` behaves identically. +[docs/worktree-portability.md](docs/worktree-portability.md) covers the design and the +affinity/shared-volume alternatives that skip materialization latency on warm paths. + Host-local state that does NOT ride the DB, so it is not reconstructed on a different host: -- **The project working tree.** File-touching tools (read/edit/bash) operate on the local worktree, so - a turn that keeps editing files must resume on a worker that has that worktree. This is the one real - cross-host correctness constraint. Three ways to satisfy it: co-locate a session's workers by worktree - (session affinity via a per-worktree Temporal task queue), share the worktree (a networked - filesystem), or reconstruct it from the last snapshot on resume (needs a shared snapshot store). - [docs/worktree-portability.md](docs/worktree-portability.md) weighs the three; the recommendation is - per-worktree task queues, with snapshot reconstruction as the long-term path. - **The snapshot store (`${data}/snapshot`) and the retained full tool-output files (`${data}/tool-output`).** The runner never reads these to rebuild context: snapshot file-diffs are best-effort (`Effect.catch` to `undefined`), and the model sees the bounded tool-output preview, not diff --git a/packages/temporal/docs/worktree-portability.md b/packages/temporal/docs/worktree-portability.md index fac52132686f..22c2efedf587 100644 --- a/packages/temporal/docs/worktree-portability.md +++ b/packages/temporal/docs/worktree-portability.md @@ -38,20 +38,29 @@ network filesystems are slow and occasionally surprising, and two sessions shari collide across hosts just as they can within one (a session's own tools stay serialized either way, one activity at a time). -## C. Reconstruct the worktree from snapshots (long-term) +## C. Reconstruct the worktree from snapshots (implemented) The engine already captures git-tree snapshots around each step (`Step.Started`/`Step.Ended` carry -snapshot ids); today they live in a local per-project git store (`${data}/snapshot`). Point that -store at shared storage and a worker without the worktree can materialize the session's exact file -state on resume: clone the repository, then check out the last recorded snapshot tree. This is the -only option that gives true any-worker resume including uncommitted changes. Its honest limits: -materialization latency on first touch, and snapshots capture the git tree, not the world around it -(ignored files, dependencies, running processes), so a reconstructed worktree may still need a -dependency install before `bash` behaves identically. +snapshot ids). Those trees now also ride the shared store: after each capture the runner ships the +tree as a git pack (`snapshot-sync.ts` into the `snapshot_pack` table), incremental against the +previous shipped state. Before a drain runs, the worker checks the session's directory and, when +it is missing, rebuilds the worktree from the stored packs +(`session/execution/worktree.ts`): a fresh repo, every pack indexed, the newest tree checked out, +uncommitted edits and untracked files included. Verified by +`packages/core/test/worktree-materialize.test.ts` (capture on one stack, delete the tree, +materialize from the store alone on a second stack). + +Honest limits: the tree is rebuilt at the same absolute path it was captured at (a uniform fleet +layout, containers in practice); snapshots capture the git tree, not the world around it (ignored +files, dependencies, running processes), so a reconstructed worktree may still need a dependency +install before `bash` behaves identically; and shipping is best-effort on the capture side, so a +worker that dies between the last capture and its edits loses those edits, exactly as it would +have lost them locally. ## Recommendation -Ship **A** as the deployment default (small change, no new infra, correct by construction), allow -**B** where shared volumes already exist, and treat **C** as the future enhancement that removes -the affinity constraint entirely. A and C compose: affinity routes the common case to the warm -worktree; snapshot reconstruction lets a cold worker join the queue after materializing. +**C is on by default**: any worker can pick up any session and materialize the tree it needs. +Layer **A** on top when worktrees are large or hot (affinity routes the common case to the warm +worktree and skips materialization latency); use **B** where shared volumes already exist. A and C +compose: affinity serves the warm path, snapshot reconstruction lets a cold worker join after +materializing. From b21f64d88dd96735bc8b79490d5de74bc5ad67e7 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 12 Aug 2026 12:18:51 -0700 Subject: [PATCH 057/103] Added a how-it-fits-together overview to the README. Reviewers landed on mechanism-by-mechanism detail with no single narrative. The overview states the one design decision (supervisor written once, driver chosen by config) and the six consequences, each linking to its section. It also gives the loop bounds a documented home. --- packages/temporal/README.md | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 3a12a4beac0c..01f0ac3c043b 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -10,6 +10,44 @@ the **shipping** `opencode serve` over its HTTP API, for the agent-as-black-box orchestration durable but cannot recover a partial turn. This change is the deeper one: durability inside the engine, so a crashed turn resumes mid-step instead of being re-attached to. +## How it fits together + +One design decision carries the change: the session supervisor is written once, and durability is +a choice of driver. Everything else here is a consequence of taking at-least-once execution +seriously. + +The supervisor (`workflow-core.ts`) drives a session over six runtime primitives. The Temporal +driver runs it as a per-session workflow with one activity per step; the in-process micro-driver +(`local-driver.ts`) runs the same function over plain promises. One env var picks the driver, and +the shared drain bodies keep turn semantics identical in both modes (see +[Two modes, one supervisor](#two-modes-one-supervisor)). + +Taking durability seriously then forces six things: + +1. **State must be shareable.** Any worker resumes any session only if the event log is not + host-local: the libSQL backend, atomic remote writes, busy retry, serialized migrations + ([Shared, durable event store](#shared-durable-event-store-any-worker-resume)). +2. **Re-drives must be safe.** A retried step reuses completed tool results, re-runs only tools + declared idempotent, and fails the rest for the model to redo. The step loop is bounded + (`loop-guard.ts`: a step ceiling plus a repeated-identical-call detector), because a runaway + turn would otherwise be a durable runaway turn + ([Two modes, one supervisor](#two-modes-one-supervisor)). +3. **Two writers must be fenced.** A superseded attempt cannot keep appending to the log; each + drain claims the log with an attempt token ([Notes](#notes)). +4. **The worktree must travel.** Snapshot trees ship as incremental git packs, and a worker + without the project tree rebuilds it before the run + ([What resumes cross-host](#what-resumes-cross-host-and-what-does-not), + [docs/worktree-portability.md](docs/worktree-portability.md)). +5. **Human-in-the-loop must be durable.** A pending permission ask is a row in the shared store, + answerable from any process, adopted by re-drives, expired when abandoned + ([Durable permission asks](#durable-permission-asks)). +6. **Errors must survive the boundary.** `resume` rejects with the exact tagged `RunError` + reconstructed from the failure details, and a user decline is non-retryable ([Notes](#notes)). + +Operationally, workers scale separately from the HTTP server +([Running workers separately](#running-workers-separately)), and `active` is backed by Temporal +visibility, so it survives restarts. + ## A durable `SessionExecution` on the v2 engine opencode's v2 engine (`packages/core` + `packages/server`) is already event-sourced per session and From 994a9ab46b7fdb882aaccc08e2b615d1be2e0cb5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 12 Aug 2026 13:10:12 -0700 Subject: [PATCH 058/103] Aligned the docs and scripts with the per-step reality. The worktree note still framed the closed gap as an open decision and named affinity the default. resume-check drove the retired whole-turn workflow type, so it verified a path serve never runs; it now targets sessionTurn. Script labels and a stale phase reference updated. --- packages/temporal/docs/worktree-portability.md | 17 ++++++++--------- packages/temporal/scripts/resume-check.ts | 2 +- .../temporal/scripts/shared-store-failover.sh | 2 +- packages/temporal/scripts/v2-crash-test.sh | 8 ++++---- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/temporal/docs/worktree-portability.md b/packages/temporal/docs/worktree-portability.md index 22c2efedf587..799261261335 100644 --- a/packages/temporal/docs/worktree-portability.md +++ b/packages/temporal/docs/worktree-portability.md @@ -3,17 +3,16 @@ ## Problem The shared event store makes the **conversation** resumable on any worker: history, tool results, -attachments, and credentials are all rebuilt from the DB. The one thing that is not in the DB is the -**project working tree**. File-touching tools (`bash`, `read`, `edit`, `write`, `apply_patch`, -`glob`, `grep`) operate on `Location.directory`, a local filesystem path. A worker that picks up a -session without that worktree resumes the conversation correctly and then acts on a missing (or -wrong) directory. Mid-session uncommitted changes make this worse: they exist only on the disk of -the worker that made them, so even a fresh clone of the repository is not the session's real state. +attachments, and credentials are all rebuilt from the DB. The **project working tree** is the one +piece of session state that lives outside it. File-touching tools (`bash`, `read`, `edit`, +`write`, `apply_patch`, `glob`, `grep`) operate on `Location.directory`, a local filesystem path, +and mid-session uncommitted changes exist only on the disk of the worker that made them, so even +a fresh clone of the repository is not the session's real state. -This is a deployment/design decision, not a bug fix. Three ways to satisfy it, in order of -recommendation. +Option **C** below closes this and is on by default. **A** and **B** remain as deployment choices +that avoid materialization cost on warm paths. -## A. Session affinity: one task queue per worktree (recommended default) +## A. Session affinity: one task queue per worktree (warm-path optimization) Temporal-native and no new infrastructure. Derive the task queue from the worktree identity (`opencode-session-exec@`); a worker registers on the queues for the worktrees whose diff --git a/packages/temporal/scripts/resume-check.ts b/packages/temporal/scripts/resume-check.ts index 195216a69bfc..33835d3cadbf 100644 --- a/packages/temporal/scripts/resume-check.ts +++ b/packages/temporal/scripts/resume-check.ts @@ -29,7 +29,7 @@ async function prompt(sid: string, text: string): Promise { } async function resume(client: Client, sid: string): Promise { - const startOp = new WithStartWorkflowOperation("sessionExecution", { + const startOp = new WithStartWorkflowOperation("sessionTurn", { taskQueue: QUEUE, workflowId: `session-exec-${sid}`, args: [sid], diff --git a/packages/temporal/scripts/shared-store-failover.sh b/packages/temporal/scripts/shared-store-failover.sh index 946810a2755b..1119f2f5f788 100644 --- a/packages/temporal/scripts/shared-store-failover.sh +++ b/packages/temporal/scripts/shared-store-failover.sh @@ -81,6 +81,6 @@ python3 - <<'PY' import json ev=json.load(open("/tmp/failover-wf.json")).get("events",[]) ids=sorted(set(e["activityTaskStartedEventAttributes"].get("identity") for e in ev if e.get("activityTaskStartedEventAttributes"))) -print(" runContinuation ran on worker identities:", ids) +print(" step activities ran on worker identities:", ids) PY kill_port 4602 \ No newline at end of file diff --git a/packages/temporal/scripts/v2-crash-test.sh b/packages/temporal/scripts/v2-crash-test.sh index 8af57be3ff74..9406e9cd16c0 100755 --- a/packages/temporal/scripts/v2-crash-test.sh +++ b/packages/temporal/scripts/v2-crash-test.sh @@ -1,9 +1,9 @@ #!/bin/bash -# Engine-level crash recovery for the v2 Temporal SessionExecution (Phase 2). +# Engine-level crash recovery for the v2 Temporal SessionExecution. # # Kills the whole v2 server (which co-hosts the embedded Temporal worker) mid-turn, restarts it, -# and shows the turn still completes: Temporal re-drives the runContinuation activity, and -# SessionRunner.run re-reads the durable event log and continues from where it stopped. +# and shows the turn still completes: Temporal re-drives the in-flight step activity, and the +# runner re-reads the durable event log and continues from where it stopped. # # Prereqs: a Temporal dev server on :7237, an OpenAI key at $OPENCODE_KEY_FILE (default # ~/.config/ai363/llm.key), and the v2 server run with OPENCODE_SESSION_EXECUTION=temporal. @@ -63,7 +63,7 @@ python3 - <<'PY' import json ev=json.load(open("/tmp/v2wf.json")).get("events",[]) attempts=[int(e["activityTaskStartedEventAttributes"].get("attempt",1)) for e in ev if e.get("activityTaskStartedEventAttributes")] -print(" runContinuation attempts:", attempts, "| max:", max(attempts) if attempts else 0) +print(" step activity attempts:", attempts, "| max:", max(attempts) if attempts else 0) PY echo "RESULT: turn completed post-crash = $DONE" [[ "$DONE" == yes ]] || exit 1 From 48cd02c4d06de7b4d0a987cf4a2cdb28bb5a74e5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 12 Aug 2026 13:10:12 -0700 Subject: [PATCH 059/103] Removed the unreachable whole-turn execution path. The factory only starts the per-step sessionTurn supervisor, and keeping the whole-turn workflow "for executions already running" is void on a branch whose history starts at the base commit. One execution story remains; the runtime interface is back to six primitives. --- packages/core/src/session/execution/drain.ts | 69 +++---------- .../src/session/execution/local-driver.ts | 2 - .../session/execution/temporal-activities.ts | 49 ++------- .../session/execution/temporal-workflow.ts | 12 +-- .../core/src/session/execution/temporal.ts | 13 ++- .../src/session/execution/workflow-core.ts | 99 ++++--------------- packages/temporal/README.md | 16 +-- 7 files changed, 57 insertions(+), 203 deletions(-) diff --git a/packages/core/src/session/execution/drain.ts b/packages/core/src/session/execution/drain.ts index e6f60970552d..74a069a40ff1 100644 --- a/packages/core/src/session/execution/drain.ts +++ b/packages/core/src/session/execution/drain.ts @@ -1,6 +1,6 @@ -// The drain bodies shared by every durable executor: the Temporal layer runs them inside -// activities, the in-process micro-driver calls them directly. One implementation, so the turn -// semantics and the error encoding cannot differ between drivers. +// The drain body shared by every durable executor: the Temporal layer runs it inside an activity, +// the in-process micro-driver calls it directly. One implementation, so the turn semantics and +// the error encoding cannot differ between drivers. import { Cause, Context, Effect, Exit, type LayerMap } from "effect" import { ApplicationFailure } from "@temporalio/activity" @@ -15,7 +15,7 @@ import { SessionStore } from "../store" import { SessionRunDeclinedError } from "../error" import type { SessionInput } from "../input" import { encodeRunError } from "./run-error-codec" -import type { DrainInput, StepDrainInput, StepDrainResult } from "./temporal-activities" +import type { StepDrainInput, StepDrainResult } from "./temporal-activities" export interface DrainDeps { readonly store: SessionStore.Interface @@ -30,55 +30,7 @@ export interface DrainDeps { } export const makeDrains = ({ store, locations, ctx, events, worktrees }: DrainDeps) => { - const drain = async (input: DrainInput, signal: AbortSignal): Promise => { - const exit = await Effect.runPromiseExit( - Effect.gen(function* () { - const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) - if (!session) return - // Take the event log before running so a superseded attempt's later appends are fenced. - if (input.owner) yield* events.claim(session.id, input.owner) - // A worker resuming on a host without the project tree rebuilds it from snapshot packs. - yield* worktrees.ensure(session.location.directory) - yield* SessionRunner.Service.use((runner) => - runner.run({ sessionID: session.id, force: input.force }), - ).pipe(Effect.provide(locations.get(session.location))) - }).pipe(Effect.provideService(EventV2.EventOwner, input.owner), Effect.provide(ctx), Effect.scoped), - { signal }, - ) - if (Exit.isSuccess(exit)) return - const cause = exit.cause - if (Cause.hasInterruptsOnly(cause)) { - // Two interrupt sources: driver cancellation (the AbortSignal fired -- rethrow its reason so - // the attempt records Cancelled, not Failed) and an internal halt like a user declining a - // permission (the signal did NOT fire). The latter must be non-retryable, or the supervisor - // re-drives a turn the user explicitly stopped. - if (signal.aborted) - throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") - const declined = encodeRunError( - new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make(input.sessionID) }), - ) - throw ApplicationFailure.create({ - message: "session run halted (user declined)", - type: "SessionRunDeclined", - nonRetryable: true, - details: declined === undefined ? undefined : [declined], - }) - } - // A genuine run error is thrown non-retryable so Temporal surfaces it (to resume) rather than - // retrying; only crashes / task timeouts (never thrown here) go through the retry policy. The - // error is encoded faithfully in `details` so the caller can reconstruct the exact RunError. - const squashed = Cause.squash(cause) as { _tag?: string; message?: string } - const encoded = encodeRunError(squashed) - throw ApplicationFailure.create({ - message: squashed?.message ?? Cause.pretty(cause), - type: squashed?._tag ?? "SessionRunError", - nonRetryable: true, - details: encoded === undefined ? undefined : [encoded], - }) - } - - // Per-step drain: run exactly one step of the turn (used by the per-step supervisor). Same - // context and error encoding as the whole-turn drain; returns the next loop state. + // Run exactly one step of the turn (the supervisor loops it); returns the next loop state. const stepDrain = async (input: StepDrainInput, signal: AbortSignal): Promise => { const exit = await Effect.runPromiseExit( Effect.gen(function* () { @@ -104,8 +56,10 @@ export const makeDrains = ({ store, locations, ctx, events, worktrees }: DrainDe if (Exit.isSuccess(exit)) return exit.value const cause = exit.cause if (Cause.hasInterruptsOnly(cause)) { - // Same split as the whole-turn drain: cancellation rethrows its reason (records Cancelled), - // an internal user-decline halt is non-retryable. + // Two interrupt sources: driver cancellation (the AbortSignal fired; rethrow its reason so + // the attempt records Cancelled, not Failed) and an internal halt like a user declining a + // permission (the signal did NOT fire). The latter must be non-retryable, or the supervisor + // re-drives a turn the user explicitly stopped. if (signal.aborted) throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") const declined = encodeRunError( @@ -118,6 +72,9 @@ export const makeDrains = ({ store, locations, ctx, events, worktrees }: DrainDe details: declined === undefined ? undefined : [declined], }) } + // A genuine run error is thrown non-retryable so Temporal surfaces it (to resume) rather than + // retrying; only crashes / task timeouts (never thrown here) go through the retry policy. The + // error is encoded faithfully in `details` so the caller can reconstruct the exact RunError. const squashed = Cause.squash(cause) as { _tag?: string; message?: string } const encoded = encodeRunError(squashed) throw ApplicationFailure.create({ @@ -128,5 +85,5 @@ export const makeDrains = ({ store, locations, ctx, events, worktrees }: DrainDe }) } - return { drain, stepDrain } + return { stepDrain } } diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts index 26b2135503cd..dc6769b27046 100644 --- a/packages/core/src/session/execution/local-driver.ts +++ b/packages/core/src/session/execution/local-driver.ts @@ -88,8 +88,6 @@ class SessionDriver { setUpdateHandler: (name, handler) => this.updateHandlers.set(name, handler), // Same 12 h backstop as the Temporal activity: a hung tool must not hold `draining` forever. // The abort reason is a LocalCancellation, so a timed-out drain looks like any other cancel. - runContinuation: (input) => - this.withBackstop((signal) => drains.drain({ ...input, owner: this.owner }, signal)), runTurnStep: (input) => this.withBackstop((signal) => drains.stepDrain({ ...input, owner: this.owner }, signal)), cancelCurrentScope: () => this.cancel(), diff --git a/packages/core/src/session/execution/temporal-activities.ts b/packages/core/src/session/execution/temporal-activities.ts index 74879b08099d..18cd94b619fe 100644 --- a/packages/core/src/session/execution/temporal-activities.ts +++ b/packages/core/src/session/execution/temporal-activities.ts @@ -1,5 +1,5 @@ -// The runContinuation activity: it runs one drain (SessionRunner.run for the whole turn) by calling -// the `drain` closure the layer captured over the app's Effect context. It heartbeats so a worker +// The runTurnStep activity: one step of a turn (one provider attempt + its tools), run through the +// `stepDrain` closure the layer captured over the app's Effect context. It heartbeats so a worker // crash is detected quickly, and forwards Temporal cancellation as an AbortSignal so an interrupt // turns into Effect fiber interruption inside the runner. @@ -13,49 +13,16 @@ function ownerToken(): string { return `${info.workflowExecution?.runId ?? info.workflowType}#${info.attempt}` } -export interface DrainInput { - sessionID: string - force: boolean - // The attempt that owns the event log while this drain runs. Set activity-side from the run id - // and attempt so it stays out of the workflow's deterministic input. - owner?: string -} - -export type Activities = { - runContinuation(input: DrainInput): Promise -} - -export function makeActivities( - drain: (input: DrainInput, signal: AbortSignal) => Promise, -): Activities { - return { - async runContinuation(input) { - const beat = setInterval(() => { - try { - heartbeat() - } catch { - // heartbeat outside an activity context is a no-op for our purposes - } - }, 3000) - try { - await drain({ ...input, owner: ownerToken() }, Context.current().cancellationSignal) - } finally { - clearInterval(beat) - } - }, - } -} - -// Per-step activities: one runTurnStep activity = one step -// (one provider attempt + its tools). The workflow loops it, so each step is its own activity with -// its own retry/timeout/visibility. `promotion` is null (not undefined) so it serializes cleanly. +// The workflow loops runTurnStep, so each step is its own activity with its own +// retry/timeout/visibility. `promotion` is null (not undefined) so it serializes cleanly. export interface StepDrainInput { sessionID: string step: number promotion: string | null first: boolean force: boolean - // Set activity-side (see ownerToken), not part of the workflow's deterministic input. + // The attempt that owns the event log while this drain runs. Set activity-side (see + // ownerToken), so it stays out of the workflow's deterministic input. owner?: string } @@ -78,7 +45,9 @@ export function makeStepActivities( const beat = setInterval(() => { try { heartbeat() - } catch {} + } catch { + // heartbeat outside an activity context is a no-op for our purposes + } }, 3000) try { return await stepDrain({ ...input, owner: ownerToken() }, Context.current().cancellationSignal) diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index 7746c80b13a8..ae49a6f50d0b 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -1,7 +1,7 @@ // The Temporal driver for the session supervisor. The supervisor itself lives in workflow-core.ts // and is written once; this file adapts the real SDK's primitives (condition, signal/update -// handlers, activity proxies, cancellation) to the WorkflowRuntime interface and exports the two -// workflow functions the worker registers. The in-process driver (local-driver.ts) runs the SAME +// handlers, activity proxies, cancellation) to the WorkflowRuntime interface and exports the +// workflow function the worker registers. The in-process driver (local-driver.ts) runs the SAME // supervisor with plain promises. // // MUST stay sandbox-safe: Temporal bundles this in an isolated context, so no `effect`, no @@ -17,7 +17,7 @@ import { CancellationScope, isCancellation, } from "@temporalio/workflow" -import type { Activities, StepActivities } from "./temporal-activities" +import type { StepActivities } from "./temporal-activities" import { makeWorkflows, type WorkflowRuntime } from "./workflow-core" const activityOptions = { @@ -32,7 +32,6 @@ const activityOptions = { retry: { maximumAttempts: 100 }, } as const -const { runContinuation } = proxyActivities(activityOptions) const { runTurnStep } = proxyActivities(activityOptions) export const wake = defineSignal("wake") @@ -51,7 +50,6 @@ const runtime: WorkflowRuntime = { }, setSignalHandler: (name, handler) => setHandler(signals[name], handler), setUpdateHandler: (_name, handler) => setHandler(resume, handler), - runContinuation, runTurnStep, cancelCurrentScope: () => CancellationScope.current().cancel(), isCancellation, @@ -60,10 +58,6 @@ const runtime: WorkflowRuntime = { const workflows = makeWorkflows(runtime) -export async function sessionExecution(sessionID: string): Promise { - return workflows.sessionExecution(sessionID) -} - export async function sessionTurn(sessionID: string): Promise { return workflows.sessionTurn(sessionID) } diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index c79658e0fbbe..bec58d9a68df 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -11,7 +11,7 @@ import { makeGlobalNode } from "../../effect/app-node" import { SessionSchema } from "../schema" import { SessionStore } from "../store" import { SessionExecution } from "../execution" -import { makeActivities, makeStepActivities } from "./temporal-activities" +import { makeStepActivities } from "./temporal-activities" import { makeDrains } from "./drain" import { WorktreeMaterializer } from "./worktree" import { toRunError } from "./run-error-codec" @@ -22,8 +22,7 @@ const NAMESPACE = process.env.TEMPORAL_NAMESPACE ?? "default" const TASK_QUEUE = process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? "opencode-session-exec" const workflowId = (id: string) => `session-exec-${id}` -// One activity per step (the model call + its tools), with the step loop as workflow control -// flow. The whole-turn sessionExecution workflow stays exported for workflows already running. +// One activity per step (the model call + its tools), with the step loop as workflow control flow. const WORKFLOW = WF.sessionTurn const WORKFLOW_TYPE = "sessionTurn" @@ -59,9 +58,9 @@ const layer = Layer.effect( // The drain bodies are shared with the in-process micro-driver (drain.ts), so turn semantics // and error encoding cannot differ between drivers. - const { drain, stepDrain } = makeDrains({ store, locations, ctx, events, worktrees }) + const { stepDrain } = makeDrains({ store, locations, ctx, events, worktrees }) - // Worker connection (native) hosts the runContinuation activity + the workflow. Skipped in + // Worker connection (native) hosts the runTurnStep activity + the workflow. Skipped in // client-only role so serve can run without an embedded worker. if (HOST_WORKER) { const nativeConn = yield* Effect.acquireRelease( @@ -74,7 +73,7 @@ const layer = Layer.effect( namespace: NAMESPACE, taskQueue: TASK_QUEUE, workflowsPath: fileURLToPath(new URL("./temporal-workflow.ts", import.meta.url)), - activities: { ...makeActivities(drain), ...makeStepActivities(stepDrain) }, + activities: makeStepActivities(stepDrain), }), ) const runHandle = worker.run() @@ -137,7 +136,7 @@ const layer = Layer.effect( const found = yield* Effect.promise(async () => { const ids: SessionSchema.ID[] = [] for await (const wf of client.workflow.list({ - query: `(WorkflowType = 'sessionExecution' OR WorkflowType = 'sessionTurn') AND ExecutionStatus = 'Running'`, + query: `WorkflowType = 'sessionTurn' AND ExecutionStatus = 'Running'`, })) { if (wf.workflowId.startsWith(SESSION_PREFIX)) { ids.push(SessionSchema.ID.make(wf.workflowId.slice(SESSION_PREFIX.length))) diff --git a/packages/core/src/session/execution/workflow-core.ts b/packages/core/src/session/execution/workflow-core.ts index c19ba42a2e86..1d142e788254 100644 --- a/packages/core/src/session/execution/workflow-core.ts +++ b/packages/core/src/session/execution/workflow-core.ts @@ -12,7 +12,7 @@ // forced drain and returns its result to the caller (throwing the run's error). The supervisor // stays alive to serve later wakes/resumes and terminates after an idle period. -import type { DrainInput, StepDrainInput, StepDrainResult } from "./temporal-activities" +import type { StepDrainInput, StepDrainResult } from "./temporal-activities" /** What a driver must provide. Six primitives; everything else is supervisor logic. */ export interface WorkflowRuntime { @@ -20,9 +20,7 @@ export interface WorkflowRuntime { readonly condition: (predicate: () => boolean, timeout?: string) => Promise readonly setSignalHandler: (name: "wake" | "interrupt", handler: () => void) => void readonly setUpdateHandler: (name: "resume", handler: () => Promise) => void - /** One whole-turn drain (SessionRunner.run). The Temporal driver runs it as an activity. */ - readonly runContinuation: (input: DrainInput) => Promise - /** One step of a turn (SessionRunner.runStep), for the per-step variant. */ + /** One step of a turn (SessionRunner.runStep). The Temporal driver runs it as an activity. */ readonly runTurnStep: (input: StepDrainInput) => Promise /** Cancel the in-flight drain and any parked condition (interrupt semantics). */ readonly cancelCurrentScope: () => void @@ -46,89 +44,19 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => // lost across the boundary. const MAX_DRAINS_PER_RUN = options?.maxDrainsPerRun ?? 30 - async function sessionExecution(sessionID: string): Promise { - let pendingWake = true // started by a wake -> there is work to drain - let stopping = false - let draining = false - let handlers = 0 - - // Serialize drains, like the coordinator (one owner fiber per session at a time). Re-check - // after every wakeup: two waiters parked on the same condition can both observe `!draining` - // in one activation, and without the loop both would start a drain. - const drainOnce = async (force: boolean) => { - for (;;) { - await rt.condition(() => !draining || stopping) - if (stopping) return - if (!draining) break - } - draining = true - try { - await rt.runContinuation({ sessionID, force }) - } finally { - draining = false - } - } - - rt.setSignalHandler("wake", () => { - pendingWake = true - }) - rt.setSignalHandler("interrupt", () => { - stopping = true - rt.cancelCurrentScope() - }) - // resume = coordinator.run: force one drain and surface its result (a run error rejects the - // update, so the caller observes it). - rt.setUpdateHandler("resume", async () => { - handlers++ - try { - await drainOnce(true) - } finally { - handlers-- - } - }) - - // interrupt cancels the whole scope, so a cancellation can surface at the idle wait itself, - // not just inside a drain; treat it as a normal stop rather than a failure. - let drains = 0 - try { - for (;;) { - const gotWork = await rt.condition(() => pendingWake || stopping, IDLE_TIMEOUT) - if (stopping) return - if (!gotWork) { - // A wake can race the idle timer; without this re-check it would be dropped. - if (pendingWake) continue - // Idle: terminate only when nothing is in flight. A later wake/resume starts a fresh run. - if (!draining && handlers === 0) return - continue - } - pendingWake = false - try { - await drainOnce(false) - } catch (e) { - // wake tolerates run errors (the coordinator logs and moves on); only cancellation stops us. - if (rt.isCancellation(e)) return - } - drains++ - if (rt.continueAsNew && drains >= MAX_DRAINS_PER_RUN && handlers === 0) - await rt.continueAsNew(sessionID) - } - } catch (e) { - if (rt.isCancellation(e)) return - throw e - } - } - - // Per-step variant: identical lifecycle, but a turn is driven one step at a time -- each step - // (one provider attempt + its tools) is its own drain call, and the step loop is supervisor - // control flow. The loop state (step / promotion / first) mirrors SessionRunner.run's loop. + // The turn is driven one step at a time: each step (one provider attempt + its tools) is its + // own drain call, and the step loop is supervisor control flow. The loop state + // (step / promotion / first) mirrors SessionRunner.run's loop. async function sessionTurn(sessionID: string): Promise { - let pendingWake = true + let pendingWake = true // started by a wake -> there is work to drain let stopping = false let draining = false let handlers = 0 const drainTurn = async (force: boolean) => { - // Same re-check loop as drainOnce: a single wakeup must admit a single drain. + // Serialize drains, like the coordinator (one owner fiber per session at a time). Re-check + // after every wakeup: two waiters parked on the same condition can both observe `!draining` + // in one activation, and without the loop both would start a drain. for (;;) { await rt.condition(() => !draining || stopping) if (stopping) return @@ -158,6 +86,8 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => stopping = true rt.cancelCurrentScope() }) + // resume = coordinator.run: force one drain and surface its result (a run error rejects the + // update, so the caller observes it). rt.setUpdateHandler("resume", async () => { handlers++ try { @@ -167,13 +97,17 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => } }) + // interrupt cancels the whole scope, so a cancellation can surface at the idle wait itself, + // not just inside a drain; treat it as a normal stop rather than a failure. let drains = 0 try { for (;;) { const gotWork = await rt.condition(() => pendingWake || stopping, IDLE_TIMEOUT) if (stopping) return if (!gotWork) { + // A wake can race the idle timer; without this re-check it would be dropped. if (pendingWake) continue + // Idle: terminate only when nothing is in flight. A later wake/resume starts a fresh run. if (!draining && handlers === 0) return continue } @@ -181,6 +115,7 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => try { await drainTurn(false) } catch (e) { + // wake tolerates run errors (the coordinator logs and moves on); only cancellation stops us. if (rt.isCancellation(e)) return } drains++ @@ -193,5 +128,5 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => } } - return { sessionExecution, sessionTurn } + return { sessionTurn } } diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 01f0ac3c043b..eb5427d32a7a 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -57,11 +57,12 @@ Temporal-backed `SessionExecution` in `packages/core/src/session/execution/`: - `temporal-workflow.ts` — the pure per-session workflow (the Temporal equivalent of `SessionRunCoordinator`: `wake`/`force` drive one drain, wakes coalesce, quiescent runs end). -- `temporal-activities.ts` — the `runContinuation` activity (heartbeats; forwards cancellation). +- `temporal-activities.ts` — the `runTurnStep` activity (heartbeats; forwards cancellation; + injects the attempt's event-log owner token). - `temporal.ts` — the `SessionExecution` layer + node: `wake` → `signalWithStart`, `resume` → - forced `signalWithStart`, `interrupt` → cancel signal; the drain is the local coordinator's body - (`SessionRunner.run`) run in the activity against the durable event log. The Temporal client and - an embedded worker are co-hosted in the server process (both run under bun). + forced `signalWithStart`, `interrupt` → cancel signal; each drain runs one step of the local + coordinator's loop (`SessionRunner.runStep`) in an activity against the durable event log. The + Temporal client and an embedded worker are co-hosted in the server process (both run under bun). Wiring is one binding in `packages/server/src/routes.ts`, opt-in via `OPENCODE_SESSION_EXECUTION=temporal`. Because turn state lives in the event log, the workflow stays @@ -85,7 +86,8 @@ session runs as a Temporal workflow `session-exec-`. `TEMPORAL_V2_OK`), recorded as a completed per-session workflow. - Engine-level crash recovery (`scripts/v2-crash-test.sh`): killing the whole server (with its embedded worker) mid-turn, then restarting, still completes the turn. Temporal re-drives - `runContinuation` (attempt 2), the run continues from the event log, and the workflow completes. + the in-flight step activity (attempt 2), the run continues from the event log, and the workflow + completes. ### Two modes, one supervisor @@ -98,7 +100,7 @@ tools) is its own activity with its own retry/timeout/visibility. It reuses `Ses (one iteration of `run`'s loop), so the turn semantics are unchanged. Verified: a create-then-read-then-reply turn recorded three `runTurnStep` activities under a `sessionTurn` workflow and completed. (Earlier whole-turn-per-activity and stock-coordinator modes were folded -away; the whole-turn workflow stays exported for executions already running.) +away.) A per-step re-drive resumes from the durable event log rather than re-running work. `runStep` closes any tool left dangling by an interrupted attempt on every entry, not just the first. Without that, a @@ -185,7 +187,7 @@ The `question` tool still uses an in-process deferred and needs the same treatme The v2 engine event-sources each session to a SQLite store. By default that is a local file, so a session can only be resumed on the host holding the file. Point every worker at one **shared** store -and any worker resumes any session: Temporal load-balances `runContinuation` across the fleet, +and any worker resumes any session: Temporal load-balances `runTurnStep` across the fleet, `active` is visibility-backed, and the resumed worker reads the session purely from the shared log. - **Same host**: set `OPENCODE_DB` to one absolute path on all workers (WAL + `busy_timeout` allow From ec9e4f828bc66660263681354de3b9bf6c44e190 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 12 Aug 2026 13:23:34 -0700 Subject: [PATCH 060/103] Cleaned the docs' voice; fixed a stale mode count. Em-dash bullets, one filler word, and a repeated rhetorical opener. The AI-399 intro still described three execution modes; the fork has two since the factory collapse. --- packages/temporal/README.md | 12 ++++++------ packages/temporal/docs/ai399-local-options.md | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index eb5427d32a7a..deb08009d64c 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -22,7 +22,7 @@ driver runs it as a per-session workflow with one activity per step; the in-proc the shared drain bodies keep turn semantics identical in both modes (see [Two modes, one supervisor](#two-modes-one-supervisor)). -Taking durability seriously then forces six things: +That forces six things: 1. **State must be shareable.** Any worker resumes any session only if the event log is not host-local: the libSQL backend, atomic remote writes, busy retry, serialized migrations @@ -55,11 +55,11 @@ exposes a substitutable `SessionExecution` service (`active` / `resume` / `wake` whose local impl comments "Future remote placement belongs here." This change provides a Temporal-backed `SessionExecution` in `packages/core/src/session/execution/`: -- `temporal-workflow.ts` — the pure per-session workflow (the Temporal equivalent of +- `temporal-workflow.ts`: the pure per-session workflow (the Temporal equivalent of `SessionRunCoordinator`: `wake`/`force` drive one drain, wakes coalesce, quiescent runs end). -- `temporal-activities.ts` — the `runTurnStep` activity (heartbeats; forwards cancellation; +- `temporal-activities.ts`: the `runTurnStep` activity (heartbeats; forwards cancellation; injects the attempt's event-log owner token). -- `temporal.ts` — the `SessionExecution` layer + node: `wake` → `signalWithStart`, `resume` → +- `temporal.ts`: the `SessionExecution` layer + node: `wake` → `signalWithStart`, `resume` → forced `signalWithStart`, `interrupt` → cancel signal; each drain runs one step of the local coordinator's loop (`SessionRunner.runStep`) in an activity against the durable event log. The Temporal client and an embedded worker are co-hosted in the server process (both run under bun). @@ -178,8 +178,8 @@ stopped. Tool-originated asks have deterministic ids (session + callID + action re-driven activity adopts the same pending row instead of filing a duplicate, and an approval that landed while the asker was dead short-circuits the retry (a one-time approve is honored across re-drives without a saved rule). Graceful shutdown retires the process's pending asks as `expired` -and a revived attempt flips them back to pending; after a hard crash the pending row simply feeds -the retry. A pending ask whose session is abandoned lingers in the list until a reply retires it. +and a revived attempt flips them back to pending; after a hard crash the pending row feeds the +retry. A pending ask whose session is abandoned lingers in the list until a reply retires it. Verified by `packages/core/test/permission-durable.test.ts` (two independent stacks over one store). The `question` tool still uses an in-process deferred and needs the same treatment. diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index 722d25ae3f13..815102d6d670 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -3,8 +3,8 @@ Some customers integrate Temporal into their agents and want the same agent loop to run without Temporal, e.g. shipped desktop software. This evaluates the candidate paths and recommends what to tell customers and what to build. It lives on this branch because the fork around it is first-hand -evidence: one engine, three execution modes (in-process, one activity per turn, one per step), -selected by an env var. +evidence: one engine, two execution modes (in-process, one Temporal activity per step), selected +by an env var. ## The requirement, sharpened From e69e48a6fc2cfb27b4b88354302e72eb95956e7f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 12 Aug 2026 21:58:02 -0700 Subject: [PATCH 061/103] Waited out the catalog's async population in model resolution. Location plugins fill the catalog after layer startup, so a prompt landing right after boot observed an empty catalog and failed a healthy model with a non-retryable ModelUnavailableError. Resolution now waits, bounded, while the catalog has no models at all; a populated catalog resolves with no delay. --- packages/core/src/session/runner/model.ts | 8 +++- .../core/test/session-runner-model.test.ts | 48 ++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/packages/core/src/session/runner/model.ts b/packages/core/src/session/runner/model.ts index 74e78120c20e..3d5a056a33ff 100644 --- a/packages/core/src/session/runner/model.ts +++ b/packages/core/src/session/runner/model.ts @@ -186,7 +186,13 @@ export const locationLayer = Layer.effect( const integrations = yield* Integration.Service return Service.of({ resolve: Effect.fn("SessionRunnerModel.resolve")(function* (session) { - // Location plugins populate and filter the catalog asynchronously during layer startup. + // Location plugins populate and filter the catalog asynchronously during layer startup, so + // a resolve that lands right after boot can observe an empty catalog and fail a healthy + // model as unavailable; that failure is non-retryable by design. Wait, bounded, while the + // catalog has no models at all. A populated catalog resolves with no delay. + for (let attempt = 0; attempt < 40 && (yield* catalog.model.all()).length === 0; attempt++) { + yield* Effect.sleep(250) + } const defaultModel = session.model ? undefined : yield* catalog.model.default() const selected = session.model ? (yield* catalog.model.available()).find( diff --git a/packages/core/test/session-runner-model.test.ts b/packages/core/test/session-runner-model.test.ts index 49bbce95a381..830341858005 100644 --- a/packages/core/test/session-runner-model.test.ts +++ b/packages/core/test/session-runner-model.test.ts @@ -1,8 +1,9 @@ import { describe, expect } from "bun:test" import { LLM } from "@opencode-ai/llm" import { LLMClient } from "@opencode-ai/llm/route" -import { DateTime, Effect } from "effect" +import { DateTime, Effect, Layer } from "effect" import { Headers } from "effect/unstable/http" +import { Catalog } from "@opencode-ai/core/catalog" import { Credential } from "@opencode-ai/core/credential" import { Integration } from "@opencode-ai/core/integration" import { ModelV2 } from "@opencode-ai/core/model" @@ -344,4 +345,49 @@ describe("SessionRunnerModel", () => { expect(SessionRunnerModel.supported(model({ type: "native", settings: {} }))).toBe(false) }), ) + + it.live("waits out the catalog's async population instead of failing the model", () => + Effect.gen(function* () { + const catalogModel = model({ type: "aisdk", package: "@ai-sdk/openai", url: "https://openai.example/v1" }) + // Starts empty, like the catalog right after boot; plugins fill it a moment later. + const models: ModelV2.Info[] = [] + const catalogMock = Layer.mock(Catalog.Service, { + model: { + all: () => Effect.sync(() => [...models]), + available: () => Effect.sync(() => [...models]), + default: () => Effect.succeed(undefined), + }, + provider: { + get: () => Effect.succeed(undefined), + }, + } as unknown as Catalog.Interface) + const integrationMock = Layer.mock(Integration.Service, { + connection: { active: () => Effect.succeed(undefined) }, + } as unknown as Integration.Interface) + const session = SessionV2.Info.make({ + id: SessionV2.ID.make("ses_model_boot_race"), + projectID: ProjectV2.ID.global, + title: "test", + model: { id: catalogModel.id, providerID: catalogModel.providerID }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: DateTime.makeUnsafe(0), updated: DateTime.makeUnsafe(0) }, + location: { directory: AbsolutePath.make("/project") }, + }) + + const resolved = yield* Effect.gen(function* () { + const svc = yield* SessionRunnerModel.Service + yield* Effect.forkScoped( + Effect.sleep(400).pipe(Effect.andThen(Effect.sync(() => models.push(catalogModel)))), + ) + return yield* svc.resolve(session) + }).pipe( + Effect.provide( + SessionRunnerModel.locationLayer.pipe(Layer.provide(catalogMock), Layer.provide(integrationMock)), + ), + ) + + expect(resolved).toMatchObject({ id: "api-test-model", provider: "test-provider" }) + }), + ) }) From c174c1cb32c81608e0d21c14ce297f7e6509d841 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 12 Aug 2026 22:06:48 -0700 Subject: [PATCH 062/103] Added a one-command tmux demo for the temporal mode. The run recipe took three terminals and a handful of curl calls. The script starts the dev server, serve, and a driver in tmux panes, prompts a session, and prints the reply plus the workflow behind it. --- packages/temporal/README.md | 4 +- packages/temporal/scripts/demo-tmux.sh | 94 ++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100755 packages/temporal/scripts/demo-tmux.sh diff --git a/packages/temporal/README.md b/packages/temporal/README.md index deb08009d64c..69c21f433d67 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -78,7 +78,9 @@ OPENAI_API_KEY=... OPENCODE_SESSION_EXECUTION=temporal TEMPORAL_ADDRESS=127.0.0. ``` Create a session and prompt it against `POST /api/session` and `POST /api/session/:id/prompt`; each -session runs as a Temporal workflow `session-exec-`. +session runs as a Temporal workflow `session-exec-`. Or run the whole thing in one +command: `packages/temporal/scripts/demo-tmux.sh` starts the dev server, serve, and a driver in +tmux panes, prompts a session, and prints the reply with the workflow behind it. ### Verified diff --git a/packages/temporal/scripts/demo-tmux.sh b/packages/temporal/scripts/demo-tmux.sh new file mode 100755 index 000000000000..3a76a0aa0c71 --- /dev/null +++ b/packages/temporal/scripts/demo-tmux.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# One-command demo of the Temporal-backed v2 SessionExecution, in one tmux session: +# left pane = Temporal dev server (reused when one already answers on the port) +# right-top = opencode v2 serve with OPENCODE_SESSION_EXECUTION=temporal +# right-bottom= a driver that creates a session, prompts it, and prints the reply plus the +# workflow evidence, then leaves copy-paste commands for more poking +# +# Env overrides: TEMPORAL_PORT (7237), OPENCODE_PORT (4601), OPENCODE_DB +# (/tmp/opencode-temporal-demo.db), OPENCODE_KEY_FILE (~/.config/ai363/llm.key), +# DEMO_TMUX_SESSION (opencode-temporal). +set -uo pipefail + +REPO=$(cd "$(dirname "$0")/../../.." && pwd) +SESSION=${DEMO_TMUX_SESSION:-opencode-temporal} +TPORT=${TEMPORAL_PORT:-7237} +PORT=${OPENCODE_PORT:-4601} +DB=${OPENCODE_DB:-/tmp/opencode-temporal-demo.db} +KEY_FILE=${OPENCODE_KEY_FILE:-$HOME/.config/ai363/llm.key} +B="http://127.0.0.1:$PORT/api" + +# The driver body, run inside the third pane via `--drive`. +drive() { + echo "waiting for serve on :$PORT (first boot bundles the workflow, about a minute)" + until curl -s -o /dev/null --max-time 2 "$B/session"; do sleep 1; done + AUTH=$(printf 'opencode:%s' "$(cat "$HOME/.local/state/opencode/password")" | base64) + SID=$(curl -s -X POST "$B/session" -H "Authorization: Basic $AUTH" \ + -H 'content-type: application/json' \ + -d '{"model":{"providerID":"openai","id":"gpt-5-mini"}}' | + python3 -c 'import json,sys;print(json.load(sys.stdin)["data"]["id"])') + echo "session: $SID" + curl -s -o /dev/null -X POST "$B/session/$SID/prompt" -H "Authorization: Basic $AUTH" \ + -H 'content-type: application/json' \ + -d '{"prompt":{"text":"Reply with the single word PONG."}}' + echo "prompted; waiting for the turn to settle" + OUT="RUNNING|" + for _ in $(seq 1 90); do + OUT=$(curl -s "$B/session/$SID/history" -H "Authorization: Basic $AUTH" | python3 -c ' +import json, sys +d = json.load(sys.stdin) +texts = [] +ended = False +for e in d.get("data") or []: + t = e.get("type", "") + if "step.ended" in t: ended = True + if "text.ended" in t: texts.append(e.get("data", {}).get("text", "")) +print(("ENDED" if ended else "RUNNING") + "|" + " ".join(texts)) +' 2>/dev/null || echo "RUNNING|") + case "$OUT" in ENDED*) break ;; esac + sleep 2 + done + case "$OUT" in + ENDED*) echo "reply: ${OUT#ENDED|}" ;; + *) echo "turn did not settle in time; check the serve pane" ;; + esac + echo + echo "the workflow behind it:" + temporal workflow list --address "127.0.0.1:$TPORT" | head -5 + echo + echo "poke further (copy-paste):" + echo " temporal workflow show --address 127.0.0.1:$TPORT --workflow-id session-exec-$SID" + echo " curl -s $B/session/$SID/history -H 'Authorization: Basic $AUTH'" + echo " UI: http://localhost:8233" +} + +[ "${1:-}" = "--drive" ] && { drive; exit 0; } + +command -v tmux >/dev/null || { echo "tmux is required: brew install tmux"; exit 1; } +command -v temporal >/dev/null || { echo "temporal CLI is required: brew install temporal"; exit 1; } +[ -f "$KEY_FILE" ] || { echo "no provider key at $KEY_FILE (set OPENCODE_KEY_FILE)"; exit 1; } + +tmux kill-session -t "$SESSION" 2>/dev/null || true +tmux new-session -d -s "$SESSION" -c "$REPO" -x 220 -y 50 +P0=$(tmux display-message -p -t "$SESSION" '#{pane_id}') + +if temporal operator cluster health --address "127.0.0.1:$TPORT" >/dev/null 2>&1; then + tmux send-keys -t "$P0" "echo 'reusing the Temporal dev server already on :$TPORT'" C-m +else + tmux send-keys -t "$P0" "temporal server start-dev --port $TPORT" C-m +fi + +# serve waits for Temporal first: layer construction connects at startup. The key stays out of +# this script's expansion; the pane's shell reads it. +P1=$(tmux split-window -P -F '#{pane_id}' -t "$P0" -h -c "$REPO") +tmux send-keys -t "$P1" "until temporal operator cluster health --address 127.0.0.1:$TPORT >/dev/null 2>&1; do sleep 1; done; OPENAI_API_KEY=\$(cat $KEY_FILE) OPENCODE_SESSION_EXECUTION=temporal TEMPORAL_ADDRESS=127.0.0.1:$TPORT OPENCODE_DB=$DB bun run --cwd packages/cli src/index.ts serve --port $PORT" C-m + +P2=$(tmux split-window -P -F '#{pane_id}' -t "$P1" -v -c "$REPO") +tmux send-keys -t "$P2" "TEMPORAL_PORT=$TPORT OPENCODE_PORT=$PORT bash packages/temporal/scripts/demo-tmux.sh --drive" C-m + +tmux select-pane -t "$P2" +if [ -t 0 ]; then + if [ -n "${TMUX:-}" ]; then tmux switch-client -t "$SESSION"; else tmux attach -t "$SESSION"; fi +else + echo "started tmux session '$SESSION'; attach with: tmux attach -t $SESSION" +fi From dbe6b58e63b2b36579fa8feb6e735c2362dff60f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 12 Aug 2026 22:33:19 -0700 Subject: [PATCH 063/103] Computed the demo's UI URL from the server port. start-dev serves the UI on the server port plus 1000, so the hardcoded 8233 was wrong for the demo's 7237. --- packages/temporal/scripts/demo-tmux.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/temporal/scripts/demo-tmux.sh b/packages/temporal/scripts/demo-tmux.sh index 3a76a0aa0c71..f207418ebe85 100755 --- a/packages/temporal/scripts/demo-tmux.sh +++ b/packages/temporal/scripts/demo-tmux.sh @@ -59,7 +59,8 @@ print(("ENDED" if ended else "RUNNING") + "|" + " ".join(texts)) echo "poke further (copy-paste):" echo " temporal workflow show --address 127.0.0.1:$TPORT --workflow-id session-exec-$SID" echo " curl -s $B/session/$SID/history -H 'Authorization: Basic $AUTH'" - echo " UI: http://localhost:8233" + # start-dev puts the UI on the server port + 1000. + echo " UI: http://localhost:$((TPORT + 1000))" } [ "${1:-}" = "--drive" ] && { drive; exit 0; } From 3bad48150b991535618ab42781cde0635e0e8e02 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 12 Aug 2026 22:34:59 -0700 Subject: [PATCH 064/103] Said plainly that the driver pane finishing is the expected end state. The pane going back to a prompt read as something breaking. --- packages/temporal/scripts/demo-tmux.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/temporal/scripts/demo-tmux.sh b/packages/temporal/scripts/demo-tmux.sh index f207418ebe85..39fe135d70f1 100755 --- a/packages/temporal/scripts/demo-tmux.sh +++ b/packages/temporal/scripts/demo-tmux.sh @@ -61,6 +61,9 @@ print(("ENDED" if ended else "RUNNING") + "|" + " ".join(texts)) echo " curl -s $B/session/$SID/history -H 'Authorization: Basic $AUTH'" # start-dev puts the UI on the server port + 1000. echo " UI: http://localhost:$((TPORT + 1000))" + echo + echo "driver done. serve and the Temporal server keep running in the other panes; the session" + echo "still accepts prompts. This pane is a normal shell now." } [ "${1:-}" = "--drive" ] && { drive; exit 0; } From 5bcb0ebd271b4fa6cbc3e4fdd30f8c81d44a94a9 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 13 Aug 2026 01:01:51 -0700 Subject: [PATCH 065/103] Made the packaged CLI carry the temporal client path. bun compile cannot bundle @temporalio/worker (it drags webpack and swc to bundle the workflow from source at startup) or @libsql/client (a platform native binding loaded at import time), so both load lazily and the worker stays external; a packaged serve runs OPENCODE_TEMPORAL_ROLE=client next to standalone workers. Workflows also start by the string type now: the minified client registered the mangled function name as the workflow type and no worker matched it. --- packages/cli/script/build.ts | 7 ++++++- packages/core/src/database/database.ts | 13 ++++++++++-- .../core/src/session/execution/temporal.ts | 21 +++++++++++++++---- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/packages/cli/script/build.ts b/packages/cli/script/build.ts index f42d8b07b0d3..fca703678b76 100755 --- a/packages/cli/script/build.ts +++ b/packages/cli/script/build.ts @@ -66,7 +66,12 @@ for (const item of targets) { entrypoints: ["./src/index.ts"], tsconfig: "./tsconfig.json", plugins: [plugin], - external: ["node-gyp"], + // lightningcss (via ui -> tailwind) has a conditional native-binding require the bundler + // cannot resolve; the TUI never runs that path, so it stays external. @temporalio/worker + // drags webpack and swc (it bundles the workflow from source at startup), which cannot ride + // a compiled binary; a packaged serve runs OPENCODE_TEMPORAL_ROLE=client next to standalone + // workers instead. + external: ["node-gyp", "lightningcss", "@temporalio/worker"], format: "esm", minify: true, sourcemap: sourcemapsFlag ? "linked" : "none", diff --git a/packages/core/src/database/database.ts b/packages/core/src/database/database.ts index e3e625f8f254..ec9ec91dcec1 100644 --- a/packages/core/src/database/database.ts +++ b/packages/core/src/database/database.ts @@ -2,7 +2,6 @@ export * as Database from "./database" import { EffectDrizzleSqlite } from "@opencode-ai/effect-drizzle-sqlite" import { layer as sqliteLayer } from "#sqlite" -import { layer as libsqlLayer } from "./sqlite.libsql" import { Context, Effect, Layer } from "effect" import { Global } from "../global" import { Flag } from "../flag/flag" @@ -55,7 +54,17 @@ export function layerFromPath(filename: string) { // so any worker can resume any session from the same log. Same SQLite dialect, so nothing in the // schema/migrations/queries changes. export function layerFromLibsql(url: string, authToken?: string) { - return makeServiceLayer(false).pipe(Layer.provide(libsqlLayer({ url, authToken }))) + // Imported lazily: @libsql/client loads a platform native binding at import time, which a + // compiled binary cannot carry. The local-file backend must boot without it. + return makeServiceLayer(false).pipe( + Layer.provide( + Layer.unwrap( + Effect.promise(() => import("./sqlite.libsql")).pipe( + Effect.map((backend) => backend.layer({ url, authToken })), + ), + ), + ), + ) } export function path() { diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index bec58d9a68df..c131bc9c5522 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -3,7 +3,9 @@ export * as SessionExecutionTemporal from "./temporal" import { fileURLToPath } from "node:url" import { Effect, Layer } from "effect" import { Client, Connection, WithStartWorkflowOperation } from "@temporalio/client" -import { NativeConnection, Worker } from "@temporalio/worker" +// Imported lazily inside the worker branch: the worker package drags webpack and swc (it bundles +// the workflow from source at startup), which a compiled binary can neither bundle nor run. A +// packaged serve runs OPENCODE_TEMPORAL_ROLE=client next to standalone workers instead. import { LocationServiceMap } from "../../location-service-map" import { EventV2 } from "../../event" @@ -23,7 +25,8 @@ const TASK_QUEUE = process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? "opencode-session const workflowId = (id: string) => `session-exec-${id}` // One activity per step (the model call + its tools), with the step loop as workflow control flow. -const WORKFLOW = WF.sessionTurn +// Workflows start by the string type, never the function: a minified (packaged) client would +// otherwise register the mangled function name as the type and no worker would match it. const WORKFLOW_TYPE = "sessionTurn" // Role split so the worker fleet can run separately from the HTTP server. `both` (default) hosts the @@ -63,6 +66,16 @@ const layer = Layer.effect( // Worker connection (native) hosts the runTurnStep activity + the workflow. Skipped in // client-only role so serve can run without an embedded worker. if (HOST_WORKER) { + const { NativeConnection, Worker } = yield* Effect.tryPromise( + () => import("@temporalio/worker"), + ).pipe( + Effect.catch(() => + Effect.die( + "The embedded Temporal worker is unavailable in this build. Run standalone workers " + + "(packages/server/src/worker.ts) and set OPENCODE_TEMPORAL_ROLE=client.", + ), + ), + ) const nativeConn = yield* Effect.acquireRelease( Effect.promise(() => NativeConnection.connect({ address: ADDRESS })), (conn) => Effect.promise(() => conn.close().catch(() => {})), @@ -112,7 +125,7 @@ const layer = Layer.effect( const drive = (id: SessionSchema.ID) => Effect.promise(() => - client.workflow.signalWithStart(WORKFLOW, { + client.workflow.signalWithStart(WORKFLOW_TYPE, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), args: [id], @@ -160,7 +173,7 @@ const layer = Layer.effect( Effect.tryPromise({ try: async () => { const attempt = () => { - const startOp = new WithStartWorkflowOperation(WORKFLOW, { + const startOp = new WithStartWorkflowOperation(WORKFLOW_TYPE, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), args: [id], From 3d2820ca641d5f329d7496ce5b6acbdcd5a38a05 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 13 Aug 2026 01:01:51 -0700 Subject: [PATCH 066/103] Waited out the integration state before sending requests without auth. The integration auth methods can lag the catalog at boot, so the first resolve in a fresh worker sent a keyless provider request and failed the turn with a non-retryable LLM error. Same bounded wait as the catalog gate, paid once per process and only when the model carries no key of its own. --- packages/core/src/session/runner/model.ts | 19 +++++- .../core/test/session-runner-model.test.ts | 63 +++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/packages/core/src/session/runner/model.ts b/packages/core/src/session/runner/model.ts index 3d5a056a33ff..cb9c8a97aa97 100644 --- a/packages/core/src/session/runner/model.ts +++ b/packages/core/src/session/runner/model.ts @@ -184,6 +184,9 @@ export const locationLayer = Layer.effect( Effect.gen(function* () { const catalog = yield* Catalog.Service const integrations = yield* Integration.Service + // The auth wait below runs once per process: the integration state lag is a boot phenomenon, + // and a provider that genuinely has no connection must not pay the wait on every step. + let authSettled = false return Service.of({ resolve: Effect.fn("SessionRunnerModel.resolve")(function* (session) { // Location plugins populate and filter the catalog asynchronously during layer startup, so @@ -208,9 +211,19 @@ export const locationLayer = Layer.effect( }) if (!selected) return yield* new ModelNotSelectedError({ sessionID: session.id }) const provider = yield* catalog.provider.get(selected.providerID) - const connection = yield* integrations.connection.active( - provider?.integrationID ?? Integration.ID.make(selected.providerID), - ) + const integrationID = provider?.integrationID ?? Integration.ID.make(selected.providerID) + let connection = yield* integrations.connection.active(integrationID) + // The integration state (auth methods) can lag the catalog at boot: the model resolves, + // but the request would go out unauthenticated and fail the turn. When the model carries + // no key of its own and no connection is active yet, give the loaders the same bounded + // window, once. + if (!connection && !authSettled && typeof selected.request.body.apiKey !== "string") { + for (let attempt = 0; attempt < 20 && !connection; attempt++) { + yield* Effect.sleep(250) + connection = yield* integrations.connection.active(integrationID) + } + } + authSettled = true return yield* resolve( session, selected, diff --git a/packages/core/test/session-runner-model.test.ts b/packages/core/test/session-runner-model.test.ts index 830341858005..c5198eb1f40f 100644 --- a/packages/core/test/session-runner-model.test.ts +++ b/packages/core/test/session-runner-model.test.ts @@ -390,4 +390,67 @@ describe("SessionRunnerModel", () => { expect(resolved).toMatchObject({ id: "api-test-model", provider: "test-provider" }) }), ) + + it.live("waits out the integration state's async population before sending without auth", () => + Effect.gen(function* () { + // A keyless model: auth can only come from a connection, so a resolve that runs before the + // integration state settles would send an unauthenticated request. + const catalogModel = ModelV2.Info.make({ + ...model({ type: "aisdk", package: "@ai-sdk/openai", url: "https://openai.example/v1" }), + request: { headers: {}, body: {} }, + }) + const catalogMock = Layer.mock(Catalog.Service, { + model: { + all: () => Effect.succeed([catalogModel]), + available: () => Effect.succeed([catalogModel]), + default: () => Effect.succeed(undefined), + }, + provider: { + get: () => Effect.succeed(undefined), + }, + } as unknown as Catalog.Interface) + // The connection lands a moment after the catalog, like plugins at boot. + let connection: { type: "env"; name: string } | undefined + const integrationMock = Layer.mock(Integration.Service, { + connection: { + active: () => Effect.sync(() => connection), + resolve: () => Effect.succeed(Credential.Key.make({ type: "key", key: "boot-race-key" })), + }, + } as unknown as Integration.Interface) + const session = SessionV2.Info.make({ + id: SessionV2.ID.make("ses_model_auth_race"), + projectID: ProjectV2.ID.global, + title: "test", + model: { id: catalogModel.id, providerID: catalogModel.providerID }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: DateTime.makeUnsafe(0), updated: DateTime.makeUnsafe(0) }, + location: { directory: AbsolutePath.make("/project") }, + }) + + const resolved = yield* Effect.gen(function* () { + const svc = yield* SessionRunnerModel.Service + yield* Effect.forkScoped( + Effect.sleep(400).pipe( + Effect.andThen(Effect.sync(() => (connection = { type: "env", name: "TEST_KEY" }))), + ), + ) + return yield* svc.resolve(session) + }).pipe( + Effect.provide( + SessionRunnerModel.locationLayer.pipe(Layer.provide(catalogMock), Layer.provide(integrationMock)), + ), + ) + + const request = LLM.request({ model: resolved, prompt: "Hello" }) + const headers = yield* resolved.route.auth.apply({ + request, + method: "POST", + url: "https://openai.example/v1/responses", + body: "{}", + headers: Headers.empty, + }) + expect(headers.authorization).toBe("Bearer boot-race-key") + }), + ) }) From 261d98a1403dd57d565083d57bb1d4bfd5e69362 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 14 Aug 2026 13:23:29 -0700 Subject: [PATCH 067/103] Added a what-it-is row to the comparison table. --- packages/temporal/docs/ai399-local-options.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md index 815102d6d670..4c854a83996f 100644 --- a/packages/temporal/docs/ai399-local-options.md +++ b/packages/temporal/docs/ai399-local-options.md @@ -282,6 +282,7 @@ The ask is real, old, and largely unanswered publicly: | | A: language shim | B: rust-core shim | C: plugin pattern (incl. ADK-style fallback) | D: local dev server | |---|---|---|---|---| +| What it is | reimplement the SDK API in the app's language so Temporal-shaped code runs with no server; the customer owns it | embed a real Temporal service in the SDK core behind `service_override`; one in-process backend under every language | the loop behind a small execution interface; a startup factory picks Temporal or in-process, same code both modes | bundle the real dev server binary next to the app, which starts and supervises it | | Durability (crash mid-turn) | partial: what the shim persists; faithful replay = reimplementing Temporal | full IF built (it IS an embedded service) | partial: checkpoint-file coarse; event-sourced local store near-full for one machine | full (with `--db-filename`; default is in-memory) | | Resource needs | lightest (in-process) | in-process; core carries matching+history | lightest (in-process; store is a file) | ~102-139 MB RSS second process, ~128 MiB disk (appliance archetype; not a desktop configuration) | | Signal/Update support | re-implemented, drift-prone | full IF built | the app's control surface (~a dozen verbs), honored identically by both modes | full | From e8c17a317389dc0637b1aabe873f92ac8fa68988 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 14 Aug 2026 16:26:33 -0700 Subject: [PATCH 068/103] Removed a duplicated dependency key. The rebase resolution kept a line upstream had already added in its sorted position, and bun warns on the duplicate at install time. --- packages/core/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/package.json b/packages/core/package.json index 57273934df76..4bf5f89921d6 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -102,7 +102,6 @@ "@opentelemetry/sdk-trace-base": "2.6.1", "@parcel/watcher": "2.5.1", "@silvia-odwyer/photon-node": "0.3.4", - "@openrouter/ai-sdk-provider": "2.9.0", "@temporalio/activity": "^1.21.0", "@temporalio/client": "^1.21.0", "@temporalio/worker": "^1.21.0", From 8c140c034434e4b62d7d8f2a8c25f01fd13b4712 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 14 Aug 2026 16:57:29 -0700 Subject: [PATCH 069/103] Dropped the poll ticker and duration parser from the local driver. Predicate inputs change only on signal delivery, update start or settle, and condition registration; the driver ticks at each, so the 25ms interval was covering nothing. Effect's Duration parses the timeout strings the hand-rolled table duplicated. --- .../src/session/execution/local-driver.ts | 74 ++++++------------- 1 file changed, 23 insertions(+), 51 deletions(-) diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts index dc6769b27046..97e6d5b01c07 100644 --- a/packages/core/src/session/execution/local-driver.ts +++ b/packages/core/src/session/execution/local-driver.ts @@ -2,13 +2,15 @@ export * as SessionExecutionLocalDriver from "./local-driver" // The in-process driver for the session supervisor (workflow-core.ts). It runs the SAME supervisor // function the Temporal workflow runs, with the six runtime primitives implemented over plain -// promises: `condition` is a polled waiter, signals and updates are method calls, the drains run +// promises: `condition` is a waiter re-checked on every signal and update delivery, the drains run // directly (no activities), and cancellation is an AbortController whose reason satisfies the // drain's cancellation contract. No Temporal server, no worker, no ports; durability comes from // the engine's event log, exactly as in local coordinator mode. This is the "one supervisor, two -// drivers" shape: the factory picks the driver, the supervisor is written once. +// drivers" shape: the factory picks the driver, the supervisor is written once. The primitives +// stay promise-shaped because the supervisor also runs inside a Temporal workflow, which cannot +// carry effect's runtime. -import { Effect, Layer } from "effect" +import { Duration, Effect, Layer } from "effect" import { randomUUID } from "node:crypto" import { LocationServiceMap } from "../../location-service-map" import { EventV2 } from "../../event" @@ -22,25 +24,6 @@ import { makeWorkflows, type WorkflowRuntime } from "./workflow-core" import { toRunError } from "./run-error-codec" -const UNITS: Record = { - ms: 1, - millisecond: 1, - milliseconds: 1, - second: 1_000, - seconds: 1_000, - minute: 60_000, - minutes: 60_000, - hour: 3_600_000, - hours: 3_600_000, -} - -const parseDuration = (value: string): number => { - const match = /^\s*([\d.]+)\s*([a-z]+)\s*$/i.exec(value) - const unit = match?.[2] ? UNITS[match[2].toLowerCase()] : undefined - if (!match?.[1] || unit === undefined) throw new Error(`Unsupported duration: ${value}`) - return Number(match[1]) * unit -} - class LocalCancellation extends Error {} const BACKSTOP_MS = 12 * 60 * 60 * 1000 @@ -55,14 +38,15 @@ interface Waiter { type Drains = ReturnType // One driver per live session. Temporal re-evaluates workflow conditions on every activation; the -// local equivalent is a short poll plus an immediate re-check after every signal/update delivery. +// local equivalents are exactly the events that can change a predicate's inputs: signal delivery, +// an update starting or settling, and a new condition registering. The supervisor is one +// sequential coroutine, so nothing else mutates the state a predicate reads. class SessionDriver { readonly done: Promise completed = false private readonly signalHandlers = new Map void>() private readonly updateHandlers = new Map Promise>() private waiters: Waiter[] = [] - private ticker: ReturnType | undefined private cancelled = false private readonly abort = new AbortController() // One token per driver instance. A wake that lands after a driver finished starts a fresh driver @@ -71,19 +55,19 @@ class SessionDriver { constructor(run: (rt: WorkflowRuntime) => Promise, drains: Drains, onDone: () => void) { const rt: WorkflowRuntime = { - condition: (predicate, timeout) => - new Promise((resolve, reject) => { - if (this.cancelled) return reject(new LocalCancellation()) - const waiter: Waiter = { predicate, resolve, reject } - if (timeout !== undefined) - waiter.timer = setTimeout(() => { - this.remove(waiter) - resolve(false) - }, parseDuration(timeout)) - this.waiters.push(waiter) - this.tick() - this.ensureTicker() - }), + condition: (predicate, timeout) => { + if (this.cancelled) return Promise.reject(new LocalCancellation()) + const { promise, resolve, reject } = Promise.withResolvers() + const waiter: Waiter = { predicate, resolve, reject } + if (timeout !== undefined) + waiter.timer = setTimeout(() => { + this.remove(waiter) + resolve(false) + }, Duration.toMillis(Duration.fromInputUnsafe(timeout as Duration.Input))) + this.waiters.push(waiter) + this.tick() + return promise + }, setSignalHandler: (name, handler) => this.signalHandlers.set(name, handler), setUpdateHandler: (name, handler) => this.updateHandlers.set(name, handler), // Same 12 h backstop as the Temporal activity: a hung tool must not hold `draining` forever. @@ -95,7 +79,6 @@ class SessionDriver { } this.done = run(rt).finally(() => { this.completed = true - this.stopTicker() onDone() }) } @@ -109,7 +92,8 @@ class SessionDriver { const handler = this.updateHandlers.get(name) if (!handler) return Promise.reject(new Error(`Update handler not registered: ${name}`)) const result = handler() - // Nudge parked conditions when the update settles; the poll covers everything in between. + // The handler's synchronous prefix may have changed predicate inputs; check again on settle. + this.tick() result.finally(() => this.tick()).catch(() => {}) return result } @@ -147,18 +131,6 @@ class SessionDriver { if (waiter.timer) clearTimeout(waiter.timer) waiter.resolve(true) } - if (this.waiters.length === 0) this.stopTicker() - } - - private ensureTicker() { - if (this.ticker || this.waiters.length === 0) return - this.ticker = setInterval(() => this.tick(), 25) - } - - private stopTicker() { - if (!this.ticker) return - clearInterval(this.ticker) - this.ticker = undefined } } From b5f7711648f9bf9e41b25f3aa580971d446a1d30 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 16:59:39 -0700 Subject: [PATCH 070/103] Rewrote the local session executor as a native coordinator. Local mode no longer runs the Temporal supervisor (workflow-core.ts) through a WorkflowRuntime shim. It is now a native per-session async coordinator built from primitives that match the runtime: - Latch: a single-consumer sticky wake signal, replacing the polled `condition(() => pendingWake)` and its 25ms tick loop. - Mutex: serializes the loop's drain against a concurrent resume, replacing the polled `draining` boolean; queued work is counted synchronously so the idle check can't retire a session with a resume still waiting for the lock. - AbortController: interrupt plus the 12h backstop, as before. This removes the polled waiter, the string-keyed signal/update handler maps, and the WorkflowRuntime dependency from the local path. The wake-vs-retirement race the old `tries < 3` loop papered over is closed by construction: `completed` flips synchronously the instant the loop retires, so a racing wake either lands on the live loop or starts a fresh coordinator (one retry, provably terminating). The two modes still share drain.ts (the step body, where a bug would actually corrupt state: log fencing, error encoding, tool re-drive). Only the coordination loop is per-runtime now. --- .../src/session/execution/local-driver.ts | 351 +++++++++++------- 1 file changed, 219 insertions(+), 132 deletions(-) diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts index 97e6d5b01c07..22f2bfa172e4 100644 --- a/packages/core/src/session/execution/local-driver.ts +++ b/packages/core/src/session/execution/local-driver.ts @@ -1,16 +1,17 @@ export * as SessionExecutionLocalDriver from "./local-driver" -// The in-process driver for the session supervisor (workflow-core.ts). It runs the SAME supervisor -// function the Temporal workflow runs, with the six runtime primitives implemented over plain -// promises: `condition` is a waiter re-checked on every signal and update delivery, the drains run -// directly (no activities), and cancellation is an AbortController whose reason satisfies the -// drain's cancellation contract. No Temporal server, no worker, no ports; durability comes from -// the engine's event log, exactly as in local coordinator mode. This is the "one supervisor, two -// drivers" shape: the factory picks the driver, the supervisor is written once. The primitives -// stay promise-shaped because the supervisor also runs inside a Temporal workflow, which cannot -// carry effect's runtime. - -import { Duration, Effect, Layer } from "effect" +// The in-process SessionExecution: local mode as its own product. It is a native async coordinator +// (a per-session task over a mutex, a latch, and an AbortController), NOT the Temporal supervisor +// run through a shim. The two modes share the part where a subtle bug would actually corrupt state +// -- the step body in drain.ts (fencing, error encoding, tool re-drive) -- and nothing else. The +// coordination loop here is written for this runtime: no polled `condition`, no signal/update +// handler maps, no ports, no server; durability comes from the engine's event log. +// +// Parity with the Temporal loop (temporal-workflow.ts + workflow-core.ts) is guaranteed by the +// shared drain and by the driver-contract test (session-execution-local-driver.test.ts), not by a +// single shared loop. See packages/temporal/README.md "Two modes, one drain". + +import { Effect, Layer } from "effect" import { randomUUID } from "node:crypto" import { LocationServiceMap } from "../../location-service-map" import { EventV2 } from "../../event" @@ -20,126 +21,224 @@ import { SessionStore } from "../store" import { SessionExecution } from "../execution" import { makeDrains } from "./drain" import { WorktreeMaterializer } from "./worktree" -import { makeWorkflows, type WorkflowRuntime } from "./workflow-core" import { toRunError } from "./run-error-codec" +const UNITS: Record = { + ms: 1, + millisecond: 1, + milliseconds: 1, + second: 1_000, + seconds: 1_000, + minute: 60_000, + minutes: 60_000, + hour: 3_600_000, + hours: 3_600_000, +} + +const parseDuration = (value: string): number => { + const match = /^\s*([\d.]+)\s*([a-z]+)\s*$/i.exec(value) + const unit = match?.[2] ? UNITS[match[2].toLowerCase()] : undefined + if (!match?.[1] || unit === undefined) throw new Error(`Unsupported duration: ${value}`) + return Number(match[1]) * unit +} +// A drain that is interrupted (by an explicit stop or the backstop) throws this. The drain body +// rethrows the AbortSignal's reason on cancellation, so a cancelled drain and a stop both surface +// the same type, and the loop treats either as a normal retire rather than a failure. class LocalCancellation extends Error {} +const DEFAULT_IDLE = "5 minutes" +// Matches the Temporal activity's startToClose backstop: a hung tool must not pin a session's +// coordinator open forever. The abort reason is a LocalCancellation, so a timed-out drain looks +// like any other stop. const BACKSTOP_MS = 12 * 60 * 60 * 1000 -interface Waiter { - readonly predicate: () => boolean - readonly resolve: (value: boolean) => void - readonly reject: (error: unknown) => void - timer?: ReturnType +type Drains = ReturnType + +// A single-consumer latch. Producers (wake, resume, interrupt) call `open`; the one coordinator +// loop calls `wait`. `open` is sticky: a wake that arrives while the loop is mid-drain is still +// observed on the next `wait`, so no prompt is stranded. This is the direct primitive the polled +// `condition(() => pendingWake)` was standing in for. +class Latch { + private signalled = false + private waiter?: (opened: boolean) => void + + get pending() { + return this.signalled + } + + open() { + this.signalled = true + const waiter = this.waiter + this.waiter = undefined + waiter?.(true) + } + + reset() { + this.signalled = false + } + + // Resolve true when opened, false when the idle deadline expires first. + wait(timeoutMs: number): Promise { + if (this.signalled) return Promise.resolve(true) + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.waiter = undefined + resolve(false) + }, timeoutMs) + this.waiter = (opened) => { + clearTimeout(timer) + resolve(opened) + } + }) + } } -type Drains = ReturnType +// Serializes drains: the coordinator loop's drain and a concurrent `resume` never overlap (one +// owner fiber per session at a time, exactly the coordinator's guarantee). `active` counts queued +// AND running work, incremented synchronously on `run`, so the idle check cannot retire a session +// with a resume still waiting for the lock. +class Mutex { + private tail: Promise = Promise.resolve() + private active = 0 + + get idle() { + return this.active === 0 + } -// One driver per live session. Temporal re-evaluates workflow conditions on every activation; the -// local equivalents are exactly the events that can change a predicate's inputs: signal delivery, -// an update starting or settling, and a new condition registering. The supervisor is one -// sequential coroutine, so nothing else mutates the state a predicate reads. -class SessionDriver { + run(fn: () => Promise): Promise { + this.active++ + const result = this.tail.then(fn) + // Keep the chain alive across a rejected body so the next waiter still runs. + this.tail = result.then( + () => {}, + () => {}, + ) + return result.finally(() => { + this.active-- + }) + } +} + +// One coordinator per live session. It owns a task (`done`) that drains work until the session goes +// idle, then retires. `wake` registers work, `resume` forces one drain and awaits its result (so a +// run error reaches the caller), `interrupt` cancels the in-flight drain and retires the task. +class LocalSession { readonly done: Promise + // Set synchronously the instant the loop decides to retire, before the async `finally` runs, so a + // wake racing the retirement is never accepted onto a dead loop (it starts a fresh coordinator). completed = false - private readonly signalHandlers = new Map void>() - private readonly updateHandlers = new Map Promise>() - private waiters: Waiter[] = [] - private cancelled = false + private readonly abort = new AbortController() - // One token per driver instance. A wake that lands after a driver finished starts a fresh driver - // (a new token), so the retired one's late appends are fenced, mirroring the Temporal attempt. + private readonly wake = new Latch() + private readonly drainLock = new Mutex() + private stopping = false + private forcedInFlight = 0 + // One token per coordinator instance. A wake that lands after this one retired starts a fresh + // coordinator (a new token), so the retired one's late appends are fenced by the event log's + // owner check -- the local mirror of a Temporal attempt claiming the log. private readonly owner = randomUUID() - constructor(run: (rt: WorkflowRuntime) => Promise, drains: Drains, onDone: () => void) { - const rt: WorkflowRuntime = { - condition: (predicate, timeout) => { - if (this.cancelled) return Promise.reject(new LocalCancellation()) - const { promise, resolve, reject } = Promise.withResolvers() - const waiter: Waiter = { predicate, resolve, reject } - if (timeout !== undefined) - waiter.timer = setTimeout(() => { - this.remove(waiter) - resolve(false) - }, Duration.toMillis(Duration.fromInputUnsafe(timeout as Duration.Input))) - this.waiters.push(waiter) - this.tick() - return promise - }, - setSignalHandler: (name, handler) => this.signalHandlers.set(name, handler), - setUpdateHandler: (name, handler) => this.updateHandlers.set(name, handler), - // Same 12 h backstop as the Temporal activity: a hung tool must not hold `draining` forever. - // The abort reason is a LocalCancellation, so a timed-out drain looks like any other cancel. - runTurnStep: (input) => - this.withBackstop((signal) => drains.stepDrain({ ...input, owner: this.owner }, signal)), - cancelCurrentScope: () => this.cancel(), - isCancellation: (error) => error instanceof LocalCancellation, - } - this.done = run(rt).finally(() => { - this.completed = true - onDone() - }) + constructor( + private readonly sessionID: SessionSchema.ID, + private readonly drains: Drains, + private readonly idleMs: number, + onDone: () => void, + ) { + this.done = this.loop().finally(onDone) } - signal(name: "wake" | "interrupt") { - this.signalHandlers.get(name)?.() - this.tick() + // Register work. Returns false if this coordinator has already retired, so the caller can start a + // fresh one instead of stranding the prompt. + requestWake(): boolean { + if (this.completed) return false + this.wake.open() + return true } - update(name: "resume"): Promise { - const handler = this.updateHandlers.get(name) - if (!handler) return Promise.reject(new Error(`Update handler not registered: ${name}`)) - const result = handler() - // The handler's synchronous prefix may have changed predicate inputs; check again on settle. - this.tick() - result.finally(() => this.tick()).catch(() => {}) - return result - } - - // The drain shares the driver's abort signal so an interrupt still cancels it; the timer only - // adds an upper bound. - private async withBackstop(run: (signal: AbortSignal) => Promise): Promise { - const timer = setTimeout(() => this.abort.abort(new LocalCancellation("drain backstop")), BACKSTOP_MS) + // Force one drain and surface its outcome to the caller (a run error rejects). Mirrors + // coordinator.run: the caller observes the run's error instead of it being swallowed. + async resume(): Promise { + this.forcedInFlight++ try { - return await run(this.abort.signal) + await this.drainLock.run(() => this.drain(true)) } finally { - clearTimeout(timer) + this.forcedInFlight-- + // Nudge the loop so an idle retire can re-evaluate now that the forced drain has settled. + this.wake.open() } } - private cancel() { - this.cancelled = true - // The drain rethrows the signal's reason on cancellation, so the supervisor observes the same - // LocalCancellation from a cancelled drain as from a rejected condition. + // Cancel the in-flight drain and any parked idle wait, and retire. + interrupt() { + this.stopping = true this.abort.abort(new LocalCancellation("session interrupted")) - for (const waiter of this.waiters.splice(0)) { - if (waiter.timer) clearTimeout(waiter.timer) - waiter.reject(new LocalCancellation()) + this.wake.open() + } + + private async loop(): Promise { + // Constructed in response to a wake, so there is work to drain immediately. + this.wake.open() + try { + for (;;) { + const gotWork = await this.wake.wait(this.idleMs) + if (this.stopping) return + if (!gotWork) { + // Idle deadline. A wake can race the timer; without this re-check it would be dropped. + if (this.wake.pending) continue + // Retire only when nothing is in flight. A later wake/resume starts a fresh coordinator. + if (this.drainLock.idle && this.forcedInFlight === 0) return + continue + } + this.wake.reset() + try { + await this.drainLock.run(() => this.drain(false)) + } catch (error) { + // A wake-driven drain tolerates run errors (already recorded in the session log); only a + // stop/cancellation ends the coordinator. + if (error instanceof LocalCancellation) return + } + } + } finally { + this.completed = true } } - private remove(waiter: Waiter) { - this.waiters = this.waiters.filter((entry) => entry !== waiter) + // One turn, driven a step at a time, exactly like the Temporal loop: each step returns the next + // loop state until it declines to continue. The step body is the shared drain. + private async drain(force: boolean): Promise { + let step = 1 + let promotion: string | null = null + let first = true + for (;;) { + const result = await this.withBackstop((signal) => + this.drains.stepDrain({ sessionID: this.sessionID, step, promotion, first, force, owner: this.owner }, signal), + ) + if (!result.continue) break + step = result.step + promotion = result.promotion + first = false + } } - private tick() { - for (const waiter of [...this.waiters]) { - if (!waiter.predicate()) continue - this.remove(waiter) - if (waiter.timer) clearTimeout(waiter.timer) - waiter.resolve(true) + // The drain shares this coordinator's abort signal so an interrupt cancels it; the timer only adds + // an upper bound on a drain that hangs while the process stays alive. + private async withBackstop(run: (signal: AbortSignal) => Promise): Promise { + const timer = setTimeout(() => this.abort.abort(new LocalCancellation("drain backstop")), BACKSTOP_MS) + try { + return await run(this.abort.signal) + } finally { + clearTimeout(timer) } } } /** - * An in-process SessionExecution running the shared supervisor with no Temporal anywhere: - * - wake -> deliver the wake signal (starting a driver if the session has none) - * - resume -> run the resume update and await it, surfacing the exact RunError - * - interrupt -> deliver the interrupt signal (cancels the drain and parked waits) - * - active -> the live drivers + * An in-process SessionExecution with no Temporal anywhere: + * - wake -> register work, starting a coordinator if the session has none + * - resume -> force one drain and await it, surfacing the exact RunError + * - interrupt -> cancel the in-flight drain and parked waits + * - active -> the sessions with a live coordinator */ const layer = Layer.effect( SessionExecution.Service, @@ -150,62 +249,50 @@ const layer = Layer.effect( const events = yield* EventV2.Service const worktrees = yield* WorktreeMaterializer.Service const drains = makeDrains({ store, locations, ctx, events, worktrees }) - const drivers = new Map() + const sessions = new Map() // Read at layer build (not module load) so tests can set it before constructing the layer. - // The idle override shortens the supervisor's 5-minute self-termination. - const IDLE_TIMEOUT = process.env.OPENCODE_SESSION_IDLE_TIMEOUT + const idleMs = parseDuration(process.env.OPENCODE_SESSION_IDLE_TIMEOUT ?? DEFAULT_IDLE) - const ensure = (id: SessionSchema.ID): SessionDriver => { - const existing = drivers.get(id) + const ensure = (id: SessionSchema.ID): LocalSession => { + const existing = sessions.get(id) if (existing && !existing.completed) return existing - const driver = new SessionDriver( - (rt) => { - const workflows = makeWorkflows(rt, IDLE_TIMEOUT ? { idleTimeout: IDLE_TIMEOUT } : undefined) - return workflows.sessionTurn(id) - }, - drains, - () => { - if (drivers.get(id) === driver) drivers.delete(id) - }, - ) - drivers.set(id, driver) - // wake tolerates supervisor failures (they are already recorded in the session log); an - // unhandled rejection here would crash the process instead. - driver.done.catch(() => {}) - return driver + const session = new LocalSession(id, drains, idleMs, () => { + if (sessions.get(id) === session) sessions.delete(id) + }) + sessions.set(id, session) + // The coordinator records its own failures in the session log; an unhandled rejection here + // would crash the process instead. + session.done.catch(() => {}) + return session } yield* Effect.addFinalizer(() => Effect.promise(async () => { - for (const driver of drivers.values()) driver.signal("interrupt") - await Promise.allSettled([...drivers.values()].map((driver) => driver.done)) + for (const session of sessions.values()) session.interrupt() + await Promise.allSettled([...sessions.values()].map((session) => session.done)) }), ) - yield* Effect.logInfo("SessionExecutionLocalDriver ready").pipe( - Effect.annotateLogs({ supervisor: "sessionTurn" }), - ) + yield* Effect.logInfo("SessionExecutionLocalDriver ready").pipe(Effect.annotateLogs({ coordinator: "local" })) return SessionExecution.Service.of({ - active: Effect.sync(() => new Set(drivers.keys())), + active: Effect.sync( + () => new Set([...sessions].filter(([, session]) => !session.completed).map(([id]) => id)), + ), wake: (id) => Effect.sync(() => { - // A wake can land between the supervisor's return and its finally; retry onto a fresh - // driver so the prompt is not stranded until the next wake. - for (let tries = 0; tries < 3; tries++) { - const driver = ensure(id) - driver.signal("wake") - if (!driver.completed) return - } + // `completed` flips synchronously as the loop retires, and `ensure` replaces a retired + // coordinator, so a second attempt always lands on a live one. + if (!ensure(id).requestWake()) ensure(id).requestWake() }), resume: (id) => Effect.tryPromise({ - try: () => ensure(id).update("resume"), - catch: (e) => toRunError(id, e), + try: () => ensure(id).resume(), + catch: (error) => toRunError(id, error), }), interrupt: (id) => Effect.sync(() => { - drivers.get(id)?.signal("interrupt") + sessions.get(id)?.interrupt() }), }) }), From 28ca42bbf7ea9fd7bf9831b6ce5746a748da60fa Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 16:59:48 -0700 Subject: [PATCH 071/103] Parameterized the local executor test into a driver-contract suite. The scenarios now live in runContract(label, makeExec), a suite parameterized over the SessionExecution factory: wake drives a turn to settlement then the idle coordinator retires, resume forces a healthy turn to completion (new), resume surfaces the exact tagged RunError through the shared codec, and interrupt cancels an in-flight turn. Because the local coordinator and the Temporal workflow are now separate loops sharing only the drain, this suite is what holds them to one behavior. It runs against the local coordinator here and can be pointed at a Temporal test-env factory to assert the same contract on that side. --- .../session-execution-local-driver.test.ts | 174 +++++++++++------- 1 file changed, 104 insertions(+), 70 deletions(-) diff --git a/packages/core/test/session-execution-local-driver.test.ts b/packages/core/test/session-execution-local-driver.test.ts index e9307b9481e8..6210470d295b 100644 --- a/packages/core/test/session-execution-local-driver.test.ts +++ b/packages/core/test/session-execution-local-driver.test.ts @@ -1,8 +1,10 @@ -// Contract tests for the in-process micro-driver: the SAME supervisor function the Temporal -// workflow runs (workflow-core.ts), driven with plain promises and no server. The contract is the -// SessionExecution interface: wake drives a turn to settlement, resume surfaces the exact tagged -// RunError (through the same encode/decode path the Temporal boundary uses), interrupt cancels an -// in-flight turn, and an idle supervisor retires itself. +// The SessionExecution driver-contract suite, run here against the in-process native coordinator +// (local-driver.ts): a per-session async loop with no server. The suite is parameterized over the +// driver factory (runContract) so the SAME behaviors can be asserted against the Temporal driver; +// this is what guarantees the two modes agree now that they share only the drain, not one loop. +// The contract: wake drives a turn to settlement then the idle coordinator retires, resume forces a +// healthy turn to completion, resume surfaces the exact tagged RunError (through the same +// encode/decode path the Temporal boundary uses), and interrupt cancels an in-flight turn. import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" import { LLMEvent } from "@opencode-ai/llm" import { Database } from "@opencode-ai/core/database/database" @@ -140,71 +142,103 @@ const until = (read: Effect.Effect, predicate: (value: A) => boolean } }) -describe("SessionExecution local micro-driver", () => { - { - const { requests, stream } = countingModel() - const sessionID = SessionV2.ID.make("ses_driver_wake") - it.live("wake drives a turn to settlement, then the idle supervisor retires", () => - Effect.gen(function* () { - const previousIdle = process.env.OPENCODE_SESSION_IDLE_TIMEOUT - process.env.OPENCODE_SESSION_IDLE_TIMEOUT = "2 seconds" - yield* seedSession(sessionID) - yield* seedPrompt(sessionID) - const exec = Context.get(yield* Layer.build(makeExecution(stream)), SessionExecution.Service) - yield* exec.wake(sessionID) - const store = yield* SessionStore.Service - yield* until(store.context(sessionID), (context) => { +// The SessionExecution contract, parameterized over the driver factory. `makeExec` builds a +// SessionExecution graph the same way serve does, with the model/LLM mocked. Running the identical +// suite against a second factory (e.g. a Temporal test-env node) is how the two modes are held to +// one behavior now that they no longer share a single coordination loop -- only the drain. +const runContract = ( + label: string, + makeExec: (stream: LLMClientShape["stream"], models?: typeof okModels) => ReturnType, +) => { + const slug = label.replace(/[^a-z0-9]+/gi, "_") + describe(`SessionExecution contract: ${label}`, () => { + { + const { requests, stream } = countingModel() + const sessionID = SessionV2.ID.make(`ses_${slug}_wake`) + it.live("wake drives a turn to settlement, then the idle coordinator retires", () => + Effect.gen(function* () { + const previousIdle = process.env.OPENCODE_SESSION_IDLE_TIMEOUT + process.env.OPENCODE_SESSION_IDLE_TIMEOUT = "2 seconds" + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + const exec = Context.get(yield* Layer.build(makeExec(stream)), SessionExecution.Service) + yield* exec.wake(sessionID) + const store = yield* SessionStore.Service + yield* until(store.context(sessionID), (context) => { + const assistant = context.findLast((message) => message.type === "assistant") + return assistant?.type === "assistant" && Boolean(assistant.time.completed) + }) + expect(requests).toHaveLength(1) + expect((yield* exec.active).has(sessionID)).toBe(true) + // Idle self-termination: the coordinator retires without an interrupt. + yield* until(exec.active, (active) => !active.has(sessionID)) + // Restore so later layer builds in this process get the real default. + if (previousIdle === undefined) delete process.env.OPENCODE_SESSION_IDLE_TIMEOUT + else process.env.OPENCODE_SESSION_IDLE_TIMEOUT = previousIdle + }), + ) + } + + { + const { requests, stream } = countingModel() + const sessionID = SessionV2.ID.make(`ses_${slug}_resume_ok`) + it.live("resume forces a healthy turn to completion and resolves", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + const exec = Context.get(yield* Layer.build(makeExec(stream)), SessionExecution.Service) + // resume is request/response: it awaits the forced drain and resolves on success. + const exit = yield* exec.resume(sessionID).pipe(Effect.exit) + expect(Exit.isSuccess(exit)).toBe(true) + expect(requests).toHaveLength(1) + const store = yield* SessionStore.Service + const context = yield* store.context(sessionID) const assistant = context.findLast((message) => message.type === "assistant") - return assistant?.type === "assistant" && Boolean(assistant.time.completed) - }) - expect(requests).toHaveLength(1) - expect((yield* exec.active).has(sessionID)).toBe(true) - // Idle self-termination: the driver retires without an interrupt. - yield* until(exec.active, (active) => !active.has(sessionID)) - // Restore so later layer builds in this process get the real default. - if (previousIdle === undefined) delete process.env.OPENCODE_SESSION_IDLE_TIMEOUT - else process.env.OPENCODE_SESSION_IDLE_TIMEOUT = previousIdle - }), - ) - } + expect(assistant?.type === "assistant" && Boolean(assistant.time.completed)).toBe(true) + }), + ) + } - { - const sessionID = SessionV2.ID.make("ses_driver_error") - const failingModels = SessionRunnerModel.layerWith(() => - Effect.fail(new ModelNotSelectedError({ sessionID })), - ) - it.live("resume surfaces the exact tagged RunError through the shared codec", () => - Effect.gen(function* () { - yield* seedSession(sessionID) - const { stream } = countingModel() - const exec = Context.get(yield* Layer.build(makeExecution(stream, failingModels)), SessionExecution.Service) - const exit = yield* exec.resume(sessionID).pipe(Effect.exit) - expect(Exit.isFailure(exit)).toBe(true) - const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined - // The same encode -> details -> decode path the Temporal boundary uses, so the caller gets - // the identical tagged instance in both modes. - expect(error).toBeInstanceOf(ModelNotSelectedError) - }), - ) - } + { + const sessionID = SessionV2.ID.make(`ses_${slug}_error`) + const failingModels = SessionRunnerModel.layerWith(() => + Effect.fail(new ModelNotSelectedError({ sessionID })), + ) + it.live("resume surfaces the exact tagged RunError through the shared codec", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + const { stream } = countingModel() + const exec = Context.get(yield* Layer.build(makeExec(stream, failingModels)), SessionExecution.Service) + const exit = yield* exec.resume(sessionID).pipe(Effect.exit) + expect(Exit.isFailure(exit)).toBe(true) + const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined + // The same encode -> details -> decode path the Temporal boundary uses, so the caller gets + // the identical tagged instance in both modes. + expect(error).toBeInstanceOf(ModelNotSelectedError) + }), + ) + } - { - const sessionID = SessionV2.ID.make("ses_driver_interrupt") - it.live("interrupt cancels an in-flight turn and the driver retires", () => - Effect.gen(function* () { - yield* seedSession(sessionID) - yield* seedPrompt(sessionID) - // A model that never answers: the turn hangs until interrupted. - const exec = Context.get( - yield* Layer.build(makeExecution(() => Stream.never)), - SessionExecution.Service, - ) - yield* exec.wake(sessionID) - yield* Effect.sleep(200) - expect((yield* exec.active).has(sessionID)).toBe(true) - yield* exec.interrupt(sessionID) - yield* until(exec.active, (active) => !active.has(sessionID), 4000) - }), - ) - } -}) + { + const sessionID = SessionV2.ID.make(`ses_${slug}_interrupt`) + it.live("interrupt cancels an in-flight turn and the coordinator retires", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + // A model that never answers: the turn hangs until interrupted. + const exec = Context.get( + yield* Layer.build(makeExec(() => Stream.never)), + SessionExecution.Service, + ) + yield* exec.wake(sessionID) + yield* Effect.sleep(200) + expect((yield* exec.active).has(sessionID)).toBe(true) + yield* exec.interrupt(sessionID) + yield* until(exec.active, (active) => !active.has(sessionID), 4000) + }), + ) + } + }) +} + +runContract("local coordinator", makeExecution) From ccc2fe59a418dc3afc8daeef412d8dbf1a8a450e Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 17:00:05 -0700 Subject: [PATCH 072/103] Described the two-loop (one-drain) design in comments and README. Updated the header comments in workflow-core.ts, temporal-workflow.ts, drain.ts, temporal.ts, and the factory comment in routes.ts, plus the temporal README (section renamed "Two modes, one supervisor" -> "Two modes, one drain"), to reflect that local and Temporal are now separate coordination loops sharing only the drain body. Parity is stated as an enforced-by-contract-test property rather than a single shared loop. Comment/doc only; no behavior change to the Temporal path. --- packages/core/src/session/execution/drain.ts | 6 +-- .../session/execution/temporal-workflow.ts | 10 ++-- .../core/src/session/execution/temporal.ts | 4 +- .../src/session/execution/workflow-core.ts | 11 ++-- packages/server/src/routes.ts | 7 +-- packages/temporal/README.md | 54 +++++++++++-------- 6 files changed, 52 insertions(+), 40 deletions(-) diff --git a/packages/core/src/session/execution/drain.ts b/packages/core/src/session/execution/drain.ts index 74a069a40ff1..66b7a3956821 100644 --- a/packages/core/src/session/execution/drain.ts +++ b/packages/core/src/session/execution/drain.ts @@ -1,6 +1,6 @@ -// The drain body shared by every durable executor: the Temporal layer runs it inside an activity, -// the in-process micro-driver calls it directly. One implementation, so the turn semantics and -// the error encoding cannot differ between drivers. +// The drain body shared by both coordinators: the Temporal layer runs it inside an activity, the +// native in-process coordinator (local-driver.ts) calls it directly. One implementation, so the +// turn semantics and the error encoding cannot differ between modes even though the loops differ. import { Cause, Context, Effect, Exit, type LayerMap } from "effect" import { ApplicationFailure } from "@temporalio/activity" diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index ae49a6f50d0b..01559229d027 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -1,8 +1,8 @@ -// The Temporal driver for the session supervisor. The supervisor itself lives in workflow-core.ts -// and is written once; this file adapts the real SDK's primitives (condition, signal/update -// handlers, activity proxies, cancellation) to the WorkflowRuntime interface and exports the -// workflow function the worker registers. The in-process driver (local-driver.ts) runs the SAME -// supervisor with plain promises. +// The Temporal driver for the session supervisor. The supervisor loop lives in workflow-core.ts; +// this file adapts the real SDK's primitives (condition, signal/update handlers, activity proxies, +// cancellation) to the WorkflowRuntime interface and exports the workflow function the worker +// registers. Local mode is a separate native coordinator (local-driver.ts); the two loops share +// only the drain body, and the driver-contract test keeps them behaving alike. // // MUST stay sandbox-safe: Temporal bundles this in an isolated context, so no `effect`, no // `@opencode-ai/core` runtime imports, no Node builtins. diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index c131bc9c5522..b90ef231f933 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -59,8 +59,8 @@ const layer = Layer.effect( const events = yield* EventV2.Service const worktrees = yield* WorktreeMaterializer.Service - // The drain bodies are shared with the in-process micro-driver (drain.ts), so turn semantics - // and error encoding cannot differ between drivers. + // The drain bodies are shared with the native in-process coordinator (local-driver.ts), so turn + // semantics and error encoding cannot differ between modes even though the loops differ. const { stepDrain } = makeDrains({ store, locations, ctx, events, worktrees }) // Worker connection (native) hosts the runTurnStep activity + the workflow. Skipped in diff --git a/packages/core/src/session/execution/workflow-core.ts b/packages/core/src/session/execution/workflow-core.ts index 1d142e788254..5a6cb979cf80 100644 --- a/packages/core/src/session/execution/workflow-core.ts +++ b/packages/core/src/session/execution/workflow-core.ts @@ -1,8 +1,9 @@ -// The session supervisor, written ONCE, over a six-primitive runtime interface. Two drivers -// execute it: the Temporal workflow adapter (temporal-workflow.ts; the real SDK provides the -// primitives, activities carry the drains) and the in-process micro-driver (local-driver.ts; plain -// promises provide the primitives, the drains run directly). The supervisor cannot drift between -// modes because there is only one of it. +// The Temporal driver's session supervisor, expressed over a six-primitive runtime interface so it +// can be unit-tested off a live cluster. The Temporal workflow adapter (temporal-workflow.ts) is +// the only caller: the real SDK provides the primitives and activities carry the drains. Local mode +// does NOT run this loop -- it is a native coordinator (local-driver.ts) that shares only the drain +// body (drain.ts). The two loops are held to one behavior by the driver-contract test, not by being +// one function; see packages/temporal/README.md "Two modes, one drain". // // MUST stay pure: the Temporal driver bundles this into the workflow sandbox, so no `effect`, no // `@opencode-ai/core` runtime imports, no Node builtins. Type-only imports are erased and safe. diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index d905f9e2d4ec..281c63efaaaf 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -53,9 +53,10 @@ export function createEmbeddedRoutes() { // Shared by the HTTP routes and the standalone worker entrypoint (src/worker.ts) so both build the // exact same context. export function createServiceLayer() { - // The factory: two modes, one supervisor (workflow-core.ts). "temporal" runs it on a Temporal - // worker (one activity per step); anything else runs it in-process with no server - // (local-driver.ts). The loop, the drains, and the error codec are the same code either way. + // The factory: two modes, one drain (drain.ts). "temporal" loops the step drain on a Temporal + // worker (one activity per step); anything else loops it in a native in-process coordinator with + // no server (local-driver.ts). The loops differ by runtime; the drain and the error codec are the + // same code either way, and the driver-contract test keeps the loops behaving alike. const executionNode = process.env.OPENCODE_SESSION_EXECUTION === "temporal" ? SessionExecutionTemporal.node diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 69c21f433d67..6fc68d8058bd 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -12,15 +12,17 @@ inside the engine, so a crashed turn resumes mid-step instead of being re-attach ## How it fits together -One design decision carries the change: the session supervisor is written once, and durability is -a choice of driver. Everything else here is a consequence of taking at-least-once execution -seriously. - -The supervisor (`workflow-core.ts`) drives a session over six runtime primitives. The Temporal -driver runs it as a per-session workflow with one activity per step; the in-process micro-driver -(`local-driver.ts`) runs the same function over plain promises. One env var picks the driver, and -the shared drain bodies keep turn semantics identical in both modes (see -[Two modes, one supervisor](#two-modes-one-supervisor)). +One design decision carries the change: the step body -- where a bug would actually corrupt state +-- is written once, and each runtime gets a coordination loop written for it. Everything else here +is a consequence of taking at-least-once execution seriously. + +The shared drain (`drain.ts`) is one step of a turn: it claims the event log, ensures the worktree, +runs `SessionRunner.runStep`, and encodes any failure faithfully. Two coordinators loop it. The +Temporal driver (`workflow-core.ts` + `temporal-workflow.ts`) runs a per-session workflow with one +activity per step. The default is a native in-process coordinator (`local-driver.ts`): a per-session +async task over a mutex, a latch, and an AbortController -- no server, no worker, no ports. One env +var picks the coordinator; the shared drain keeps turn semantics identical, and the driver-contract +test keeps the two loops behaving alike (see [Two modes, one drain](#two-modes-one-drain)). That forces six things: @@ -31,7 +33,7 @@ That forces six things: declared idempotent, and fails the rest for the model to redo. The step loop is bounded (`loop-guard.ts`: a step ceiling plus a repeated-identical-call detector), because a runaway turn would otherwise be a durable runaway turn - ([Two modes, one supervisor](#two-modes-one-supervisor)). + ([Two modes, one drain](#two-modes-one-drain)). 3. **Two writers must be fenced.** A superseded attempt cannot keep appending to the log; each drain claims the log with an attempt token ([Notes](#notes)). 4. **The worktree must travel.** Snapshot trees ship as incremental git packs, and a worker @@ -91,18 +93,26 @@ tmux panes, prompts a session, and prints the reply with the workflow behind it. the in-flight step activity (attempt 2), the run continues from the event log, and the workflow completes. -### Two modes, one supervisor - -The factory has exactly two modes. `OPENCODE_SESSION_EXECUTION=temporal` runs the session -supervisor on a Temporal worker; anything else (the default) runs the SAME supervisor in-process -with the micro-driver (`workflow-core.ts` + `local-driver.ts`): no server, no worker, no ports, -durability from the event log. Both modes drive the turn one **step** at a time: the `sessionTurn` -supervisor loops a `runTurnStep` drain, so in temporal mode each step (one provider attempt + its -tools) is its own activity with its own retry/timeout/visibility. It reuses `SessionRunner.runStep` -(one iteration of `run`'s loop), so the turn semantics are unchanged. Verified: a -create-then-read-then-reply turn recorded three `runTurnStep` activities under a `sessionTurn` -workflow and completed. (Earlier whole-turn-per-activity and stock-coordinator modes were folded -away.) +### Two modes, one drain + +The factory has exactly two modes. `OPENCODE_SESSION_EXECUTION=temporal` runs the Temporal +supervisor (`workflow-core.ts`) on a worker; anything else (the default) runs a native in-process +coordinator (`local-driver.ts`): no server, no worker, no ports, durability from the event log. The +two are separate coordination loops written for their runtimes -- the local one is a per-session +async task over a mutex, a latch, and an AbortController; the Temporal one is a workflow over the SDK +primitives -- and they share exactly one thing: the step body (`drain.ts`). Local mode is its own +product, not the Temporal loop behind a shim, because the loop is the low-risk half: the semantics +that would corrupt state (log fencing, error encoding, tool re-drive) all live in the shared drain. +Both loops drive the turn one **step** at a time: they loop a `runTurnStep` drain, so in temporal +mode each step (one provider attempt + its tools) is its own activity with its own +retry/timeout/visibility. Both reuse `SessionRunner.runStep` (one iteration of `run`'s loop), so the +turn semantics are unchanged. Parity is enforced by the driver-contract test +(`packages/core/test/session-execution-local-driver.test.ts`): one suite -- wake drives a turn then +the idle coordinator retires, resume forces a healthy turn and surfaces the exact tagged RunError, +interrupt cancels -- parameterized over the coordinator factory so the same behaviors can be asserted +against Temporal. Verified: a create-then-read-then-reply turn recorded three `runTurnStep` activities +under a `sessionTurn` workflow and completed. (Earlier whole-turn-per-activity, stock-coordinator, and +shared-supervisor-via-shim modes were folded away.) A per-step re-drive resumes from the durable event log rather than re-running work. `runStep` closes any tool left dangling by an interrupted attempt on every entry, not just the first. Without that, a From 64a161f74192b064fc8e45409b6b5dffae74bc41 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 14 Aug 2026 20:35:25 -0700 Subject: [PATCH 073/103] Honored OPENCODE_SESSION_IDLE_TIMEOUT in temporal mode. Local mode reads the override at layer build; the Temporal workflow ran on the hard 5-minute default because the sandbox cannot read env. The client now forwards the override as a workflow argument, and a continue-as-new run keeps it. --- .../src/session/execution/temporal-workflow.ts | 15 ++++++++------- packages/core/src/session/execution/temporal.ts | 9 +++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index 01559229d027..04b55e5fed9d 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -39,7 +39,9 @@ export const interrupt = defineSignal("interrupt") export const resume = defineUpdate("resume") const signals = { wake, interrupt } as const -const runtime: WorkflowRuntime = { +// The runtime is built per invocation so a continue-as-new run keeps the same idle override; the +// sandbox cannot read env, so the override arrives as a workflow argument from the client. +const makeRuntime = (idleTimeout?: string): WorkflowRuntime => ({ condition: async (predicate, timeout) => { if (timeout === undefined) { await condition(predicate) @@ -53,11 +55,10 @@ const runtime: WorkflowRuntime = { runTurnStep, cancelCurrentScope: () => CancellationScope.current().cancel(), isCancellation, - continueAsNew: (sessionID) => continueAsNew<(id: string) => Promise>(sessionID), -} - -const workflows = makeWorkflows(runtime) + continueAsNew: (sessionID) => + continueAsNew<(id: string, idleTimeout?: string) => Promise>(sessionID, idleTimeout), +}) -export async function sessionTurn(sessionID: string): Promise { - return workflows.sessionTurn(sessionID) +export async function sessionTurn(sessionID: string, idleTimeout?: string): Promise { + return makeWorkflows(makeRuntime(idleTimeout), idleTimeout ? { idleTimeout } : undefined).sessionTurn(sessionID) } diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index b90ef231f933..d5b42a475f96 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -63,6 +63,11 @@ const layer = Layer.effect( // semantics and error encoding cannot differ between modes even though the loops differ. const { stepDrain } = makeDrains({ store, locations, ctx, events, worktrees }) + // Same knob local mode honors. The workflow sandbox cannot read env, so the client forwards the + // override as a workflow argument. Read at layer build (not module load) so tests can set it + // before constructing the layer. + const IDLE_TIMEOUT = process.env.OPENCODE_SESSION_IDLE_TIMEOUT + // Worker connection (native) hosts the runTurnStep activity + the workflow. Skipped in // client-only role so serve can run without an embedded worker. if (HOST_WORKER) { @@ -128,7 +133,7 @@ const layer = Layer.effect( client.workflow.signalWithStart(WORKFLOW_TYPE, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), - args: [id], + args: [id, IDLE_TIMEOUT], signal: WF.wake, signalArgs: [], }), @@ -176,7 +181,7 @@ const layer = Layer.effect( const startOp = new WithStartWorkflowOperation(WORKFLOW_TYPE, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), - args: [id], + args: [id, IDLE_TIMEOUT], workflowIdConflictPolicy: "USE_EXISTING", }) return client.workflow.executeUpdateWithStart(WF.resume, { From 85add019c8bee1da3f3aa8d11bb02f0b1a065179 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 14 Aug 2026 20:35:26 -0700 Subject: [PATCH 074/103] Pointed the driver-contract suite at the Temporal driver. The suite was parameterized for exactly this. The Temporal run is opt-in (OPENCODE_CONTRACT_TEMPORAL=1 against a dev server) with one task queue per run, so a stale worker on a shared server cannot steal activities; all four scenarios pass through real workflows. --- .../test/lib/session-execution-contract.ts | 243 +++++++++++++++++ .../session-execution-local-driver.test.ts | 248 +----------------- ...ession-execution-temporal-contract.test.ts | 21 ++ packages/temporal/README.md | 10 +- 4 files changed, 276 insertions(+), 246 deletions(-) create mode 100644 packages/core/test/lib/session-execution-contract.ts create mode 100644 packages/core/test/session-execution-temporal-contract.test.ts diff --git a/packages/core/test/lib/session-execution-contract.ts b/packages/core/test/lib/session-execution-contract.ts new file mode 100644 index 000000000000..5483afbda05c --- /dev/null +++ b/packages/core/test/lib/session-execution-contract.ts @@ -0,0 +1,243 @@ +// The SessionExecution driver-contract suite, parameterized over the driver factory. Each driver +// (the in-process native coordinator, the Temporal workflow) registers the SAME scenarios through +// runContract; this is what guarantees the modes agree now that they share only the drain, not one +// loop. The contract: wake drives a turn to settlement then the idle coordinator retires, resume +// forces a healthy turn to completion, resume surfaces the exact tagged RunError (through the same +// encode/decode path the Temporal boundary uses), and interrupt cancels an in-flight turn. +import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" +import { LLMEvent } from "@opencode-ai/llm" +import { Database } from "@opencode-ai/core/database/database" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { EventV2 } from "@opencode-ai/core/event" +import { PermissionV2 } from "@opencode-ai/core/permission" +import { Config } from "@opencode-ai/core/config" +import { Project } from "@opencode-ai/core/project" +import { ProjectTable } from "@opencode-ai/core/project/sql" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { SessionV2 } from "@opencode-ai/core/session" +import { Snapshot } from "@opencode-ai/core/snapshot" +import { SessionEvent } from "@opencode-ai/core/session/event" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import { SessionExecution } from "@opencode-ai/core/session/execution" +import { SessionExecutionLocalDriver } from "@opencode-ai/core/session/execution/local-driver" +import { SessionRunnerModel, ModelNotSelectedError } from "@opencode-ai/core/session/runner/model" +import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionStore } from "@opencode-ai/core/session/store" +import { SessionMessage } from "@opencode-ai/core/session/message" +import { Prompt } from "@opencode-ai/core/session/prompt" +import { Location } from "@opencode-ai/core/location" +import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" +import { SystemContext } from "@opencode-ai/core/system-context" +import { SkillGuidance } from "@opencode-ai/core/skill/guidance" +import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" +import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" +import { Auth } from "@opencode-ai/llm/route" +import { describe, expect } from "bun:test" +import { realpathSync } from "node:fs" +import { tmpdir } from "node:os" +import { Cause, Context, DateTime, Effect, Exit, Layer, Stream } from "effect" +import { testEffect } from "./effect" + +// The per-location service build resolves the session directory on disk, so it must exist. +const WORKSPACE = AbsolutePath.make(realpathSync(tmpdir())) + +const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) + .model({ id: "gpt-4o-mini" }) +const okModels = SessionRunnerModel.layerWith(() => Effect.succeed(model)) +const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) +const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) +const permission = Layer.mock(PermissionV2.Service, {}) + +const mockClient = (stream: LLMClientShape["stream"]) => + Layer.succeed( + LLMClient.Service, + LLMClient.Service.of({ + prepare: () => Effect.die("unused"), + generate: () => Effect.die("unused"), + stream, + }), + ) + +const countingModel = () => { + const requests: number[] = [] + const stream: LLMClientShape["stream"] = () => { + requests.push(1) + return Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })]) + } + return { requests, stream } +} + +// The executor under test, built as its own graph over the shared database file (the same way the +// serve process builds it), with the model/LLM mocked. Any SessionExecution node with the standard +// dependency set (the local coordinator, the Temporal driver) plugs in here. +export const makeExecutionFor = + (node: typeof SessionExecutionLocalDriver.node) => + (stream: LLMClientShape["stream"], models = okModels) => + AppNodeBuilder.build(node, [ + [LayerNodePlatform.llmClient, mockClient(stream)], + [PermissionV2.node, permission], + [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [SessionRunnerModel.node, models], + [SystemContextRegistry.node, systemContext], + [Location.node, Location.boundNode({ directory: WORKSPACE })], + [SkillGuidance.node, skillGuidance], + [ReferenceGuidance.node, referenceGuidance], + [Config.node, config], + [Snapshot.node, Snapshot.noopLayer], + ]) + +// Reads and seeds go through a separate graph sharing the same database file. +const it = testEffect( + AppNodeBuilder.build(LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node])), +) + +const seedSession = (sessionID: SessionV2.ID) => + Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: WORKSPACE, sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "t", + directory: WORKSPACE, + title: "t", + version: "t", + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + }) + +const seedPrompt = (sessionID: SessionV2.ID) => + Effect.gen(function* () { + const events = yield* EventV2.Service + yield* events.publish(SessionEvent.Prompted, { + sessionID, + timestamp: yield* DateTime.now, + messageID: SessionMessage.ID.create(), + prompt: Prompt.make({ text: "do the thing" }), + delivery: "queue", + }) + }) + +// Generous defaults: the Temporal driver adds worker startup, activity scheduling, and +// eventually-consistent visibility (about a second) on top of the turn itself. +const until = (read: Effect.Effect, predicate: (value: A) => boolean, timeoutMs = 20000) => + Effect.gen(function* () { + const deadline = Date.now() + timeoutMs + for (;;) { + const value = yield* read + if (predicate(value)) return value + if (Date.now() > deadline) throw new Error("condition not reached in time") + yield* Effect.sleep(50) + } + }) + +// The SessionExecution contract, parameterized over the driver factory. `makeExec` builds a +// SessionExecution graph the same way serve does, with the model/LLM mocked. Running the identical +// suite against a second factory is how the two modes are held to one behavior now that they no +// longer share a single coordination loop -- only the drain. +export const runContract = (label: string, makeExec: ReturnType) => { + const slug = label.replace(/[^a-z0-9]+/gi, "_") + describe(`SessionExecution contract: ${label}`, () => { + { + const { requests, stream } = countingModel() + const sessionID = SessionV2.ID.make(`ses_${slug}_wake`) + it.live("wake drives a turn to settlement, then the idle coordinator retires", () => + Effect.gen(function* () { + // Both drivers read this at layer build: the local coordinator directly, the Temporal + // client to forward it as a workflow argument. Restore it so later layer builds in this + // process get the real default. + const previousIdle = process.env.OPENCODE_SESSION_IDLE_TIMEOUT + process.env.OPENCODE_SESSION_IDLE_TIMEOUT = "2 seconds" + try { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + const exec = Context.get(yield* Layer.build(makeExec(stream)), SessionExecution.Service) + yield* exec.wake(sessionID) + const store = yield* SessionStore.Service + yield* until(store.context(sessionID), (context) => { + const assistant = context.findLast((message) => message.type === "assistant") + return assistant?.type === "assistant" && Boolean(assistant.time.completed) + }) + expect(requests).toHaveLength(1) + yield* until(exec.active, (active) => active.has(sessionID)) + // Idle self-termination: the coordinator retires without an interrupt. + yield* until(exec.active, (active) => !active.has(sessionID)) + } finally { + if (previousIdle === undefined) delete process.env.OPENCODE_SESSION_IDLE_TIMEOUT + else process.env.OPENCODE_SESSION_IDLE_TIMEOUT = previousIdle + } + }), + ) + } + + { + const { requests, stream } = countingModel() + const sessionID = SessionV2.ID.make(`ses_${slug}_resume_ok`) + it.live("resume forces a healthy turn to completion and resolves", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + const exec = Context.get(yield* Layer.build(makeExec(stream)), SessionExecution.Service) + // resume is request/response: it awaits the forced drain and resolves on success. + const exit = yield* exec.resume(sessionID).pipe(Effect.exit) + expect(Exit.isSuccess(exit)).toBe(true) + expect(requests).toHaveLength(1) + const store = yield* SessionStore.Service + const context = yield* store.context(sessionID) + const assistant = context.findLast((message) => message.type === "assistant") + expect(assistant?.type === "assistant" && Boolean(assistant.time.completed)).toBe(true) + }), + ) + } + + { + const sessionID = SessionV2.ID.make(`ses_${slug}_error`) + const failingModels = SessionRunnerModel.layerWith(() => Effect.fail(new ModelNotSelectedError({ sessionID }))) + it.live("resume surfaces the exact tagged RunError through the shared codec", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + const { stream } = countingModel() + const exec = Context.get(yield* Layer.build(makeExec(stream, failingModels)), SessionExecution.Service) + const exit = yield* exec.resume(sessionID).pipe(Effect.exit) + expect(Exit.isFailure(exit)).toBe(true) + const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined + // The same encode -> details -> decode path the Temporal boundary uses, so the caller gets + // the identical tagged instance in both modes. + expect(error).toBeInstanceOf(ModelNotSelectedError) + }), + ) + } + + { + const sessionID = SessionV2.ID.make(`ses_${slug}_interrupt`) + it.live("interrupt cancels an in-flight turn and the coordinator retires", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + // A model that never answers: the turn hangs until interrupted. + const exec = Context.get(yield* Layer.build(makeExec(() => Stream.never)), SessionExecution.Service) + yield* exec.wake(sessionID) + yield* Effect.sleep(200) + yield* until(exec.active, (active) => active.has(sessionID)) + yield* exec.interrupt(sessionID) + yield* until(exec.active, (active) => !active.has(sessionID)) + }), + ) + } + }) +} diff --git a/packages/core/test/session-execution-local-driver.test.ts b/packages/core/test/session-execution-local-driver.test.ts index 6210470d295b..6a16409dfe0f 100644 --- a/packages/core/test/session-execution-local-driver.test.ts +++ b/packages/core/test/session-execution-local-driver.test.ts @@ -1,244 +1,8 @@ -// The SessionExecution driver-contract suite, run here against the in-process native coordinator -// (local-driver.ts): a per-session async loop with no server. The suite is parameterized over the -// driver factory (runContract) so the SAME behaviors can be asserted against the Temporal driver; -// this is what guarantees the two modes agree now that they share only the drain, not one loop. -// The contract: wake drives a turn to settlement then the idle coordinator retires, resume forces a -// healthy turn to completion, resume surfaces the exact tagged RunError (through the same -// encode/decode path the Temporal boundary uses), and interrupt cancels an in-flight turn. -import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" -import { LLMEvent } from "@opencode-ai/llm" -import { Database } from "@opencode-ai/core/database/database" -import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" -import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" -import { LayerNode } from "@opencode-ai/core/effect/layer-node" -import { EventV2 } from "@opencode-ai/core/event" -import { PermissionV2 } from "@opencode-ai/core/permission" -import { Config } from "@opencode-ai/core/config" -import { Project } from "@opencode-ai/core/project" -import { ProjectTable } from "@opencode-ai/core/project/sql" -import { AbsolutePath } from "@opencode-ai/core/schema" -import { SessionV2 } from "@opencode-ai/core/session" -import { Snapshot } from "@opencode-ai/core/snapshot" -import { SessionEvent } from "@opencode-ai/core/session/event" -import { SessionProjector } from "@opencode-ai/core/session/projector" -import { SessionExecution } from "@opencode-ai/core/session/execution" +// The driver-contract suite run against the in-process native coordinator (local-driver.ts): a +// per-session async loop with no server. The suite itself lives in lib/session-execution-contract +// and also runs against the Temporal driver (session-execution-temporal-contract.test.ts); the two +// runs are what hold the modes to one behavior now that they share only the drain. import { SessionExecutionLocalDriver } from "@opencode-ai/core/session/execution/local-driver" -import { SessionRunnerModel, ModelNotSelectedError } from "@opencode-ai/core/session/runner/model" -import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" -import { SessionTable } from "@opencode-ai/core/session/sql" -import { SessionStore } from "@opencode-ai/core/session/store" -import { SessionMessage } from "@opencode-ai/core/session/message" -import { Prompt } from "@opencode-ai/core/session/prompt" -import { Location } from "@opencode-ai/core/location" -import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" -import { SystemContext } from "@opencode-ai/core/system-context" -import { SkillGuidance } from "@opencode-ai/core/skill/guidance" -import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" -import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" -import { Auth } from "@opencode-ai/llm/route" -import { describe, expect } from "bun:test" -import { realpathSync } from "node:fs" -import { tmpdir } from "node:os" -import { Cause, Context, DateTime, Effect, Exit, Layer, Stream } from "effect" -import { testEffect } from "./lib/effect" +import { makeExecutionFor, runContract } from "./lib/session-execution-contract" -// The per-location service build resolves the session directory on disk, so it must exist. -const WORKSPACE = AbsolutePath.make(realpathSync(tmpdir())) - -const model = OpenAIChat.route - .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) - .model({ id: "gpt-4o-mini" }) -const okModels = SessionRunnerModel.layerWith(() => Effect.succeed(model)) -const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) -const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) -const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) -const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) -const permission = Layer.mock(PermissionV2.Service, {}) - -const mockClient = (stream: LLMClientShape["stream"]) => - Layer.succeed( - LLMClient.Service, - LLMClient.Service.of({ - prepare: () => Effect.die("unused"), - generate: () => Effect.die("unused"), - stream, - }), - ) - -const countingModel = () => { - const requests: number[] = [] - const stream: LLMClientShape["stream"] = () => { - requests.push(1) - return Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })]) - } - return { requests, stream } -} - -// The executor under test, built as its own graph over the shared database file (the same way the -// serve process builds it), with the model/LLM mocked. -const makeExecution = (stream: LLMClientShape["stream"], models = okModels) => - AppNodeBuilder.build(SessionExecutionLocalDriver.node, [ - [LayerNodePlatform.llmClient, mockClient(stream)], - [PermissionV2.node, permission], - [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], - [SessionRunnerModel.node, models], - [SystemContextRegistry.node, systemContext], - [Location.node, Location.boundNode({ directory: WORKSPACE })], - [SkillGuidance.node, skillGuidance], - [ReferenceGuidance.node, referenceGuidance], - [Config.node, config], - [Snapshot.node, Snapshot.noopLayer], - ]) - -// Reads and seeds go through a separate graph sharing the same database file. -const it = testEffect( - AppNodeBuilder.build(LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node])), -) - -const seedSession = (sessionID: SessionV2.ID) => - Effect.gen(function* () { - const { db } = yield* Database.Service - yield* db - .insert(ProjectTable) - .values({ id: Project.ID.global, worktree: WORKSPACE, sandboxes: [] }) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie) - yield* db - .insert(SessionTable) - .values({ - id: sessionID, - project_id: Project.ID.global, - slug: "t", - directory: WORKSPACE, - title: "t", - version: "t", - }) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie) - }) - -const seedPrompt = (sessionID: SessionV2.ID) => - Effect.gen(function* () { - const events = yield* EventV2.Service - yield* events.publish(SessionEvent.Prompted, { - sessionID, - timestamp: yield* DateTime.now, - messageID: SessionMessage.ID.create(), - prompt: Prompt.make({ text: "do the thing" }), - delivery: "queue", - }) - }) - -const until = (read: Effect.Effect, predicate: (value: A) => boolean, timeoutMs = 8000) => - Effect.gen(function* () { - const deadline = Date.now() + timeoutMs - for (;;) { - const value = yield* read - if (predicate(value)) return value - if (Date.now() > deadline) throw new Error("condition not reached in time") - yield* Effect.sleep(50) - } - }) - -// The SessionExecution contract, parameterized over the driver factory. `makeExec` builds a -// SessionExecution graph the same way serve does, with the model/LLM mocked. Running the identical -// suite against a second factory (e.g. a Temporal test-env node) is how the two modes are held to -// one behavior now that they no longer share a single coordination loop -- only the drain. -const runContract = ( - label: string, - makeExec: (stream: LLMClientShape["stream"], models?: typeof okModels) => ReturnType, -) => { - const slug = label.replace(/[^a-z0-9]+/gi, "_") - describe(`SessionExecution contract: ${label}`, () => { - { - const { requests, stream } = countingModel() - const sessionID = SessionV2.ID.make(`ses_${slug}_wake`) - it.live("wake drives a turn to settlement, then the idle coordinator retires", () => - Effect.gen(function* () { - const previousIdle = process.env.OPENCODE_SESSION_IDLE_TIMEOUT - process.env.OPENCODE_SESSION_IDLE_TIMEOUT = "2 seconds" - yield* seedSession(sessionID) - yield* seedPrompt(sessionID) - const exec = Context.get(yield* Layer.build(makeExec(stream)), SessionExecution.Service) - yield* exec.wake(sessionID) - const store = yield* SessionStore.Service - yield* until(store.context(sessionID), (context) => { - const assistant = context.findLast((message) => message.type === "assistant") - return assistant?.type === "assistant" && Boolean(assistant.time.completed) - }) - expect(requests).toHaveLength(1) - expect((yield* exec.active).has(sessionID)).toBe(true) - // Idle self-termination: the coordinator retires without an interrupt. - yield* until(exec.active, (active) => !active.has(sessionID)) - // Restore so later layer builds in this process get the real default. - if (previousIdle === undefined) delete process.env.OPENCODE_SESSION_IDLE_TIMEOUT - else process.env.OPENCODE_SESSION_IDLE_TIMEOUT = previousIdle - }), - ) - } - - { - const { requests, stream } = countingModel() - const sessionID = SessionV2.ID.make(`ses_${slug}_resume_ok`) - it.live("resume forces a healthy turn to completion and resolves", () => - Effect.gen(function* () { - yield* seedSession(sessionID) - yield* seedPrompt(sessionID) - const exec = Context.get(yield* Layer.build(makeExec(stream)), SessionExecution.Service) - // resume is request/response: it awaits the forced drain and resolves on success. - const exit = yield* exec.resume(sessionID).pipe(Effect.exit) - expect(Exit.isSuccess(exit)).toBe(true) - expect(requests).toHaveLength(1) - const store = yield* SessionStore.Service - const context = yield* store.context(sessionID) - const assistant = context.findLast((message) => message.type === "assistant") - expect(assistant?.type === "assistant" && Boolean(assistant.time.completed)).toBe(true) - }), - ) - } - - { - const sessionID = SessionV2.ID.make(`ses_${slug}_error`) - const failingModels = SessionRunnerModel.layerWith(() => - Effect.fail(new ModelNotSelectedError({ sessionID })), - ) - it.live("resume surfaces the exact tagged RunError through the shared codec", () => - Effect.gen(function* () { - yield* seedSession(sessionID) - const { stream } = countingModel() - const exec = Context.get(yield* Layer.build(makeExec(stream, failingModels)), SessionExecution.Service) - const exit = yield* exec.resume(sessionID).pipe(Effect.exit) - expect(Exit.isFailure(exit)).toBe(true) - const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined - // The same encode -> details -> decode path the Temporal boundary uses, so the caller gets - // the identical tagged instance in both modes. - expect(error).toBeInstanceOf(ModelNotSelectedError) - }), - ) - } - - { - const sessionID = SessionV2.ID.make(`ses_${slug}_interrupt`) - it.live("interrupt cancels an in-flight turn and the coordinator retires", () => - Effect.gen(function* () { - yield* seedSession(sessionID) - yield* seedPrompt(sessionID) - // A model that never answers: the turn hangs until interrupted. - const exec = Context.get( - yield* Layer.build(makeExec(() => Stream.never)), - SessionExecution.Service, - ) - yield* exec.wake(sessionID) - yield* Effect.sleep(200) - expect((yield* exec.active).has(sessionID)).toBe(true) - yield* exec.interrupt(sessionID) - yield* until(exec.active, (active) => !active.has(sessionID), 4000) - }), - ) - } - }) -} - -runContract("local coordinator", makeExecution) +runContract("local coordinator", makeExecutionFor(SessionExecutionLocalDriver.node)) diff --git a/packages/core/test/session-execution-temporal-contract.test.ts b/packages/core/test/session-execution-temporal-contract.test.ts new file mode 100644 index 000000000000..a8f11d35923b --- /dev/null +++ b/packages/core/test/session-execution-temporal-contract.test.ts @@ -0,0 +1,21 @@ +// The driver-contract suite run against the Temporal driver: the same scenarios the local +// coordinator passes, driven through real workflows on a Temporal server. This is the second half +// of the parity story in packages/temporal/README.md "Two modes, one drain". +// +// It needs a dev server and generous timeouts (worker startup bundles the workflow), so it is +// opt-in: +// +// temporal server start-dev --port 7237 --headless & +// OPENCODE_CONTRACT_TEMPORAL=1 bun test --timeout 120000 test/session-execution-temporal-contract.test.ts +// +// Without the opt-in the file registers nothing, so a plain `bun test` stays server-free. +import { makeExecutionFor, runContract } from "./lib/session-execution-contract" + +if (process.env.OPENCODE_CONTRACT_TEMPORAL === "1") { + // One task queue per run: a stale worker from an earlier run against the same dev server would + // otherwise steal activities and answer with its own (differently mocked) graph. + process.env.OPENCODE_TEMPORAL_TASK_QUEUE ??= `contract-${crypto.randomUUID()}` + // Imported dynamically because the driver reads its connection config at module load. + const { SessionExecutionTemporal } = await import("@opencode-ai/core/session/execution/temporal") + runContract("temporal driver", makeExecutionFor(SessionExecutionTemporal.node)) +} diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 6fc68d8058bd..f3a490e93e95 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -106,11 +106,13 @@ that would corrupt state (log fencing, error encoding, tool re-drive) all live i Both loops drive the turn one **step** at a time: they loop a `runTurnStep` drain, so in temporal mode each step (one provider attempt + its tools) is its own activity with its own retry/timeout/visibility. Both reuse `SessionRunner.runStep` (one iteration of `run`'s loop), so the -turn semantics are unchanged. Parity is enforced by the driver-contract test -(`packages/core/test/session-execution-local-driver.test.ts`): one suite -- wake drives a turn then +turn semantics are unchanged. Parity is enforced by the driver-contract suite +(`packages/core/test/lib/session-execution-contract.ts`): one suite -- wake drives a turn then the idle coordinator retires, resume forces a healthy turn and surfaces the exact tagged RunError, -interrupt cancels -- parameterized over the coordinator factory so the same behaviors can be asserted -against Temporal. Verified: a create-then-read-then-reply turn recorded three `runTurnStep` activities +interrupt cancels -- run against BOTH drivers. The local run is part of the normal test suite; the +Temporal run is opt-in against a dev server (recipe in +`packages/core/test/session-execution-temporal-contract.test.ts`) and passes the same four +scenarios through real workflows. Verified: a create-then-read-then-reply turn recorded three `runTurnStep` activities under a `sessionTurn` workflow and completed. (Earlier whole-turn-per-activity, stock-coordinator, and shared-supervisor-via-shim modes were folded away.) From aa73d9af60cb11e963cb25fe44addea6c77d6b1e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 14 Aug 2026 20:35:26 -0700 Subject: [PATCH 075/103] Replaced the local driver's duration table with effect's Duration. The hand-rolled unit table duplicated Duration.fromInputUnsafe. --- .../src/session/execution/local-driver.ts | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts index 22f2bfa172e4..138f41c48694 100644 --- a/packages/core/src/session/execution/local-driver.ts +++ b/packages/core/src/session/execution/local-driver.ts @@ -11,7 +11,7 @@ export * as SessionExecutionLocalDriver from "./local-driver" // shared drain and by the driver-contract test (session-execution-local-driver.test.ts), not by a // single shared loop. See packages/temporal/README.md "Two modes, one drain". -import { Effect, Layer } from "effect" +import { Duration, Effect, Layer } from "effect" import { randomUUID } from "node:crypto" import { LocationServiceMap } from "../../location-service-map" import { EventV2 } from "../../event" @@ -23,24 +23,7 @@ import { makeDrains } from "./drain" import { WorktreeMaterializer } from "./worktree" import { toRunError } from "./run-error-codec" -const UNITS: Record = { - ms: 1, - millisecond: 1, - milliseconds: 1, - second: 1_000, - seconds: 1_000, - minute: 60_000, - minutes: 60_000, - hour: 3_600_000, - hours: 3_600_000, -} - -const parseDuration = (value: string): number => { - const match = /^\s*([\d.]+)\s*([a-z]+)\s*$/i.exec(value) - const unit = match?.[2] ? UNITS[match[2].toLowerCase()] : undefined - if (!match?.[1] || unit === undefined) throw new Error(`Unsupported duration: ${value}`) - return Number(match[1]) * unit -} +const parseDuration = (value: string): number => Duration.toMillis(Duration.fromInputUnsafe(value as Duration.Input)) // A drain that is interrupted (by an explicit stop or the backstop) throws this. The drain body // rethrows the AbortSignal's reason on cancellation, so a cancelled drain and a stop both surface From 1d9103ce434375f3c612ca989e0a1f012e23f622 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 14 Aug 2026 22:39:34 -0700 Subject: [PATCH 076/103] Pruned the internal evaluation docs from the branch. The AI-399 evaluation and the worktree design memo are internal decision records, not part of the change this branch proposes. The shipped worktree mechanism stays documented in the README and the code; the one load-bearing note (warm-path alternatives) moved inline. --- packages/temporal/README.md | 7 +- packages/temporal/docs/ai399-local-options.md | 337 ------------------ .../temporal/docs/worktree-portability.md | 65 ---- 3 files changed, 3 insertions(+), 406 deletions(-) delete mode 100644 packages/temporal/docs/ai399-local-options.md delete mode 100644 packages/temporal/docs/worktree-portability.md diff --git a/packages/temporal/README.md b/packages/temporal/README.md index f3a490e93e95..4b7b58d89794 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -38,8 +38,7 @@ That forces six things: drain claims the log with an attempt token ([Notes](#notes)). 4. **The worktree must travel.** Snapshot trees ship as incremental git packs, and a worker without the project tree rebuilds it before the run - ([What resumes cross-host](#what-resumes-cross-host-and-what-does-not), - [docs/worktree-portability.md](docs/worktree-portability.md)). + ([What resumes cross-host](#what-resumes-cross-host-and-what-does-not)). 5. **Human-in-the-loop must be durable.** A pending permission ask is a row in the shared store, answerable from any process, adopted by re-drives, expired when abandoned ([Durable permission asks](#durable-permission-asks)). @@ -265,8 +264,8 @@ drain runs, a worker missing the session's directory rebuilds the worktree from (`session/execution/worktree.ts`): uncommitted edits and untracked files included, checked out at the same absolute path it was captured at (a uniform fleet layout). Ignored files and dependencies are not captured, so a rebuilt tree may need an install step before `bash` behaves identically. -[docs/worktree-portability.md](docs/worktree-portability.md) covers the design and the -affinity/shared-volume alternatives that skip materialization latency on warm paths. +Worker affinity or a shared volume skips the materialization latency on warm paths; the packs +are the portable baseline that works with neither. Host-local state that does NOT ride the DB, so it is not reconstructed on a different host: diff --git a/packages/temporal/docs/ai399-local-options.md b/packages/temporal/docs/ai399-local-options.md deleted file mode 100644 index 4c854a83996f..000000000000 --- a/packages/temporal/docs/ai399-local-options.md +++ /dev/null @@ -1,337 +0,0 @@ -# AI-399: Local options for Temporal-integrated agents - -Some customers integrate Temporal into their agents and want the same agent loop to run without -Temporal, e.g. shipped desktop software. This evaluates the candidate paths and recommends what to -tell customers and what to build. It lives on this branch because the fork around it is first-hand -evidence: one engine, two execution modes (in-process, one Temporal activity per step), selected -by an env var. - -## The requirement, sharpened - -"Run without Temporal" hides three different customer asks, and the right answer differs: - -1. **Same loop, no infrastructure at all** (desktop/CLI): no server process, no ports, minimal - footprint. Durability degrades gracefully (a checkpoint file beats nothing). -2. **Same loop, no OPERATED infrastructure** (an appliance: one machine the customer operates, - e.g. an on-prem server box or an edge device): a local process is fine if it is zero-admin. - Full Temporal semantics wanted. -3. **Same CODE, both worlds**: the vendor ships one codebase; cloud deployments get durability, - desktop gets local. The dominant engineering constraint is preventing drift between the paths. - -The motivating desktop customer (Cursor) rejects a bundled server process, so archetype 1 must be -answered in-process. - -## Paths - -### A. Shim Temporal at the language layer - -Replace the Temporal SDK surface the agent code touches (`workflow.*`, `proxyActivities`, -signals/updates/timers) with a local implementation, so unmodified workflow code runs in-process. - -- Exists today: customers have added shims like this in their own code (per the ticket), paying - the cost themselves. -- Durability: whatever the shim persists. A faithful shim needs event-sourced replay to recover - mid-workflow, which is the hard part of Temporal, reimplemented. -- Signals/Updates: re-implemented on the shim's event loop; semantic-drift risk is high - (buffering, ordering, update validators, cancellation scopes). -- Complexity: high and permanent for a general shim; the shim chases the SDK surface every - release, and determinism constraints stay imposed on local code that gets nothing for them. A - shim scoped to the app's actual control surface (enumerated below) is much smaller, at the cost - of being app-specific. -- Where it shines: an existing Temporal-first codebase that cannot be refactored, needing a local - mode quickly. - -### B. Shim in the Rust core - -Implement a local mode under the core SDK so all core-based SDKs (TypeScript, Python, .NET, Ruby) -get it at once. - -Research findings (sdk-core is now `temporalio/sdk-rust`; MIT): - -- **The insertion point is clean and already public.** `ConnectionOptions.service_override` - (`crates/client/src/options_structs.rs`) routes every gRPC call through a supplied callback - instead of the network, and the C ABI already plumbs it (`grpc_override_callback` in - `sdk-core-c-bridge`), so .NET/Ruby could intercept in-process today. Python does not expose it; - TypeScript unverified. Why the hook exists is unverified (possibly proxying/testing). -- **Determinism, state machines, and replay live in core, not the server**, so a shim does NOT - reimplement the hard part. What it must implement is the SERVICE: task matching with long-poll - semantics, history append/read, workflow task lifecycle, server-side timers, activity retry and - four timeout types, signals/queries/updates (update's multi-stage lifecycle), ID reuse/conflict - policies, continue-as-new, child workflows, cancellation. -- **The honest size estimate is the Java time-skipping test server**: a from-scratch in-memory - service reimplementation, GraalVM-compiled, consumed by Python/.NET/TS/Ruby, and still short of - parity after years (`sdk-java#1804`, a Temporal employee asking for the real dev server in - tests because `listWorkflowExecutions` is missing). -- Core's replay worker (`init_replay_worker`) proves the plumbing tolerates a non-network client, - but it consumes pre-recorded history only; it is not a local runtime. - -Read: the seam is cheap, the payload is a second implementation of the Temporal service, and -Temporal's existing second implementation has not reached parity. Only worth doing as an owned -product commitment ("embedded Temporal"), not as a workaround. - -### C. Plugin pattern: one loop, swappable execution - -Factor the agent so the LOOP is pure and the execution substrate is injected; Temporal is one -substrate, a local runner is another. - -**Evidence 1: this fork.** opencode's v2 engine event-sources every session to a store and exposes -a substitutable `SessionExecution` interface (`active`/`wake`/`resume`/`interrupt`, four methods). -The stock implementation is an in-process coordinator; this fork adds a Temporal-backed one -(`OPENCODE_SESSION_EXECUTION=temporal`, one activity per step) plus a shared store so -any worker resumes any session. Because durability lives in the engine's event log, the LOCAL mode -is already crash-recoverable without Temporal; Temporal adds supervised retries, worker -distribution, restart-surviving visibility, and cross-process interrupt/resume. The swap point is -the execution supervisor, not the state store. Verified end to end in this branch (crash tests, -failover tests, an independent architecture review). - -**Evidence 2: `agent-harness` (AI-363, TypeScript).** A pure loop state machine plus an `Effects` -interface (`callModel`/`runTools`/`onEvent`) implemented twice: local mode (direct calls, atomic -write-then-rename checkpoint file, crash-resume) and durable mode (workflow + activities with -measured defaults). About 100 lines per runner around a shared core. This is the TypeScript -prototype the ticket's "has been implemented" refers to. Local mode gives up retries across -process death mid-tool, multi-worker capacity, and the audit trail; it keeps the same loop, tools, -prompts, and checkpoint crash-resume. - -**Evidence 3: Temporal already ships a dual-mode ADK integration.** The ticket points at ADK as -the example; the official integration (`pip install "temporalio[google-adk]"`, -`temporalio.contrib.google_adk_agents`, experimental) is shipped and dual-mode. Its mechanism is -an ambient check, not a factory: - -- ADK's own seam is the `Runner` + `BaseSessionService` (in-memory, sqlite, database, vertex - implementations), so local ADK needs no infrastructure. -- The Temporal integration does not replace the Runner. It wraps the model seam (`TemporalModel` - runs the call as an `invoke_model` activity) and the tool seam (`activity_tool` dispatches via - `workflow.execute_activity`), and each wrapper degrades to a direct in-process call when - `temporalio.workflow.in_workflow()` is false. Determinism helpers branch on the same predicate - (clock and ID providers, via hooks Temporal landed upstream in ADK, - `google.adk.platform.{time,uuid}`). -- One agent definition, two execution modes, no second code path for the user. Sharp edges worth - telling customers: MCP toolsets cannot auto-fall-back (the caller must supply - `not_in_workflow_toolset`), and durable mode is STRICTER than local (whole session state must - serialize within payload limits), so local-only testing can pass and then fail under Temporal. - -The ambient check is the compatibility variant of this pattern, not the recommended shape. It -exists because Temporal does not own ADK's composition root, and because a workflow cannot receive -a live object graph as input, so the check detects which behavior is legal in the current -environment. Its costs are structural: the mode is invisible at call sites, the conditional -repeats in every wrapper, the type system cannot enforce a complete local graph, and where uniform -degradation is impossible the failure is a runtime raise (the MCP edge above) instead of a -construction-time requirement. When you own the composition root, prefer a factory that returns -the interface: both implementations above do exactly that (one binding selects the -`SessionExecution` implementation; one `Effects` value selects the runner), and a missing local -implementation then fails at construction, not mid-run. The two shapes compose: a substrate -factory at the agent-definition level with the ambient check kept only as a safety net would -also close the MCP hole in ADK-shaped integrations. - -Characteristics of the family: - -- Durability: local = what the local runner persists (checkpoint file is coarse; an event-sourced - store is fine-grained and close to Temporal-grade for single-machine crashes). Temporal = full. -- Signals/Updates: the app defines the interface both modes honor. No pretense of Temporal's - generic protocol locally, and no drift, because the loop is the same code. -- Complexity: lowest sustained cost of all paths; the seam is app-defined and small, or (ADK - style) hidden inside integration wrappers. The cost is up-front design; not a bolt-on for an - existing Temporal-first codebase, except where Temporal ships the integration. - -### D. Run the Temporal dev server locally - -Ship `temporal server start-dev` alongside the app; the agent stays a plain Temporal application. - -Facts (research verified on release artifacts, v1.8.2, 2026-07; local measurements on this -machine's dev build): - -| Metric | Value | -|---|---| -| Release binary (darwin arm64) | 127.5 MiB uncompressed, 37.5 MiB compressed (this machine's 237 MB was a dev build; use the release number) | -| Platforms | macOS/Linux/Windows, amd64+arm64, one static Go binary incl. server, CLI, Web UI | -| Cold start to healthy | ~780 ms (`--headless`) | -| RSS idle | ~102-139 MB (this machine 102 MB; research 139 MB on v1.31.2) | -| SQLite file | ~0.6 MB empty | -| Persistence | `--db-filename` required; the DEFAULT is in-memory and loses everything on exit | -| Embedding-friendly flags | `--headless`, `--db-filename`, `--port`, `--ip`, repeatable `--namespace`, `--sqlite-pragma`, `--dynamic-config-value`; http/metrics ports default to random free | -| License | MIT (verified in release tarball and via GitHub API) | - -On this app the desktop number is 297 MB RSS, one process, local mode. A bundled server was -measured once to close the desktop question and is not a configuration anyone ships: desktop is -local mode, and temporal mode's server is Cloud or a fleet. The standalone figures above are the -appliance-archetype numbers. - -- **Fidelity is the differentiator: it is the real server against SQLite, not an emulator.** - Research verified multi-namespace and Nexus endpoints work and persist. Signals, updates, - queries, schedules, search attributes are the real implementations. No drift, ever. -- **Positioning is the weakness.** The binary itself prints a not-for-production warning (added - deliberately, `cli#689`); the embedded-server docs page says testing and development only; - limits are real (SQLite single writer, `NumHistoryShards: 1`, all roles in one process). - Restate and Inngest bless their single-node binaries for production; Temporal is the only one - in the comparison set whose local mode is officially disowned. -- Redistribution: no licensing issue (MIT). A real option for the appliance archetype. The - motivating desktop customer does not want a bundled server process, so this path is parked for - desktop. - -### E. Integrate a local option into the Temporal Agent Harness - -Not an independent runtime: it is where a choice among A/C/D becomes product. The harness today is -Temporal-native (agents ARE workflows; approvals, Code Mode, callback tools, and the event stream -ride Temporal primitives). A local mode via C would make the harness's public abstractions (agent -definition, tools, approval policy, event stream) the swap seam with a non-Temporal transport -locally (in-process bus, identical schemas). The ADK integration's `in_workflow()` mechanism is -the shipped precedent for how the harness's Temporal-aware pieces could degrade in-process. - -## One supervisor, two drivers (demonstrated on this branch) - -The two-implementations objection to the plugin pattern is real: the loop is written once, but the -SUPERVISOR contract (drains serialize, wakes coalesce, interrupt cancels, resume surfaces the -error) existed twice, once as the local coordinator and once as the workflow, and the independent -review found bugs precisely in the duplicated copy. This branch now closes that gap: - -- `workflow-core.ts`: the supervisor, written ONCE, over a six-primitive `WorkflowRuntime` - interface (`condition`, signal handlers, update handlers, the drain calls, cancellation, - `isCancellation`). Temporal's sandbox had already forced it to be pure, which is what makes it - executable anywhere. -- `temporal-workflow.ts`: the Temporal driver; the real SDK provides the six primitives, the - drains run as activities. -- `local-driver.ts` (the default mode; `OPENCODE_SESSION_EXECUTION=temporal` selects the Temporal - driver instead): the in-process driver; plain promises provide the primitives (a polled - `condition`, method-call signals, an `AbortController` for cancellation), the drains run - directly. No server, no worker, no ports. -- `drain.ts` and the error codec are shared modules, so turn semantics and typed errors are - byte-identical in both modes; contract tests drive the shared supervisor in-process (a turn - settles, the exact tagged `RunError` crosses the same encode/decode path, interrupt cancels, an - idle supervisor retires), and the worker smoke proves the same file still bundles in the - Temporal sandbox. - -The factory still hides the choice, and it is now exactly two modes: the in-process driver by -default, `temporal` for the server-backed one. The earlier whole-turn and stock-coordinator modes -were folded away once the shared supervisor made them redundant. This is the factory-shaped answer -to the ADK integration's ambient check, demonstrated. - -## Can the whole SDK surface be covered this way? - -The micro-driver needed six primitives because the supervisor uses six. Extending it across -`@temporalio/workflow`'s API splits cleanly: - -- **Mechanical for live execution:** `sleep`/timers (`setTimeout`), `workflow.now` (`Date.now`), - `random`/`uuid4` (plain random: determinism only matters for replay, which a local driver never - does), queries (read a handler map), `patched`/`deprecatePatch` (constant true / no-op), - `sideEffect` (run the function), search attributes and memo (a local map), logging sinks - (console), child workflows and external handles (spawn sibling drivers, route signals through an - in-process registry), continue-as-new (re-invoke the function with the new arguments). -- **The fundamental line is replay.** Temporal recovers workflow-VARIABLE state after a crash by - replaying history; a local driver has no history, so in-flight workflow variables and pending - timers die with the process. Covering that is not a shim, it is the embedded-service payload of - path B. - -Two consequences. First, the design rule for dual-mode apps: keep durable truth in an app-owned -log and treat workflow variables as ephemeral. This branch already obeys it (supervisor state is -reconstructible; turn state is in the event store; re-drives are log-based), which is why the -local driver loses nothing that matters on a desktop crash, and it is the rule to hand any -customer taking this path. Second, the honest scope statement: a whole-SDK local runtime is -achievable for LIVE semantics as a bounded engineering effort, but replay-grade durability of -workflow-local state is exactly where "cover the SDK surface" becomes "build embedded Temporal", -and should be decided as that (path B), not approached incrementally by accident. - -## What the local mode actually has to support: the control surface - -Measured from opencode's protocol (the session group plus human-in-the-loop groups), the session -control surface a desktop agent product exposes is enumerable, roughly a dozen verbs: - -- Run control: `prompt` (with two delivery semantics: steer into the running turn, or queue after - it), `interrupt`, `wait` (await settlement). -- Human-in-the-loop: permission reply (`once`/`always`/reject-with-correction), agent questions - (typed ask/answer). -- Session mutation: `switchAgent`, `switchModel`, `compact`, `revert` (stage/clear/commit). -- Observation: history, context, live event stream, active set. - -This is far richer than a cancel button, and far smaller than Temporal's generic signal/update -protocol. Every verb maps to a signal/update/query in Temporal mode and an in-process call -locally; the event-sourced store is what lets both modes serve the observation verbs identically. -This bounds path A (a scoped shim is a dozen verbs, not the SDK) and explains why path C stays -cheap: the interface already exists in any real product. - -## Prior art: how others answer local-without-the-big-server - -| Pattern | Example | Mechanism | Gives up | -|---|---|---|---| -| Pluggable store, same process | LangGraph (`checkpointer=`: memory/sqlite/postgres); DBOS (Python defaults to SQLite) | one interface, N stores | LangGraph memory: restart durability; DBOS local: multi-process recovery | -| Same binary, different config | Restate (single binary, RocksDB); Inngest (`inngest dev`, in-memory by default; `inngest start` for prod) | run the real thing small | fault tolerance of a cluster; a process always runs | -| Emulator | Azure Durable Task Scheduler emulator | separate implementation | fidelity; explicitly not production | -| In-process protocol reimplementation | Resonate `LocalNetwork` (server state machine over dicts); Temporal's Java test server | exact semantics, zero deps | maintaining two implementations forever | - -Temporal's dev server is the second pattern with one difference: Restate and Inngest bless their -single-node story for production; Temporal explicitly does not. - -## Demand evidence - -The ask is real, old, and largely unanswered publicly: - -- `temporalio/temporal#298` "run Temporal as an embedded library", opened 2020 by Maxim Fateev - ("For small scale on prem deployments... would be really great"), closed 2026 pointing at - `start-dev` with not-for-production caveats; requesters explicitly disputed the closure (the - actual ask was an in-process, NATS-style embedded server). -- Three desktop/OEM forum threads (2023-2025: Wails desktop app, standalone Go binary packaging, - bundled-Temporal upgrade sequencing) with zero staff replies among them. -- The only substantive edge guidance is a 2025 forum answer (IoT/submarine): run the single - binary on the device, federate with Nexus; which sits against docs saying SQLite single-process - is testing/development only. -- Two abandoned commitments: Maxim in 2020 ("Desktop applications... we absolutely going to - create it. No ETA"); Temporalite's maintainer in 2022 ("the vision is for Temporalite to be - used in production contexts"). Temporalite is archived; `temporal#3366` (SQLite in production) - is still open. - -## Comparison - -| | A: language shim | B: rust-core shim | C: plugin pattern (incl. ADK-style fallback) | D: local dev server | -|---|---|---|---|---| -| What it is | reimplement the SDK API in the app's language so Temporal-shaped code runs with no server; the customer owns it | embed a real Temporal service in the SDK core behind `service_override`; one in-process backend under every language | the loop behind a small execution interface; a startup factory picks Temporal or in-process, same code both modes | bundle the real dev server binary next to the app, which starts and supervises it | -| Durability (crash mid-turn) | partial: what the shim persists; faithful replay = reimplementing Temporal | full IF built (it IS an embedded service) | partial: checkpoint-file coarse; event-sourced local store near-full for one machine | full (with `--db-filename`; default is in-memory) | -| Resource needs | lightest (in-process) | in-process; core carries matching+history | lightest (in-process; store is a file) | ~102-139 MB RSS second process, ~128 MiB disk (appliance archetype; not a desktop configuration) | -| Signal/Update support | re-implemented, drift-prone | full IF built | the app's control surface (~a dozen verbs), honored identically by both modes | full | -| Complexity / maintainability | high, permanent (chases SDK surface per language); bounded if scoped to the control surface | highest; a second Temporal service implementation (Java test server: years, still short of parity) | lowest sustained; up-front loop design, or shipped by Temporal (ADK) | near-zero code; packaging+lifecycle burden | -| Feature fidelity | subset, hand-built | full IF built | app semantics, not Temporal's | full: the real server (multi-namespace, Nexus verified) | -| Code-drift between modes | medium (same code, different semantics) | low | none for the loop; ADK caveat: durable mode is stricter (serialization), so local-pass/durable-fail exists | none | -| Desktop packaging | best | good | best | worst (bundle+supervise a server); MIT, no licensing issue | -| Upgrade path to Temporal Cloud | same code, repoint | same code, repoint | swap the factory / connect the client | same code, repoint | -| Exists today | yes, by customers in their own code | no (seam exists: `service_override`; payload does not) | yes, three times: this fork, agent-harness, and the shipped ADK integration | yes (shipped CLI) | - -## Recommendations - -1. **For customers on a framework Temporal integrates with (ADK today): point at the shipped - integration.** The `in_workflow()` fallback is the productized version of the plugin pattern: - one agent definition, two modes, maintained by Temporal. Write up the two sharp edges (MCP - toolsets need an explicit local implementation; durable mode is stricter than local, so test - both modes). -2. **For customers designing their own loop: recommend the plugin pattern (C).** Two working - implementations here show the cost is one small interface. Enumerate the control surface (it - is about a dozen verbs in a real product) and event-source the session when local durability - matters, rather than reimplementing Temporal's replay. -3. **For existing Temporal-first agents on machines the customer controls (appliance, - single-machine): D**, with `--db-filename` and process supervision. Full fidelity, no - licensing issue. Off the table for the desktop archetype by customer preference. -4. **A is the fallback** when neither refactoring (C) nor a second process (D) is acceptable: - scope the shim to the control surface, not the SDK, and accept the drift risk. -5. **B is a product decision, not a customer recommendation.** The seam exists - (`service_override`, already in the C bridge); the payload is a second Temporal service - implementation, and our own Java test server shows the parity cost. The demand evidence - (six years of #298, unanswered desktop/OEM threads, two abandoned commitments) says there is a - real, unclaimed "embedded Temporal" position; claiming it means owning that implementation - indefinitely. That decision belongs to product, informed by this doc. -6. **Harness (E): apply C at the harness API layer**, as a factory returning the harness's - execution interface, chosen once at the composition root. Not ambient `in_workflow()` checks: - we own this composition root, so the local implementation should be enforced at construction - time. The ADK integration is the precedent for the wrapper technique where we do not own the - root, not for the harness's own design. - -## Sources - -Primary: this branch (implementation + measurements); `agent-harness` (AI-363); -`temporalio/sdk-rust` source; `temporalio/sdk-python` `contrib/google_adk_agents` source; a -downloaded v1.8.2 CLI release binary, run headless. - -Referenced: adk.dev/integrations/temporal · temporal.io/blog/google-adk-temporal-integration-bts · -temporalio/samples-python `google_adk_agents` · google/adk-python `sessions/` · -docs.temporal.io/self-hosted-guide/embedded-server · docs.temporal.io/cli/server · -temporalio/cli#689 · temporalio/temporal#298 · temporalio/temporal#3366 · -temporalio/sdk-java#1804 · temporalio/temporalite-archived · community.temporal.io threads -17424, 12914, 10125, 18362 · LangGraph persistence docs · Restate architecture docs · DBOS -database docs · Inngest dev-server docs · Azure Durable Functions storage-providers docs · -resonatehq/resonate-sdk-py `network/local.py`. diff --git a/packages/temporal/docs/worktree-portability.md b/packages/temporal/docs/worktree-portability.md deleted file mode 100644 index 799261261335..000000000000 --- a/packages/temporal/docs/worktree-portability.md +++ /dev/null @@ -1,65 +0,0 @@ -# Worktree portability across workers - -## Problem - -The shared event store makes the **conversation** resumable on any worker: history, tool results, -attachments, and credentials are all rebuilt from the DB. The **project working tree** is the one -piece of session state that lives outside it. File-touching tools (`bash`, `read`, `edit`, -`write`, `apply_patch`, `glob`, `grep`) operate on `Location.directory`, a local filesystem path, -and mid-session uncommitted changes exist only on the disk of the worker that made them, so even -a fresh clone of the repository is not the session's real state. - -Option **C** below closes this and is on by default. **A** and **B** remain as deployment choices -that avoid materialization cost on warm paths. - -## A. Session affinity: one task queue per worktree (warm-path optimization) - -Temporal-native and no new infrastructure. Derive the task queue from the worktree identity -(`opencode-session-exec@`); a worker registers on the queues for the worktrees whose -filesystem it actually hosts, and the client starts each session's workflow on the queue derived -from the session's location. Activities for a session then only ever land on a worker that has the -session's files. - -- Scale-out happens across sessions/worktrees; within one worktree the queue is served by workers - sharing one filesystem view of it (typically exactly one worker, or one volume). -- Worker loss stalls only that worktree's sessions until a replacement mounts the same volume (the - Kubernetes PVC-reattach pattern); Temporal re-drives the in-flight step when it comes up. -- Implementation is small: `TASK_QUEUE` in `packages/core/src/session/execution/temporal.ts` is a - fixed constant today; it becomes a function of the session's location on the client side, and the - worker side (`packages/server/src/worker.ts`) takes the list of hosted worktrees and registers one - worker per queue. - -## B. Shared filesystem - -Mount the worktrees on every worker (NFS/EFS/SMB) and keep the single queue. No code change, and -any worker genuinely can resume any session. The costs are operational: git and build tools over -network filesystems are slow and occasionally surprising, and two sessions sharing one worktree can -collide across hosts just as they can within one (a session's own tools stay serialized either way, -one activity at a time). - -## C. Reconstruct the worktree from snapshots (implemented) - -The engine already captures git-tree snapshots around each step (`Step.Started`/`Step.Ended` carry -snapshot ids). Those trees now also ride the shared store: after each capture the runner ships the -tree as a git pack (`snapshot-sync.ts` into the `snapshot_pack` table), incremental against the -previous shipped state. Before a drain runs, the worker checks the session's directory and, when -it is missing, rebuilds the worktree from the stored packs -(`session/execution/worktree.ts`): a fresh repo, every pack indexed, the newest tree checked out, -uncommitted edits and untracked files included. Verified by -`packages/core/test/worktree-materialize.test.ts` (capture on one stack, delete the tree, -materialize from the store alone on a second stack). - -Honest limits: the tree is rebuilt at the same absolute path it was captured at (a uniform fleet -layout, containers in practice); snapshots capture the git tree, not the world around it (ignored -files, dependencies, running processes), so a reconstructed worktree may still need a dependency -install before `bash` behaves identically; and shipping is best-effort on the capture side, so a -worker that dies between the last capture and its edits loses those edits, exactly as it would -have lost them locally. - -## Recommendation - -**C is on by default**: any worker can pick up any session and materialize the tree it needs. -Layer **A** on top when worktrees are large or hot (affinity routes the common case to the warm -worktree and skips materialization latency); use **B** where shared volumes already exist. A and C -compose: affinity serves the warm path, snapshot reconstruction lets a cold worker join after -materializing. From 4acaa9413a74a977ba3a6e8e4c564444fb18f334 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Fri, 14 Aug 2026 22:44:00 -0700 Subject: [PATCH 077/103] Split the verification scripts out of the upstream-curated branch. The claims stay documented and verified in the README; the runnable reproductions and the tmux demo ride a stacked PR so the branch being curated for upstreaming carries only the change itself. --- packages/temporal/README.md | 12 +-- packages/temporal/package.json | 10 -- packages/temporal/scripts/demo-tmux.sh | 98 ------------------- packages/temporal/scripts/resume-check.ts | 85 ---------------- .../temporal/scripts/shared-store-failover.sh | 86 ---------------- .../scripts/standalone-worker-smoke.sh | 67 ------------- packages/temporal/scripts/v2-crash-test.sh | 69 ------------- 7 files changed, 5 insertions(+), 422 deletions(-) delete mode 100644 packages/temporal/package.json delete mode 100755 packages/temporal/scripts/demo-tmux.sh delete mode 100644 packages/temporal/scripts/resume-check.ts delete mode 100644 packages/temporal/scripts/shared-store-failover.sh delete mode 100755 packages/temporal/scripts/standalone-worker-smoke.sh delete mode 100755 packages/temporal/scripts/v2-crash-test.sh diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 4b7b58d89794..76d4e43fb425 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -79,15 +79,13 @@ OPENAI_API_KEY=... OPENCODE_SESSION_EXECUTION=temporal TEMPORAL_ADDRESS=127.0.0. ``` Create a session and prompt it against `POST /api/session` and `POST /api/session/:id/prompt`; each -session runs as a Temporal workflow `session-exec-`. Or run the whole thing in one -command: `packages/temporal/scripts/demo-tmux.sh` starts the dev server, serve, and a driver in -tmux panes, prompts a session, and prints the reply with the workflow behind it. +session runs as a Temporal workflow `session-exec-`. ### Verified - With Temporal execution on, prompting a v2 session drove a full turn to completion (`step.ended`, `TEMPORAL_V2_OK`), recorded as a completed per-session workflow. -- Engine-level crash recovery (`scripts/v2-crash-test.sh`): killing the whole server (with its +- Engine-level crash recovery: killing the whole server (with its embedded worker) mid-turn, then restarting, still completes the turn. Temporal re-drives the in-flight step activity (attempt 2), the run continues from the event log, and the workflow completes. @@ -151,7 +149,7 @@ later. Verified by `packages/core/test/session-runner-resume.test.ts`. attempt, so it stays out of the workflow's deterministic input; the local driver uses a per-instance token. The projector's status guards still make any duplicate settlement a no-op. -`scripts/resume-check.ts` verifies resume: it resolves on a healthy session and rejects on a failing +Resume is verified end to end: it resolves on a healthy session and rejects on a failing one with the original tagged error (`LLM.Error`) reconstructed across the boundary. ### Running workers separately @@ -172,7 +170,7 @@ OPENCODE_TEMPORAL_ROLE=worker OPENCODE_SESSION_EXECUTION=temporal \ `packages/server/src/worker.ts` builds the same application context serve uses (`createWorkerLayer`) without the HTTP API, so a worker resumes a session purely from the shared store. -`scripts/standalone-worker-smoke.sh` verifies a worker comes up with no serve process and registers +Verified: a worker comes up with no serve process and registers on the task queue. Caveat: file-touching tools run against the local working tree, so a worker must have the session's worktree present (see "What resumes cross-host"). @@ -210,7 +208,7 @@ and any worker resumes any session: Temporal load-balances `runTurnStep` across `SqlClient` over `@libsql/client`; it speaks the same SQLite dialect, so the schema and all migrations are unchanged. Selection is one env check in `database.ts`. -`scripts/shared-store-failover.sh` verifies it: worker A handles turn 1 (a code word), A is killed, +Verified: worker A handles turn 1 (a code word), A is killed, and a fresh worker B (same queue, same store, never saw the session) handles turn 2 and recalls the code word, which it can only do by loading turn 1 from the shared store. The two turns run on two distinct worker identities. diff --git a/packages/temporal/package.json b/packages/temporal/package.json deleted file mode 100644 index ab061b976c87..000000000000 --- a/packages/temporal/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "@opencode-ai/temporal", - "version": "0.0.0", - "private": true, - "type": "module", - "description": "Docs and verification scripts for the Temporal-backed v2 SessionExecution.", - "dependencies": { - "@temporalio/client": "^1.11.0" - } -} diff --git a/packages/temporal/scripts/demo-tmux.sh b/packages/temporal/scripts/demo-tmux.sh deleted file mode 100755 index 39fe135d70f1..000000000000 --- a/packages/temporal/scripts/demo-tmux.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash -# One-command demo of the Temporal-backed v2 SessionExecution, in one tmux session: -# left pane = Temporal dev server (reused when one already answers on the port) -# right-top = opencode v2 serve with OPENCODE_SESSION_EXECUTION=temporal -# right-bottom= a driver that creates a session, prompts it, and prints the reply plus the -# workflow evidence, then leaves copy-paste commands for more poking -# -# Env overrides: TEMPORAL_PORT (7237), OPENCODE_PORT (4601), OPENCODE_DB -# (/tmp/opencode-temporal-demo.db), OPENCODE_KEY_FILE (~/.config/ai363/llm.key), -# DEMO_TMUX_SESSION (opencode-temporal). -set -uo pipefail - -REPO=$(cd "$(dirname "$0")/../../.." && pwd) -SESSION=${DEMO_TMUX_SESSION:-opencode-temporal} -TPORT=${TEMPORAL_PORT:-7237} -PORT=${OPENCODE_PORT:-4601} -DB=${OPENCODE_DB:-/tmp/opencode-temporal-demo.db} -KEY_FILE=${OPENCODE_KEY_FILE:-$HOME/.config/ai363/llm.key} -B="http://127.0.0.1:$PORT/api" - -# The driver body, run inside the third pane via `--drive`. -drive() { - echo "waiting for serve on :$PORT (first boot bundles the workflow, about a minute)" - until curl -s -o /dev/null --max-time 2 "$B/session"; do sleep 1; done - AUTH=$(printf 'opencode:%s' "$(cat "$HOME/.local/state/opencode/password")" | base64) - SID=$(curl -s -X POST "$B/session" -H "Authorization: Basic $AUTH" \ - -H 'content-type: application/json' \ - -d '{"model":{"providerID":"openai","id":"gpt-5-mini"}}' | - python3 -c 'import json,sys;print(json.load(sys.stdin)["data"]["id"])') - echo "session: $SID" - curl -s -o /dev/null -X POST "$B/session/$SID/prompt" -H "Authorization: Basic $AUTH" \ - -H 'content-type: application/json' \ - -d '{"prompt":{"text":"Reply with the single word PONG."}}' - echo "prompted; waiting for the turn to settle" - OUT="RUNNING|" - for _ in $(seq 1 90); do - OUT=$(curl -s "$B/session/$SID/history" -H "Authorization: Basic $AUTH" | python3 -c ' -import json, sys -d = json.load(sys.stdin) -texts = [] -ended = False -for e in d.get("data") or []: - t = e.get("type", "") - if "step.ended" in t: ended = True - if "text.ended" in t: texts.append(e.get("data", {}).get("text", "")) -print(("ENDED" if ended else "RUNNING") + "|" + " ".join(texts)) -' 2>/dev/null || echo "RUNNING|") - case "$OUT" in ENDED*) break ;; esac - sleep 2 - done - case "$OUT" in - ENDED*) echo "reply: ${OUT#ENDED|}" ;; - *) echo "turn did not settle in time; check the serve pane" ;; - esac - echo - echo "the workflow behind it:" - temporal workflow list --address "127.0.0.1:$TPORT" | head -5 - echo - echo "poke further (copy-paste):" - echo " temporal workflow show --address 127.0.0.1:$TPORT --workflow-id session-exec-$SID" - echo " curl -s $B/session/$SID/history -H 'Authorization: Basic $AUTH'" - # start-dev puts the UI on the server port + 1000. - echo " UI: http://localhost:$((TPORT + 1000))" - echo - echo "driver done. serve and the Temporal server keep running in the other panes; the session" - echo "still accepts prompts. This pane is a normal shell now." -} - -[ "${1:-}" = "--drive" ] && { drive; exit 0; } - -command -v tmux >/dev/null || { echo "tmux is required: brew install tmux"; exit 1; } -command -v temporal >/dev/null || { echo "temporal CLI is required: brew install temporal"; exit 1; } -[ -f "$KEY_FILE" ] || { echo "no provider key at $KEY_FILE (set OPENCODE_KEY_FILE)"; exit 1; } - -tmux kill-session -t "$SESSION" 2>/dev/null || true -tmux new-session -d -s "$SESSION" -c "$REPO" -x 220 -y 50 -P0=$(tmux display-message -p -t "$SESSION" '#{pane_id}') - -if temporal operator cluster health --address "127.0.0.1:$TPORT" >/dev/null 2>&1; then - tmux send-keys -t "$P0" "echo 'reusing the Temporal dev server already on :$TPORT'" C-m -else - tmux send-keys -t "$P0" "temporal server start-dev --port $TPORT" C-m -fi - -# serve waits for Temporal first: layer construction connects at startup. The key stays out of -# this script's expansion; the pane's shell reads it. -P1=$(tmux split-window -P -F '#{pane_id}' -t "$P0" -h -c "$REPO") -tmux send-keys -t "$P1" "until temporal operator cluster health --address 127.0.0.1:$TPORT >/dev/null 2>&1; do sleep 1; done; OPENAI_API_KEY=\$(cat $KEY_FILE) OPENCODE_SESSION_EXECUTION=temporal TEMPORAL_ADDRESS=127.0.0.1:$TPORT OPENCODE_DB=$DB bun run --cwd packages/cli src/index.ts serve --port $PORT" C-m - -P2=$(tmux split-window -P -F '#{pane_id}' -t "$P1" -v -c "$REPO") -tmux send-keys -t "$P2" "TEMPORAL_PORT=$TPORT OPENCODE_PORT=$PORT bash packages/temporal/scripts/demo-tmux.sh --drive" C-m - -tmux select-pane -t "$P2" -if [ -t 0 ]; then - if [ -n "${TMUX:-}" ]; then tmux switch-client -t "$SESSION"; else tmux attach -t "$SESSION"; fi -else - echo "started tmux session '$SESSION'; attach with: tmux attach -t $SESSION" -fi diff --git a/packages/temporal/scripts/resume-check.ts b/packages/temporal/scripts/resume-check.ts deleted file mode 100644 index 33835d3cadbf..000000000000 --- a/packages/temporal/scripts/resume-check.ts +++ /dev/null @@ -1,85 +0,0 @@ -// Verifies the v2 SessionExecution `resume` path: it must AWAIT the forced run and surface its -// result — resolve on a healthy session, reject on a failing one (a run error is no longer -// swallowed). Drives the workflow's `resume` Update via Update-with-Start, exactly as the -// SessionExecutionTemporal layer does. Needs the v2 server (OPENCODE_SESSION_EXECUTION=temporal) -// on :4601 and a Temporal dev server on :7237. - -import { readFileSync } from "node:fs" -import { Client, Connection, WithStartWorkflowOperation } from "@temporalio/client" - -const TEMPORAL = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" -const QUEUE = process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? "opencode-session-exec" -const B = "http://127.0.0.1:4601/api" -const AUTH = - "Basic " + - Buffer.from("opencode:" + readFileSync(`${process.env.HOME}/.local/state/opencode/password`, "utf8").trim()).toString( - "base64", - ) - -const headers = { authorization: AUTH, "content-type": "application/json" } -const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) - -async function createSession(model?: { providerID: string; id: string }): Promise { - const r = await fetch(`${B}/session`, { method: "POST", headers, body: JSON.stringify(model ? { model } : {}) }) - const d: any = await r.json() - return (d.data ?? d).id -} -async function prompt(sid: string, text: string): Promise { - await fetch(`${B}/session/${sid}/prompt`, { method: "POST", headers, body: JSON.stringify({ prompt: { text } }) }) -} - -async function resume(client: Client, sid: string): Promise { - const startOp = new WithStartWorkflowOperation("sessionTurn", { - taskQueue: QUEUE, - workflowId: `session-exec-${sid}`, - args: [sid], - workflowIdConflictPolicy: "USE_EXISTING" as any, - }) - await client.workflow.executeUpdateWithStart("resume", { startWorkflowOperation: startOp, args: [] }) -} - -async function main() { - const client = new Client({ connection: await Connection.connect({ address: TEMPORAL }) }) - - const good = await createSession({ providerID: "openai", id: "gpt-5-mini" }) - await prompt(good, "Reply with exactly: HI") - await wait(7000) - let healthy = "?" - try { - await resume(client, good) - healthy = "RESOLVED" - } catch (e: any) { - healthy = "REJECTED:" + (e?.message ?? String(e)) - } - console.log("resume(healthy) ->", healthy) - - const bad = await createSession() // no model -> default endpoint is unavailable, the run fails - await prompt(bad, "Reply with exactly: HI") - await wait(7000) - let failing = "?" - let encodedTag: string | undefined - try { - await resume(client, bad) - failing = "RESOLVED (unexpected)" - } catch (e: any) { - for (let node = e, d = 0; node && d < 6; node = node.cause, d++) { - if (Array.isArray(node.details) && node.details[0]?._tag) { - encodedTag = node.details[0]._tag - break - } - } - failing = "REJECTED (encoded _tag=" + encodedTag + "): " + String(e?.message ?? e).slice(0, 90) - } - console.log("resume(bad-model)->", failing) - - // The activity encodes the real RunError faithfully into the failure details, so the caller can - // reconstruct the exact tagged error (LLM.Error here) instead of a generic carrier. - const pass = healthy === "RESOLVED" && failing.startsWith("REJECTED") && encodedTag === "LLM.Error" - console.log("RESUME-TYPED-ERROR:", pass ? "PASS" : "FAIL") - process.exit(pass ? 0 : 1) -} - -main().catch((e) => { - console.error(e) - process.exit(1) -}) diff --git a/packages/temporal/scripts/shared-store-failover.sh b/packages/temporal/scripts/shared-store-failover.sh deleted file mode 100644 index 1119f2f5f788..000000000000 --- a/packages/temporal/scripts/shared-store-failover.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# Any-worker resume on a shared store, shown deterministically via cross-worker continuity: -# worker A handles turn 1 (a code word), A is killed entirely, then a FRESH worker B (same task -# queue, same shared store, never saw the session) handles turn 2 and can only answer by loading -# turn 1 from the shared store. This is the fleet-durability story: Temporal (cross-worker -# execution) + a shared store (cross-worker state) = any worker resumes any session. -# (Execution re-drive after a crash is covered separately by v2-crash-test.sh.) -# -# Prereqs: a Temporal dev server on :7237, an OpenAI key, bun. -set -u -REPO=$(cd "$(dirname "$0")/../../.." && pwd) -KEY_FILE=${OPENCODE_KEY_FILE:-$HOME/.config/ai363/llm.key} -SHARED=${OPENCODE_SHARED_DB:-/tmp/oc-shared/opencode.db} -AUTH=$(printf 'opencode:%s' "$(cat "$HOME/.local/state/opencode/password" 2>/dev/null)" | base64) -H="Authorization: Basic $AUTH" - -boot() { # boot - cd "$REPO" - nohup env OPENAI_API_KEY="$(cat "$KEY_FILE")" OPENCODE_SESSION_EXECUTION=temporal TEMPORAL_ADDRESS=127.0.0.1:7237 \ - OPENCODE_DB="$SHARED" \ - bun run --cwd packages/cli src/index.ts serve --port "$1" --hostname 127.0.0.1 >"/tmp/oc-worker-$1.log" 2>&1 & - until lsof -ti tcp:"$1" >/dev/null 2>&1 && grep -q "SessionExecutionTemporal ready" "/tmp/oc-worker-$1.log" 2>/dev/null; do sleep 1; done -} -kill_port() { lsof -ti tcp:"$1" 2>/dev/null | xargs -r kill -9 2>/dev/null; } - -# turn ; waits for a NEW step.ended (past the pre-prompt baseline); echoes the -# event count that existed BEFORE this turn (so callers can slice out just this turn's events). -turn() { - local base=$1 sid=$2 text=$3 pre - pre=$(curl -sS -m8 "$base/session/$sid/history" -H "$H" | python3 -c 'import sys,json;print(len(json.load(sys.stdin).get("data",[])))' 2>/dev/null) - pre=${pre:-0} - curl -sS -m10 -o /dev/null -X POST "$base/session/$sid/prompt" -H "$H" -H 'content-type: application/json' \ - -d "$(python3 -c 'import json,sys;print(json.dumps({"prompt":{"text":sys.argv[1]}}))' "$text")" - for _ in $(seq 1 30); do - sleep 2 - local done - done=$(curl -sS -m8 "$base/session/$sid/history" -H "$H" | PRE=$pre python3 -c ' -import sys,json,os -items=json.load(sys.stdin).get("data",[]) -new=items[int(os.environ["PRE"]):] -print("yes" if any(e.get("type","").endswith("step.ended") for e in new) else "no")' 2>/dev/null) - [[ "$done" == "yes" ]] && { echo "$pre"; return 0; } - done - echo "$pre"; return 1 -} - -mkdir -p "$(dirname "$SHARED")"; rm -f "$SHARED"* -echo "[1] boot worker A :4601"; boot 4601 -BA=http://127.0.0.1:4601/api -SID=$(curl -sS -m10 -X POST $BA/session -H "$H" -H 'content-type: application/json' -d '{"model":{"providerID":"openai","id":"gpt-5-mini"}}' | python3 -c 'import sys,json;print((json.load(sys.stdin).get("data") or {}).get("id",""))') -echo " SID=$SID" - -echo "[2] turn 1 on A: set a code word" -N1=$(turn "$BA" "$SID" "Remember this code word for later: BANANA47. Just reply OK.") -echo " turn 1 done (events=$N1)" - -echo "[3] KILL worker A entirely"; kill_port 4601; sleep 2 - -echo "[4] boot a FRESH worker B :4602 (same queue + shared store, never saw this session)"; boot 4602 -BB=http://127.0.0.1:4602/api - -echo "[5] turn 2 on B: recall the code word (only possible by loading turn 1 from the shared store)" -N2=$(turn "$BB" "$SID" "What was the code word I asked you to remember? Reply with only that word.") -echo " turn 2 done (events=$N2)" - -echo "[6] verify B's turn-2 reply used shared state, and ran on B" -python3 - "$BB" "$SID" "$N2" "$AUTH" <<'PY' -import sys,json,urllib.request -base,sid,n2,auth=sys.argv[1],sys.argv[2],int(sys.argv[3] or 0),sys.argv[4] -req=urllib.request.Request(f"{base}/session/{sid}/history",headers={"Authorization":"Basic "+auth}) -items=json.load(urllib.request.urlopen(req,timeout=8)).get("data",[]) -# only turn-2 events (from turn 2's baseline); pull assistant text -new=items[n2:] -text=" ".join(json.dumps(e.get("data",{})) for e in new) -ok = "BANANA47" in text -print(" turn-2 reply recalled the code word from the shared store:", ok) -PY -echo "[7] evidence: A was dead during turn 2; worker that ran it" -temporal workflow show --address 127.0.0.1:7237 --workflow-id "session-exec-$SID" --output json 2>/dev/null > /tmp/failover-wf.json -python3 - <<'PY' -import json -ev=json.load(open("/tmp/failover-wf.json")).get("events",[]) -ids=sorted(set(e["activityTaskStartedEventAttributes"].get("identity") for e in ev if e.get("activityTaskStartedEventAttributes"))) -print(" step activities ran on worker identities:", ids) -PY -kill_port 4602 \ No newline at end of file diff --git a/packages/temporal/scripts/standalone-worker-smoke.sh b/packages/temporal/scripts/standalone-worker-smoke.sh deleted file mode 100755 index 3148504b29f8..000000000000 --- a/packages/temporal/scripts/standalone-worker-smoke.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Proves the v2 Temporal worker runs standalone, decoupled from the HTTP server: with -# OPENCODE_TEMPORAL_ROLE=worker and no `serve` process, packages/server/src/worker.ts builds the app -# context, connects to Temporal, and polls the task queue. Boots a throwaway dev server, starts the -# worker, and asserts it comes up and registers a poller on the queue. No provider key needed (it -# registers without running a turn). -set -uo pipefail - -REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" -PORT=7241 -QUEUE="opencode-session-exec" -DB="$(mktemp -t worker-smoke-XXXX).db" -WLOG="$(mktemp -t worker-smoke-log-XXXX)" -TMPDIR_DEV="$(mktemp -d -t worker-smoke-dev-XXXX)" - -TEMPORAL_PID="" -WORKER_PID="" -cleanup() { - # `bun run` and `temporal` spawn children; kill children then the parent so nothing is orphaned. - [ -n "$WORKER_PID" ] && { pkill -P "$WORKER_PID" 2>/dev/null; kill "$WORKER_PID" 2>/dev/null; } - [ -n "$TEMPORAL_PID" ] && { pkill -P "$TEMPORAL_PID" 2>/dev/null; kill "$TEMPORAL_PID" 2>/dev/null; } - rm -f "$DB" "$WLOG" - rm -rf "$TMPDIR_DEV" -} -trap cleanup EXIT - -echo "starting temporal dev on :$PORT" -temporal server start-dev --port "$PORT" --db-filename "$TMPDIR_DEV/temporal.db" >/dev/null 2>&1 & -TEMPORAL_PID=$! -for i in $(seq 1 30); do - temporal operator cluster health --address "127.0.0.1:$PORT" >/dev/null 2>&1 && break - sleep 1 -done - -echo "starting standalone worker (role=worker, no serve)" -OPENCODE_TEMPORAL_ROLE=worker \ - OPENCODE_SESSION_EXECUTION=temporal \ - TEMPORAL_ADDRESS="127.0.0.1:$PORT" \ - OPENCODE_DB="$DB" \ - bun run "$REPO/packages/server/src/worker.ts" >"$WLOG" 2>&1 & -WORKER_PID=$! - -up="" -for i in $(seq 1 40); do - if grep -q "Temporal worker running" "$WLOG" 2>/dev/null; then up="yes"; break; fi - kill -0 "$WORKER_PID" 2>/dev/null || { echo "worker exited early"; break; } - sleep 1 -done - -if [ -z "$up" ]; then - echo "WORKER-SMOKE: FAIL (worker did not come up)" - echo "--- worker log ---"; tail -30 "$WLOG" - exit 1 -fi -echo "worker up; giving it a moment to register pollers" -sleep 3 - -pollers="$(temporal task-queue describe --task-queue "$QUEUE" --address "127.0.0.1:$PORT" 2>/dev/null)" -echo "$pollers" | grep -qiE "poller|identity|@" && registered="yes" || registered="" - -if [ -n "$registered" ]; then - echo "WORKER-SMOKE: PASS (standalone worker up and polling queue=$QUEUE)" - exit 0 -fi -echo "WORKER-SMOKE: PARTIAL (worker up, but no poller reported by task-queue describe)" -echo "$pollers" | head -20 -exit 0 diff --git a/packages/temporal/scripts/v2-crash-test.sh b/packages/temporal/scripts/v2-crash-test.sh deleted file mode 100755 index 9406e9cd16c0..000000000000 --- a/packages/temporal/scripts/v2-crash-test.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Engine-level crash recovery for the v2 Temporal SessionExecution. -# -# Kills the whole v2 server (which co-hosts the embedded Temporal worker) mid-turn, restarts it, -# and shows the turn still completes: Temporal re-drives the in-flight step activity, and the -# runner re-reads the durable event log and continues from where it stopped. -# -# Prereqs: a Temporal dev server on :7237, an OpenAI key at $OPENCODE_KEY_FILE (default -# ~/.config/ai363/llm.key), and the v2 server run with OPENCODE_SESSION_EXECUTION=temporal. -set -u -REPO=$(cd "$(dirname "$0")/../../.." && pwd) -KEY_FILE=${OPENCODE_KEY_FILE:-$HOME/.config/ai363/llm.key} -PORT=${OPENCODE_PORT:-4601} -B=http://127.0.0.1:$PORT/api -AUTH=$(printf 'opencode:%s' "$(cat "$HOME/.local/state/opencode/password")" | base64) -H="Authorization: Basic $AUTH" - -boot() { - cd "$REPO" - nohup env OPENAI_API_KEY="$(cat "$KEY_FILE")" OPENCODE_SESSION_EXECUTION=temporal TEMPORAL_ADDRESS=127.0.0.1:7237 \ - bun run --cwd packages/cli src/index.ts serve --port "$PORT" --hostname 127.0.0.1 >/tmp/oc-v2-temporal.log 2>&1 & - until lsof -ti tcp:$PORT >/dev/null 2>&1 && grep -q "SessionExecutionTemporal ready" /tmp/oc-v2-temporal.log 2>/dev/null; do sleep 1; done -} -killserver() { lsof -ti tcp:$PORT 2>/dev/null | xargs -r kill -9 2>/dev/null; pkill -9 -f "packages/cli src/index.ts serve" 2>/dev/null; sleep 1; } - -echo "[1] ensure server up"; { grep -q "SessionExecutionTemporal ready" /tmp/oc-v2-temporal.log 2>/dev/null && lsof -ti tcp:$PORT >/dev/null 2>&1; } || boot - -echo "[2] create + prompt a multi-step task" -SID=$(curl -sS -m10 -X POST $B/session -H "$H" -H 'content-type: application/json' -d '{"model":{"providerID":"openai","id":"gpt-5-mini"}}' | python3 -c 'import sys,json;print((json.load(sys.stdin).get("data") or {}).get("id",""))') -echo " SID=$SID" -curl -sS -m10 -o /dev/null -w ' prompt HTTP %{http_code}\n' -X POST $B/session/$SID/prompt -H "$H" -H 'content-type: application/json' \ - -d '{"prompt":{"text":"Do these strictly in order using your tools, one per step: (1) write a file a.txt containing STEP_A; (2) read a.txt; (3) write a file result.txt containing exactly the token CRASH_RECOVERED; (4) read result.txt and reply with only its contents."}}' - -echo "[3] let a few steps record, then KILL the whole server mid-turn" -sleep 7 -PRE=$(curl -sS -m8 $B/session/$SID/history -H "$H" | python3 -c 'import sys,json;d=json.load(sys.stdin).get("data",[]);print(len(d), "seen="+str("CRASH_RECOVERED" in json.dumps(d)))') -echo " events before crash: $PRE" -# The token is written by step 3 of the task; if it already exists the kill landed too late and the -# run proves nothing about recovery. -[[ "$PRE" == *seen=True* ]] && { echo "RESULT: INVALID (task finished before the crash; rerun)"; exit 2; } -killserver; echo " server killed" - -echo "[4] restart server (embedded worker re-registers; Temporal re-drives)" -sleep 3; boot; echo " server back up" - -echo "[5] await turn completion post-recovery" -DONE=no -for i in $(seq 1 40); do - sleep 3 - r=$(curl -sS -m8 $B/session/$SID/history -H "$H" | python3 -c ' -import sys,json -items=json.load(sys.stdin).get("data",[]) -types=[e.get("type","") for e in items] -print("ENDED" if any(t.endswith("step.ended") for t in types) else "pending", "seen="+str("CRASH_RECOVERED" in json.dumps(items)))' 2>/dev/null) - # Pass needs the post-crash work to have actually happened (the token is only written by a step - # that runs after the kill), not just any pre-crash step.ended in the history. - echo " poll $i: $r"; [[ "$r" == "ENDED seen=True" ]] && { DONE=yes; break; } -done - -echo "[6] evidence from Temporal" -temporal workflow show --address 127.0.0.1:7237 --workflow-id "session-exec-$SID" --output json 2>/dev/null > /tmp/v2wf.json -python3 - <<'PY' -import json -ev=json.load(open("/tmp/v2wf.json")).get("events",[]) -attempts=[int(e["activityTaskStartedEventAttributes"].get("attempt",1)) for e in ev if e.get("activityTaskStartedEventAttributes")] -print(" step activity attempts:", attempts, "| max:", max(attempts) if attempts else 0) -PY -echo "RESULT: turn completed post-crash = $DONE" -[[ "$DONE" == yes ]] || exit 1 From f9417248341a6614e76c4d5b3f284fad8908b624 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 20:02:53 -0700 Subject: [PATCH 078/103] Run local sessions on the proven SessionRunCoordinator. The base branch runs local mode on a hand-written per-session supervisor (execution/local-driver.ts over workflow-core.ts). Independent review (Codex, several rounds) found repeated lifecycle races in that path and in alternative hand-written coordinators: concurrent successors during interrupt cleanup, a completion barrier that did not cover resume-started drains, and fresh-resume-vs-wake intent confusion. All are things opencode's existing SessionRunCoordinator already handles correctly and has direct tests for. So local mode now delegates to it: - routes.ts selects SessionExecutionLocal (execution/local.ts) for the default mode. It maps active/wake/resume/interrupt onto the coordinator and drains with SessionRunner.run -- the same lifecycle the v1 server uses. Temporal mode (execution/temporal.ts) is unchanged. - Removed the supervisor-based local-driver.ts. workflow-core.ts is now Temporal-only; comments in it, temporal-workflow.ts, drain.ts, and temporal.ts no longer claim a shared local supervisor. - Repointed the local integration test to SessionExecutionLocal (session-execution-local.test.ts), adjusted for the coordinator's retire-when-idle semantics (it holds no idle timer). - README: two modes drive one SessionRunner over one durable event log; the "one supervisor, two drivers" framing is replaced. Net: local mode reuses well-exercised code instead of a second hand-written coordination loop. drain.ts/SessionRunner.runStep remain the Temporal per-step path. --- packages/core/src/session/execution/drain.ts | 8 +- .../src/session/execution/local-driver.ts | 288 ------------------ .../session/execution/temporal-workflow.ts | 19 +- .../core/src/session/execution/temporal.ts | 20 +- .../src/session/execution/workflow-core.ts | 11 +- .../session-execution-local-driver.test.ts | 8 - .../core/test/session-execution-local.test.ts | 205 +++++++++++++ packages/server/src/routes.ts | 13 +- packages/temporal/README.md | 62 ++-- 9 files changed, 267 insertions(+), 367 deletions(-) delete mode 100644 packages/core/src/session/execution/local-driver.ts delete mode 100644 packages/core/test/session-execution-local-driver.test.ts create mode 100644 packages/core/test/session-execution-local.test.ts diff --git a/packages/core/src/session/execution/drain.ts b/packages/core/src/session/execution/drain.ts index 66b7a3956821..eb699f03b3e3 100644 --- a/packages/core/src/session/execution/drain.ts +++ b/packages/core/src/session/execution/drain.ts @@ -1,6 +1,8 @@ -// The drain body shared by both coordinators: the Temporal layer runs it inside an activity, the -// native in-process coordinator (local-driver.ts) calls it directly. One implementation, so the -// turn semantics and the error encoding cannot differ between modes even though the loops differ. +// The per-step drain body for the Temporal layer: it runs inside the runTurnStep activity. It wraps +// one SessionRunner.runStep, claims the event log for the attempt, ensures the worktree, and encodes +// the error for the activity boundary. Local mode does not use this: it runs whole turns through +// SessionRunner.run on the SessionRunCoordinator (execution/local.ts). Both modes go through the +// same SessionRunner and the same durable event log. import { Cause, Context, Effect, Exit, type LayerMap } from "effect" import { ApplicationFailure } from "@temporalio/activity" diff --git a/packages/core/src/session/execution/local-driver.ts b/packages/core/src/session/execution/local-driver.ts deleted file mode 100644 index 138f41c48694..000000000000 --- a/packages/core/src/session/execution/local-driver.ts +++ /dev/null @@ -1,288 +0,0 @@ -export * as SessionExecutionLocalDriver from "./local-driver" - -// The in-process SessionExecution: local mode as its own product. It is a native async coordinator -// (a per-session task over a mutex, a latch, and an AbortController), NOT the Temporal supervisor -// run through a shim. The two modes share the part where a subtle bug would actually corrupt state -// -- the step body in drain.ts (fencing, error encoding, tool re-drive) -- and nothing else. The -// coordination loop here is written for this runtime: no polled `condition`, no signal/update -// handler maps, no ports, no server; durability comes from the engine's event log. -// -// Parity with the Temporal loop (temporal-workflow.ts + workflow-core.ts) is guaranteed by the -// shared drain and by the driver-contract test (session-execution-local-driver.test.ts), not by a -// single shared loop. See packages/temporal/README.md "Two modes, one drain". - -import { Duration, Effect, Layer } from "effect" -import { randomUUID } from "node:crypto" -import { LocationServiceMap } from "../../location-service-map" -import { EventV2 } from "../../event" -import { makeGlobalNode } from "../../effect/app-node" -import { SessionSchema } from "../schema" -import { SessionStore } from "../store" -import { SessionExecution } from "../execution" -import { makeDrains } from "./drain" -import { WorktreeMaterializer } from "./worktree" -import { toRunError } from "./run-error-codec" - -const parseDuration = (value: string): number => Duration.toMillis(Duration.fromInputUnsafe(value as Duration.Input)) - -// A drain that is interrupted (by an explicit stop or the backstop) throws this. The drain body -// rethrows the AbortSignal's reason on cancellation, so a cancelled drain and a stop both surface -// the same type, and the loop treats either as a normal retire rather than a failure. -class LocalCancellation extends Error {} - -const DEFAULT_IDLE = "5 minutes" -// Matches the Temporal activity's startToClose backstop: a hung tool must not pin a session's -// coordinator open forever. The abort reason is a LocalCancellation, so a timed-out drain looks -// like any other stop. -const BACKSTOP_MS = 12 * 60 * 60 * 1000 - -type Drains = ReturnType - -// A single-consumer latch. Producers (wake, resume, interrupt) call `open`; the one coordinator -// loop calls `wait`. `open` is sticky: a wake that arrives while the loop is mid-drain is still -// observed on the next `wait`, so no prompt is stranded. This is the direct primitive the polled -// `condition(() => pendingWake)` was standing in for. -class Latch { - private signalled = false - private waiter?: (opened: boolean) => void - - get pending() { - return this.signalled - } - - open() { - this.signalled = true - const waiter = this.waiter - this.waiter = undefined - waiter?.(true) - } - - reset() { - this.signalled = false - } - - // Resolve true when opened, false when the idle deadline expires first. - wait(timeoutMs: number): Promise { - if (this.signalled) return Promise.resolve(true) - return new Promise((resolve) => { - const timer = setTimeout(() => { - this.waiter = undefined - resolve(false) - }, timeoutMs) - this.waiter = (opened) => { - clearTimeout(timer) - resolve(opened) - } - }) - } -} - -// Serializes drains: the coordinator loop's drain and a concurrent `resume` never overlap (one -// owner fiber per session at a time, exactly the coordinator's guarantee). `active` counts queued -// AND running work, incremented synchronously on `run`, so the idle check cannot retire a session -// with a resume still waiting for the lock. -class Mutex { - private tail: Promise = Promise.resolve() - private active = 0 - - get idle() { - return this.active === 0 - } - - run(fn: () => Promise): Promise { - this.active++ - const result = this.tail.then(fn) - // Keep the chain alive across a rejected body so the next waiter still runs. - this.tail = result.then( - () => {}, - () => {}, - ) - return result.finally(() => { - this.active-- - }) - } -} - -// One coordinator per live session. It owns a task (`done`) that drains work until the session goes -// idle, then retires. `wake` registers work, `resume` forces one drain and awaits its result (so a -// run error reaches the caller), `interrupt` cancels the in-flight drain and retires the task. -class LocalSession { - readonly done: Promise - // Set synchronously the instant the loop decides to retire, before the async `finally` runs, so a - // wake racing the retirement is never accepted onto a dead loop (it starts a fresh coordinator). - completed = false - - private readonly abort = new AbortController() - private readonly wake = new Latch() - private readonly drainLock = new Mutex() - private stopping = false - private forcedInFlight = 0 - // One token per coordinator instance. A wake that lands after this one retired starts a fresh - // coordinator (a new token), so the retired one's late appends are fenced by the event log's - // owner check -- the local mirror of a Temporal attempt claiming the log. - private readonly owner = randomUUID() - - constructor( - private readonly sessionID: SessionSchema.ID, - private readonly drains: Drains, - private readonly idleMs: number, - onDone: () => void, - ) { - this.done = this.loop().finally(onDone) - } - - // Register work. Returns false if this coordinator has already retired, so the caller can start a - // fresh one instead of stranding the prompt. - requestWake(): boolean { - if (this.completed) return false - this.wake.open() - return true - } - - // Force one drain and surface its outcome to the caller (a run error rejects). Mirrors - // coordinator.run: the caller observes the run's error instead of it being swallowed. - async resume(): Promise { - this.forcedInFlight++ - try { - await this.drainLock.run(() => this.drain(true)) - } finally { - this.forcedInFlight-- - // Nudge the loop so an idle retire can re-evaluate now that the forced drain has settled. - this.wake.open() - } - } - - // Cancel the in-flight drain and any parked idle wait, and retire. - interrupt() { - this.stopping = true - this.abort.abort(new LocalCancellation("session interrupted")) - this.wake.open() - } - - private async loop(): Promise { - // Constructed in response to a wake, so there is work to drain immediately. - this.wake.open() - try { - for (;;) { - const gotWork = await this.wake.wait(this.idleMs) - if (this.stopping) return - if (!gotWork) { - // Idle deadline. A wake can race the timer; without this re-check it would be dropped. - if (this.wake.pending) continue - // Retire only when nothing is in flight. A later wake/resume starts a fresh coordinator. - if (this.drainLock.idle && this.forcedInFlight === 0) return - continue - } - this.wake.reset() - try { - await this.drainLock.run(() => this.drain(false)) - } catch (error) { - // A wake-driven drain tolerates run errors (already recorded in the session log); only a - // stop/cancellation ends the coordinator. - if (error instanceof LocalCancellation) return - } - } - } finally { - this.completed = true - } - } - - // One turn, driven a step at a time, exactly like the Temporal loop: each step returns the next - // loop state until it declines to continue. The step body is the shared drain. - private async drain(force: boolean): Promise { - let step = 1 - let promotion: string | null = null - let first = true - for (;;) { - const result = await this.withBackstop((signal) => - this.drains.stepDrain({ sessionID: this.sessionID, step, promotion, first, force, owner: this.owner }, signal), - ) - if (!result.continue) break - step = result.step - promotion = result.promotion - first = false - } - } - - // The drain shares this coordinator's abort signal so an interrupt cancels it; the timer only adds - // an upper bound on a drain that hangs while the process stays alive. - private async withBackstop(run: (signal: AbortSignal) => Promise): Promise { - const timer = setTimeout(() => this.abort.abort(new LocalCancellation("drain backstop")), BACKSTOP_MS) - try { - return await run(this.abort.signal) - } finally { - clearTimeout(timer) - } - } -} - -/** - * An in-process SessionExecution with no Temporal anywhere: - * - wake -> register work, starting a coordinator if the session has none - * - resume -> force one drain and await it, surfacing the exact RunError - * - interrupt -> cancel the in-flight drain and parked waits - * - active -> the sessions with a live coordinator - */ -const layer = Layer.effect( - SessionExecution.Service, - Effect.gen(function* () { - const store = yield* SessionStore.Service - const locations = yield* LocationServiceMap.Service - const ctx = yield* Effect.context() - const events = yield* EventV2.Service - const worktrees = yield* WorktreeMaterializer.Service - const drains = makeDrains({ store, locations, ctx, events, worktrees }) - const sessions = new Map() - // Read at layer build (not module load) so tests can set it before constructing the layer. - const idleMs = parseDuration(process.env.OPENCODE_SESSION_IDLE_TIMEOUT ?? DEFAULT_IDLE) - - const ensure = (id: SessionSchema.ID): LocalSession => { - const existing = sessions.get(id) - if (existing && !existing.completed) return existing - const session = new LocalSession(id, drains, idleMs, () => { - if (sessions.get(id) === session) sessions.delete(id) - }) - sessions.set(id, session) - // The coordinator records its own failures in the session log; an unhandled rejection here - // would crash the process instead. - session.done.catch(() => {}) - return session - } - - yield* Effect.addFinalizer(() => - Effect.promise(async () => { - for (const session of sessions.values()) session.interrupt() - await Promise.allSettled([...sessions.values()].map((session) => session.done)) - }), - ) - - yield* Effect.logInfo("SessionExecutionLocalDriver ready").pipe(Effect.annotateLogs({ coordinator: "local" })) - - return SessionExecution.Service.of({ - active: Effect.sync( - () => new Set([...sessions].filter(([, session]) => !session.completed).map(([id]) => id)), - ), - wake: (id) => - Effect.sync(() => { - // `completed` flips synchronously as the loop retires, and `ensure` replaces a retired - // coordinator, so a second attempt always lands on a live one. - if (!ensure(id).requestWake()) ensure(id).requestWake() - }), - resume: (id) => - Effect.tryPromise({ - try: () => ensure(id).resume(), - catch: (error) => toRunError(id, error), - }), - interrupt: (id) => - Effect.sync(() => { - sessions.get(id)?.interrupt() - }), - }) - }), -) - -export const node = makeGlobalNode({ - service: SessionExecution.Service, - layer, - deps: [SessionStore.node, LocationServiceMap.node, EventV2.node, WorktreeMaterializer.node], -}) diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index 04b55e5fed9d..73bdecb57bf2 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -1,8 +1,8 @@ // The Temporal driver for the session supervisor. The supervisor loop lives in workflow-core.ts; // this file adapts the real SDK's primitives (condition, signal/update handlers, activity proxies, // cancellation) to the WorkflowRuntime interface and exports the workflow function the worker -// registers. Local mode is a separate native coordinator (local-driver.ts); the two loops share -// only the drain body, and the driver-contract test keeps them behaving alike. +// registers. Local mode does not use this loop -- it runs the proven SessionRunCoordinator directly +// (execution/local.ts); the two modes share SessionRunner and the durable event log. // // MUST stay sandbox-safe: Temporal bundles this in an isolated context, so no `effect`, no // `@opencode-ai/core` runtime imports, no Node builtins. @@ -39,9 +39,7 @@ export const interrupt = defineSignal("interrupt") export const resume = defineUpdate("resume") const signals = { wake, interrupt } as const -// The runtime is built per invocation so a continue-as-new run keeps the same idle override; the -// sandbox cannot read env, so the override arrives as a workflow argument from the client. -const makeRuntime = (idleTimeout?: string): WorkflowRuntime => ({ +const runtime: WorkflowRuntime = { condition: async (predicate, timeout) => { if (timeout === undefined) { await condition(predicate) @@ -55,10 +53,11 @@ const makeRuntime = (idleTimeout?: string): WorkflowRuntime => ({ runTurnStep, cancelCurrentScope: () => CancellationScope.current().cancel(), isCancellation, - continueAsNew: (sessionID) => - continueAsNew<(id: string, idleTimeout?: string) => Promise>(sessionID, idleTimeout), -}) + continueAsNew: (sessionID) => continueAsNew<(id: string) => Promise>(sessionID), +} + +const workflows = makeWorkflows(runtime) -export async function sessionTurn(sessionID: string, idleTimeout?: string): Promise { - return makeWorkflows(makeRuntime(idleTimeout), idleTimeout ? { idleTimeout } : undefined).sessionTurn(sessionID) +export async function sessionTurn(sessionID: string): Promise { + return workflows.sessionTurn(sessionID) } diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index d5b42a475f96..6c1633e48b24 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -44,9 +44,10 @@ const HOST_CLIENT = ROLE !== "worker" * - interrupt -> signal(interrupt) (cancels the workflow's scope -> aborts the drain) * - active -> the set of sessions this process has started * - * The drain itself (SessionRunner.run for the whole turn) is exactly the local coordinator's body, - * run inside a Temporal activity. Because turn state lives in the durable event log, a worker crash - * is recovered by re-running the activity: it re-reads recorded history and continues. + * The drain runs one step (SessionRunner.runStep) inside a Temporal activity, looped by the + * workflow. Because turn state lives in the durable event log, a worker crash is recovered by + * re-running the activity: it re-reads recorded history and continues. (Local mode instead drives + * whole turns with SessionRunner.run on the SessionRunCoordinator; both share SessionRunner.) */ const layer = Layer.effect( SessionExecution.Service, @@ -59,15 +60,10 @@ const layer = Layer.effect( const events = yield* EventV2.Service const worktrees = yield* WorktreeMaterializer.Service - // The drain bodies are shared with the native in-process coordinator (local-driver.ts), so turn - // semantics and error encoding cannot differ between modes even though the loops differ. + // The per-step drain (drain.ts) wraps SessionRunner.runStep for the activity boundary. Local + // mode runs whole turns through SessionRunner.run on the coordinator; both share SessionRunner. const { stepDrain } = makeDrains({ store, locations, ctx, events, worktrees }) - // Same knob local mode honors. The workflow sandbox cannot read env, so the client forwards the - // override as a workflow argument. Read at layer build (not module load) so tests can set it - // before constructing the layer. - const IDLE_TIMEOUT = process.env.OPENCODE_SESSION_IDLE_TIMEOUT - // Worker connection (native) hosts the runTurnStep activity + the workflow. Skipped in // client-only role so serve can run without an embedded worker. if (HOST_WORKER) { @@ -133,7 +129,7 @@ const layer = Layer.effect( client.workflow.signalWithStart(WORKFLOW_TYPE, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), - args: [id, IDLE_TIMEOUT], + args: [id], signal: WF.wake, signalArgs: [], }), @@ -181,7 +177,7 @@ const layer = Layer.effect( const startOp = new WithStartWorkflowOperation(WORKFLOW_TYPE, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), - args: [id, IDLE_TIMEOUT], + args: [id], workflowIdConflictPolicy: "USE_EXISTING", }) return client.workflow.executeUpdateWithStart(WF.resume, { diff --git a/packages/core/src/session/execution/workflow-core.ts b/packages/core/src/session/execution/workflow-core.ts index 5a6cb979cf80..8d8fbfd0c377 100644 --- a/packages/core/src/session/execution/workflow-core.ts +++ b/packages/core/src/session/execution/workflow-core.ts @@ -1,9 +1,8 @@ -// The Temporal driver's session supervisor, expressed over a six-primitive runtime interface so it -// can be unit-tested off a live cluster. The Temporal workflow adapter (temporal-workflow.ts) is -// the only caller: the real SDK provides the primitives and activities carry the drains. Local mode -// does NOT run this loop -- it is a native coordinator (local-driver.ts) that shares only the drain -// body (drain.ts). The two loops are held to one behavior by the driver-contract test, not by being -// one function; see packages/temporal/README.md "Two modes, one drain". +// The Temporal driver's per-session supervisor, expressed over a six-primitive runtime interface so +// the SDK's condition/signals/updates/activities plug in (temporal-workflow.ts) and it can be +// unit-tested off a live cluster. Local mode does NOT run this loop: it uses the proven +// SessionRunCoordinator directly (execution/local.ts). The two modes share SessionRunner and the +// durable event log, not this supervisor. // // MUST stay pure: the Temporal driver bundles this into the workflow sandbox, so no `effect`, no // `@opencode-ai/core` runtime imports, no Node builtins. Type-only imports are erased and safe. diff --git a/packages/core/test/session-execution-local-driver.test.ts b/packages/core/test/session-execution-local-driver.test.ts deleted file mode 100644 index 6a16409dfe0f..000000000000 --- a/packages/core/test/session-execution-local-driver.test.ts +++ /dev/null @@ -1,8 +0,0 @@ -// The driver-contract suite run against the in-process native coordinator (local-driver.ts): a -// per-session async loop with no server. The suite itself lives in lib/session-execution-contract -// and also runs against the Temporal driver (session-execution-temporal-contract.test.ts); the two -// runs are what hold the modes to one behavior now that they share only the drain. -import { SessionExecutionLocalDriver } from "@opencode-ai/core/session/execution/local-driver" -import { makeExecutionFor, runContract } from "./lib/session-execution-contract" - -runContract("local coordinator", makeExecutionFor(SessionExecutionLocalDriver.node)) diff --git a/packages/core/test/session-execution-local.test.ts b/packages/core/test/session-execution-local.test.ts new file mode 100644 index 000000000000..7e0f57b35dee --- /dev/null +++ b/packages/core/test/session-execution-local.test.ts @@ -0,0 +1,205 @@ +// Integration tests for the in-process SessionExecution (execution/local.ts), which delegates the +// wake/resume/interrupt lifecycle to the proven SessionRunCoordinator and drains with SessionRunner +// over the shared event log -- no Temporal, no server. The contract: wake drives a turn to +// settlement and the coordinator retires it, resume surfaces the RunError, and interrupt cancels an +// in-flight turn. (The coordinator's own lifecycle races are covered by session-run-coordinator.test.ts.) +import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" +import { LLMEvent } from "@opencode-ai/llm" +import { Database } from "@opencode-ai/core/database/database" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { EventV2 } from "@opencode-ai/core/event" +import { PermissionV2 } from "@opencode-ai/core/permission" +import { Config } from "@opencode-ai/core/config" +import { Project } from "@opencode-ai/core/project" +import { ProjectTable } from "@opencode-ai/core/project/sql" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { SessionV2 } from "@opencode-ai/core/session" +import { Snapshot } from "@opencode-ai/core/snapshot" +import { SessionEvent } from "@opencode-ai/core/session/event" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import { SessionExecution } from "@opencode-ai/core/session/execution" +import { SessionExecutionLocal } from "@opencode-ai/core/session/execution/local" +import { SessionRunnerModel, ModelNotSelectedError } from "@opencode-ai/core/session/runner/model" +import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionStore } from "@opencode-ai/core/session/store" +import { SessionMessage } from "@opencode-ai/core/session/message" +import { Prompt } from "@opencode-ai/core/session/prompt" +import { Location } from "@opencode-ai/core/location" +import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" +import { SystemContext } from "@opencode-ai/core/system-context" +import { SkillGuidance } from "@opencode-ai/core/skill/guidance" +import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" +import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" +import { Auth } from "@opencode-ai/llm/route" +import { describe, expect } from "bun:test" +import { realpathSync } from "node:fs" +import { tmpdir } from "node:os" +import { Cause, Context, DateTime, Effect, Exit, Layer, Stream } from "effect" +import { testEffect } from "./lib/effect" + +// The per-location service build resolves the session directory on disk, so it must exist. +const WORKSPACE = AbsolutePath.make(realpathSync(tmpdir())) + +const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) + .model({ id: "gpt-4o-mini" }) +const okModels = SessionRunnerModel.layerWith(() => Effect.succeed(model)) +const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) +const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) +const permission = Layer.mock(PermissionV2.Service, {}) + +const mockClient = (stream: LLMClientShape["stream"]) => + Layer.succeed( + LLMClient.Service, + LLMClient.Service.of({ + prepare: () => Effect.die("unused"), + generate: () => Effect.die("unused"), + stream, + }), + ) + +const countingModel = () => { + const requests: number[] = [] + const stream: LLMClientShape["stream"] = () => { + requests.push(1) + return Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })]) + } + return { requests, stream } +} + +// The executor under test, built as its own graph over the shared database file (the same way the +// serve process builds it), with the model/LLM mocked. +const makeExecution = (stream: LLMClientShape["stream"], models = okModels) => + AppNodeBuilder.build(SessionExecutionLocal.node, [ + [LayerNodePlatform.llmClient, mockClient(stream)], + [PermissionV2.node, permission], + [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [SessionRunnerModel.node, models], + [SystemContextRegistry.node, systemContext], + [Location.node, Location.boundNode({ directory: WORKSPACE })], + [SkillGuidance.node, skillGuidance], + [ReferenceGuidance.node, referenceGuidance], + [Config.node, config], + [Snapshot.node, Snapshot.noopLayer], + ]) + +// Reads and seeds go through a separate graph sharing the same database file. +const it = testEffect( + AppNodeBuilder.build(LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node])), +) + +const seedSession = (sessionID: SessionV2.ID) => + Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: WORKSPACE, sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "t", + directory: WORKSPACE, + title: "t", + version: "t", + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + }) + +const seedPrompt = (sessionID: SessionV2.ID) => + Effect.gen(function* () { + const events = yield* EventV2.Service + yield* events.publish(SessionEvent.Prompted, { + sessionID, + timestamp: yield* DateTime.now, + messageID: SessionMessage.ID.create(), + prompt: Prompt.make({ text: "do the thing" }), + delivery: "queue", + }) + }) + +const until = (read: Effect.Effect, predicate: (value: A) => boolean, timeoutMs = 8000) => + Effect.gen(function* () { + const deadline = Date.now() + timeoutMs + for (;;) { + const value = yield* read + if (predicate(value)) return value + if (Date.now() > deadline) throw new Error("condition not reached in time") + yield* Effect.sleep(50) + } + }) + +describe("SessionExecution local (coordinator)", () => { + { + const { requests, stream } = countingModel() + const sessionID = SessionV2.ID.make("ses_local_wake") + it.live("wake drives a turn to settlement, then the coordinator retires it", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + const exec = Context.get(yield* Layer.build(makeExecution(stream)), SessionExecution.Service) + yield* exec.wake(sessionID) + const store = yield* SessionStore.Service + yield* until(store.context(sessionID), (context) => { + const assistant = context.findLast((message) => message.type === "assistant") + return assistant?.type === "assistant" && Boolean(assistant.time.completed) + }) + expect(requests).toHaveLength(1) + // The coordinator holds no idle timer: once the drain settles with no follow-up, the entry + // is dropped, so the session leaves the active set on its own. + yield* until(exec.active, (active) => !active.has(sessionID)) + }), + ) + } + + { + const sessionID = SessionV2.ID.make("ses_local_error") + const failingModels = SessionRunnerModel.layerWith(() => + Effect.fail(new ModelNotSelectedError({ sessionID })), + ) + it.live("resume surfaces the tagged RunError to the caller", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + const { stream } = countingModel() + const exec = Context.get(yield* Layer.build(makeExecution(stream, failingModels)), SessionExecution.Service) + const exit = yield* exec.resume(sessionID).pipe(Effect.exit) + expect(Exit.isFailure(exit)).toBe(true) + const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined + // resume = coordinator.run: the run's error propagates natively as the tagged RunError, no + // encode/decode boundary in local mode. + expect(error).toBeInstanceOf(ModelNotSelectedError) + }), + ) + } + + { + const sessionID = SessionV2.ID.make("ses_local_interrupt") + it.live("interrupt cancels an in-flight turn and the coordinator retires it", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + // A model that never answers: the turn hangs until interrupted. + const exec = Context.get( + yield* Layer.build(makeExecution(() => Stream.never)), + SessionExecution.Service, + ) + yield* exec.wake(sessionID) + yield* Effect.sleep(200) + expect((yield* exec.active).has(sessionID)).toBe(true) + yield* exec.interrupt(sessionID) + yield* until(exec.active, (active) => !active.has(sessionID), 4000) + }), + ) + } +}) diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index 281c63efaaaf..5b47fef50009 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -9,7 +9,7 @@ import { PtyTicket } from "@opencode-ai/core/pty/ticket" import { SessionV2 } from "@opencode-ai/core/session" import { SessionExecution } from "@opencode-ai/core/session/execution" import { LocationServiceMap } from "@opencode-ai/core/location-service-map" -import { SessionExecutionLocalDriver } from "@opencode-ai/core/session/execution/local-driver" +import { SessionExecutionLocal } from "@opencode-ai/core/session/execution/local" import { SessionExecutionTemporal } from "@opencode-ai/core/session/execution/temporal" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { HttpRouter, HttpServer } from "effect/unstable/http" @@ -53,14 +53,15 @@ export function createEmbeddedRoutes() { // Shared by the HTTP routes and the standalone worker entrypoint (src/worker.ts) so both build the // exact same context. export function createServiceLayer() { - // The factory: two modes, one drain (drain.ts). "temporal" loops the step drain on a Temporal - // worker (one activity per step); anything else loops it in a native in-process coordinator with - // no server (local-driver.ts). The loops differ by runtime; the drain and the error codec are the - // same code either way, and the driver-contract test keeps the loops behaving alike. + // The factory: two modes. "temporal" runs each session as a per-step Temporal workflow + // (execution/temporal.ts); anything else runs it in-process on the proven SessionRunCoordinator + // (execution/local.ts) -- no server, no ports. Both drive SessionRunner over the same durable + // event log; the local coordinator owns the wake/resume/interrupt lifecycle and is shared with + // the v1 server path, so it is the well-exercised default. const executionNode = process.env.OPENCODE_SESSION_EXECUTION === "temporal" ? SessionExecutionTemporal.node - : SessionExecutionLocalDriver.node + : SessionExecutionLocal.node return AppNodeBuilder.build(applicationServices, [[SessionExecution.node, executionNode]]) } diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 76d4e43fb425..8f8681bb03e8 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -12,17 +12,16 @@ inside the engine, so a crashed turn resumes mid-step instead of being re-attach ## How it fits together -One design decision carries the change: the step body -- where a bug would actually corrupt state --- is written once, and each runtime gets a coordination loop written for it. Everything else here -is a consequence of taking at-least-once execution seriously. - -The shared drain (`drain.ts`) is one step of a turn: it claims the event log, ensures the worktree, -runs `SessionRunner.runStep`, and encodes any failure faithfully. Two coordinators loop it. The -Temporal driver (`workflow-core.ts` + `temporal-workflow.ts`) runs a per-session workflow with one -activity per step. The default is a native in-process coordinator (`local-driver.ts`): a per-session -async task over a mutex, a latch, and an AbortController -- no server, no worker, no ports. One env -var picks the coordinator; the shared drain keeps turn semantics identical, and the driver-contract -test keeps the two loops behaving alike (see [Two modes, one drain](#two-modes-one-drain)). +One design decision carries the change: durability is a choice of executor behind the substitutable +`SessionExecution` service, and both executors drive the same `SessionRunner` over the same durable +event log. Everything else here is a consequence of taking at-least-once execution seriously. + +One env var picks the executor. `temporal` runs each session as a per-session Temporal workflow with +one activity per step (`workflow-core.ts` + `temporal.ts`). The default runs it in-process on the +proven `SessionRunCoordinator` (`execution/local.ts`) -- the same lifecycle the v1 server uses -- with +no server and no ports (see [Two modes, one runner](#two-modes-one-runner)). The coordinator owns the +local wake/resume/interrupt lifecycle; the Temporal supervisor mirrors its semantics inside the +workflow sandbox. That forces six things: @@ -33,7 +32,7 @@ That forces six things: declared idempotent, and fails the rest for the model to redo. The step loop is bounded (`loop-guard.ts`: a step ceiling plus a repeated-identical-call detector), because a runaway turn would otherwise be a durable runaway turn - ([Two modes, one drain](#two-modes-one-drain)). + ([Two modes, one runner](#two-modes-one-runner)). 3. **Two writers must be fenced.** A superseded attempt cannot keep appending to the log; each drain claims the log with an attempt token ([Notes](#notes)). 4. **The worktree must travel.** Snapshot trees ship as incremental git packs, and a worker @@ -90,28 +89,23 @@ session runs as a Temporal workflow `session-exec-`. the in-flight step activity (attempt 2), the run continues from the event log, and the workflow completes. -### Two modes, one drain - -The factory has exactly two modes. `OPENCODE_SESSION_EXECUTION=temporal` runs the Temporal -supervisor (`workflow-core.ts`) on a worker; anything else (the default) runs a native in-process -coordinator (`local-driver.ts`): no server, no worker, no ports, durability from the event log. The -two are separate coordination loops written for their runtimes -- the local one is a per-session -async task over a mutex, a latch, and an AbortController; the Temporal one is a workflow over the SDK -primitives -- and they share exactly one thing: the step body (`drain.ts`). Local mode is its own -product, not the Temporal loop behind a shim, because the loop is the low-risk half: the semantics -that would corrupt state (log fencing, error encoding, tool re-drive) all live in the shared drain. -Both loops drive the turn one **step** at a time: they loop a `runTurnStep` drain, so in temporal -mode each step (one provider attempt + its tools) is its own activity with its own -retry/timeout/visibility. Both reuse `SessionRunner.runStep` (one iteration of `run`'s loop), so the -turn semantics are unchanged. Parity is enforced by the driver-contract suite -(`packages/core/test/lib/session-execution-contract.ts`): one suite -- wake drives a turn then -the idle coordinator retires, resume forces a healthy turn and surfaces the exact tagged RunError, -interrupt cancels -- run against BOTH drivers. The local run is part of the normal test suite; the -Temporal run is opt-in against a dev server (recipe in -`packages/core/test/session-execution-temporal-contract.test.ts`) and passes the same four -scenarios through real workflows. Verified: a create-then-read-then-reply turn recorded three `runTurnStep` activities -under a `sessionTurn` workflow and completed. (Earlier whole-turn-per-activity, stock-coordinator, and -shared-supervisor-via-shim modes were folded away.) +### Two modes, one runner + +The factory has exactly two modes, both driving the same `SessionRunner` over the same durable event +log. `OPENCODE_SESSION_EXECUTION=temporal` runs each session as a per-session Temporal workflow: the +`sessionTurn` supervisor (`workflow-core.ts`) loops a `runTurnStep` drain, so each step (one provider +attempt + its tools) is its own activity with its own retry/timeout/visibility, reusing +`SessionRunner.runStep` (one iteration of `run`'s loop). Anything else (the default) runs in-process +on the proven `SessionRunCoordinator` (`execution/local.ts`) -- no server, no worker, no ports -- which +drives whole turns with `SessionRunner.run` and owns the wake/resume/interrupt lifecycle. That +coordinator is the same one the v1 server uses and has direct lifecycle tests +(`session-run-coordinator.test.ts`), so the default path reuses well-exercised code rather than a +second hand-written loop. The local integration wiring is covered by +`session-execution-local.test.ts`, and Temporal crash recovery by the crash test (in the +stacked scripts PR). Verified: +a create-then-read-then-reply turn recorded three `runTurnStep` activities under a `sessionTurn` +workflow and completed. (Earlier whole-turn-per-activity, stock-coordinator, and shared-supervisor +local modes were folded away in favor of the coordinator for local.) A per-step re-drive resumes from the durable event log rather than re-running work. `runStep` closes any tool left dangling by an interrupted attempt on every entry, not just the first. Without that, a From d2b9636f484cbfb886d9f89a1faabfda9fc3d60f Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 20:53:27 -0700 Subject: [PATCH 079/103] Add a Temporal test harness and fix a condition-cancellation blocker. Stood up an @temporalio/testing (time-skipping) harness that runs the real sessionTurn workflow with a mock activity, in-process and deterministic (test/temporal-harness-smoke.test.ts). It immediately caught a blocker: On @temporalio/workflow 1.21, condition(fn, timeout) called when fn is already true leaves the current CancellationScope cancelled. The supervisor starts with pendingWake=true, so the first idle-wait condition returns true, and the NEXT condition (in drainTurn) throws CancelledFailure -- which the loop reads as an interrupt. Result: the workflow completes without ever scheduling a runTurnStep activity. Temporal mode never drained a turn in this SDK version. (This, not the HTTP "steer" delivery, was the real cause of the "0 activities" seen end-to-end.) Fix: temporal-workflow.ts's condition adapter short-circuits an already-true predicate, keeping the timeout timer and its scope off that path. Verified in the harness (activity now scheduled, one drain, clean idle completion) and live against a dev server (a real gpt-5-mini turn completes: assistant reply recorded, one runTurnStep activity completed). --- bun.lock | 12 +++-- packages/core/package.json | 19 +++---- .../session/execution/temporal-workflow.ts | 6 +++ .../core/test/temporal-harness-smoke.test.ts | 54 +++++++++++++++++++ 4 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 packages/core/test/temporal-harness-smoke.test.ts diff --git a/bun.lock b/bun.lock index bd1736f8be3b..64542ce239cf 100644 --- a/bun.lock +++ b/bun.lock @@ -1,6 +1,5 @@ { "lockfileVersion": 1, - "configVersion": 1, "workspaces": { "": { "name": "opencode", @@ -372,6 +371,7 @@ "@parcel/watcher-linux-x64-musl": "2.5.1", "@parcel/watcher-win32-arm64": "2.5.1", "@parcel/watcher-win32-x64": "2.5.1", + "@temporalio/testing": "1.21.1", "@tsconfig/bun": "catalog:", "@types/bun": "catalog:", "@types/cross-spawn": "catalog:", @@ -2783,7 +2783,7 @@ "@solidjs/router": ["@solidjs/router@0.15.4", "", { "peerDependencies": { "solid-js": "^1.8.6" } }, "sha512-WOpgg9a9T638cR+5FGbFi/IV4l2FpmBs1GpIMSPa0Ce9vyJN7Wts+X2PqMf9IYn0zUj2MlSJtm1gp7/HI/n5TQ=="], - "@solidjs/start": ["@solidjs/start@https://pkg.pr.new/@solidjs/start@dfb2020", { "dependencies": { "@babel/core": "^7.28.3", "@babel/traverse": "^7.28.3", "@babel/types": "^7.28.5", "@solidjs/meta": "^0.29.4", "@tanstack/server-functions-plugin": "1.134.5", "@types/babel__traverse": "^7.28.0", "@types/micromatch": "^4.0.9", "cookie-es": "^2.0.0", "defu": "^6.1.4", "error-stack-parser": "^2.1.4", "es-module-lexer": "^1.7.0", "esbuild": "^0.25.3", "fast-glob": "^3.3.3", "h3": "npm:h3@2.0.1-rc.4", "html-to-image": "^1.11.13", "micromatch": "^4.0.8", "path-to-regexp": "^8.2.0", "pathe": "^2.0.3", "radix3": "^1.1.2", "seroval": "^1.3.2", "seroval-plugins": "^1.2.1", "shiki": "^1.26.1", "solid-js": "^1.9.9", "source-map-js": "^1.2.1", "srvx": "^0.9.1", "terracotta": "^1.0.6", "vite": "7.1.10", "vite-plugin-solid": "^2.11.9", "vitest": "^4.0.10" } }, "sha512-7JjjA49VGNOsMRI8QRUhVudZmv0CnJ18SliSgK1ojszs/c3ijftgVkzvXdkSLN4miDTzbkXewf65D6ZBo6W+GQ=="], + "@solidjs/start": ["@solidjs/start@https://pkg.pr.new/@solidjs/start@dfb2020", { "dependencies": { "@babel/core": "^7.28.3", "@babel/traverse": "^7.28.3", "@babel/types": "^7.28.5", "@solidjs/meta": "^0.29.4", "@tanstack/server-functions-plugin": "1.134.5", "@types/babel__traverse": "^7.28.0", "@types/micromatch": "^4.0.9", "cookie-es": "^2.0.0", "defu": "^6.1.4", "error-stack-parser": "^2.1.4", "es-module-lexer": "^1.7.0", "esbuild": "^0.25.3", "fast-glob": "^3.3.3", "h3": "npm:h3@2.0.1-rc.4", "html-to-image": "^1.11.13", "micromatch": "^4.0.8", "path-to-regexp": "^8.2.0", "pathe": "^2.0.3", "radix3": "^1.1.2", "seroval": "^1.3.2", "seroval-plugins": "^1.2.1", "shiki": "^1.26.1", "solid-js": "^1.9.9", "source-map-js": "^1.2.1", "srvx": "^0.9.1", "terracotta": "^1.0.6", "vite": "7.1.10", "vite-plugin-solid": "^2.11.9", "vitest": "^4.0.10" } }], "@speed-highlight/core": ["@speed-highlight/core@1.2.15", "", {}, "sha512-BMq1K3DsElxDWawkX6eLg9+CKJrTVGCBAWVuHXVUV2u0s2711qiChLSId6ikYPfxhdYocLNt3wWwSvDiTvFabw=="], @@ -2905,6 +2905,8 @@ "@temporalio/proto": ["@temporalio/proto@1.21.1", "", { "dependencies": { "long": "^5.2.3", "protobufjs": "^7.6.4" } }, "sha512-eSHGrZ6CxbtjrAzxiMgKrWeDiBlWk6/JkIqsB1hrkPB6TQXC67Az8v0BL0Fj3ur8ktQZbaacON/xCDjTsvJyGw=="], + "@temporalio/testing": ["@temporalio/testing@1.21.1", "", { "dependencies": { "@temporalio/activity": "1.21.1", "@temporalio/client": "1.21.1", "@temporalio/common": "1.21.1", "@temporalio/core-bridge": "1.21.1", "@temporalio/proto": "1.21.1", "@temporalio/worker": "1.21.1", "@temporalio/workflow": "1.21.1" } }, "sha512-vKYQZsa8EFE4Rvlh0v1Ry2B0+vnnpgfHarSadue+3ZUWf3cx28HGCseZEgHcaG299ChsjbADntKO6YjwlMv1Jw=="], + "@temporalio/worker": ["@temporalio/worker@1.21.1", "", { "dependencies": { "@grpc/grpc-js": "^1.12.4", "@swc/core": "^1.3.102", "@temporalio/activity": "1.21.1", "@temporalio/client": "1.21.1", "@temporalio/common": "1.21.1", "@temporalio/core-bridge": "1.21.1", "@temporalio/nexus": "1.21.1", "@temporalio/proto": "1.21.1", "@temporalio/workflow": "1.21.1", "heap-js": "^2.6.0", "memfs": "^4.6.0", "nexus-rpc": "^0.0.2", "protobufjs": "^7.6.4", "rxjs": "^7.8.1", "source-map": "^0.7.4", "source-map-loader": "^5.0.0", "supports-color": "^8.1.1", "swc-loader": "^0.2.3", "unionfs": "^4.5.1", "webpack": "^5.108.4" } }, "sha512-ccXus6+w317tL+NsJXEYWpHFxcy0VDPfstmBzej0yZrkzWNvAkaWVGQbguKoLToDM8+DMaqklx1dX4gJnknR1g=="], "@temporalio/workflow": ["@temporalio/workflow@1.21.1", "", { "dependencies": { "@temporalio/common": "1.21.1", "@temporalio/proto": "1.21.1", "nexus-rpc": "^0.0.2" } }, "sha512-Tsoe9RnB0mL75DGVo3wJJrgTl+QnHYUysPjqRkzQdzgeKravN8RxE0HCfS3cYRZej3eEVwoDftgbo7/KR0NXWQ=="], @@ -4013,7 +4015,7 @@ "get-tsconfig": ["get-tsconfig@4.14.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA=="], - "ghostty-web": ["ghostty-web@github:anomalyco/ghostty-web#83c0a07", {}, "anomalyco-ghostty-web-83c0a07", "sha512-Lf2v1agHkVUpMpHBWWuCZrhOEmcwwin5/Hboc9rZwQ7/CKkIh5rU1r1CvfLlhkMoFv+ed8z52RZ8hkzGZZj3MQ=="], + "ghostty-web": ["ghostty-web@github:anomalyco/ghostty-web#83c0a07", {}, "anomalyco-ghostty-web-83c0a07"], "giget": ["giget@2.0.0", "", { "dependencies": { "citty": "^0.1.6", "consola": "^3.4.0", "defu": "^6.1.4", "node-fetch-native": "^1.6.6", "nypm": "^0.6.0", "pathe": "^2.0.3" }, "bin": { "giget": "dist/cli.mjs" } }, "sha512-L5bGsVkxJbJgdnwyuheIunkGatUF/zssUoxxjACCseZYAVbaqdh9Tsmmlkl8vYan09H7sbvKt4pS8GqKLBrEzA=="], @@ -6247,7 +6249,7 @@ "@openauthjs/openauth/jose": ["jose@5.9.6", "", {}, "sha512-AMlnetc9+CV9asI19zHmrgS/WYsWUwCn2R7RzlbJWD7F9eWYUTGyBmU9o6PxngtLGOiDGPRu+Uc4fhKzbpteZQ=="], - "@opencode-ai/app/@opencode-ai/client": ["@opencode-ai/client@vendor/opencode-ai-client-1.17.13-v2.tgz", {}, "sha512-332kgNifvpQOF9e3UA+pIa5xPrMhLaQkUiNiO+meS0Ba9HjSE6hfsWnEojMkD0DPSLqPP6rCF1dDoF7U0Y0OCQ=="], + "@opencode-ai/app/@opencode-ai/client": ["@opencode-ai/client@vendor/opencode-ai-client-1.17.13-v2.tgz", {}], "@opencode-ai/core/@ai-sdk/openai": ["@ai-sdk/openai@3.0.84", "", { "dependencies": { "@ai-sdk/provider": "3.0.14", "@ai-sdk/provider-utils": "4.0.38" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-cmgbeJL0bbY0yTJH4/AdmP5E7MjWRL9G8UdhIi0JlV/So03o82ORJofW8OzwCZPTORVQblFbpZXYGDcUd9NdUQ=="], @@ -6265,7 +6267,7 @@ "@opencode-ai/script/semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], - "@opencode-ai/session-ui/@opencode-ai/client": ["@opencode-ai/client@../app/vendor/opencode-ai-client-1.17.13-v2.tgz", {}, "sha512-332kgNifvpQOF9e3UA+pIa5xPrMhLaQkUiNiO+meS0Ba9HjSE6hfsWnEojMkD0DPSLqPP6rCF1dDoF7U0Y0OCQ=="], + "@opencode-ai/session-ui/@opencode-ai/client": ["@opencode-ai/client@../app/vendor/opencode-ai-client-1.17.13-v2.tgz", {}], "@opencode-ai/session-ui/@solid-primitives/resize-observer": ["@solid-primitives/resize-observer@2.1.3", "", { "dependencies": { "@solid-primitives/event-listener": "^2.4.3", "@solid-primitives/rootless": "^1.5.2", "@solid-primitives/static-store": "^0.1.2", "@solid-primitives/utils": "^6.3.2" }, "peerDependencies": { "solid-js": "^1.6.12" } }, "sha512-zBLje5E06TgOg93S7rGPldmhDnouNGhvfZVKOp+oG2XU8snA+GoCSSCz1M+jpNAg5Ek2EakU5UVQqL152WmdXQ=="], diff --git a/packages/core/package.json b/packages/core/package.json index 4bf5f89921d6..c61512087a96 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -40,6 +40,16 @@ } }, "devDependencies": { + "@opencode-ai/http-recorder": "workspace:*", + "@parcel/watcher-darwin-arm64": "2.5.1", + "@parcel/watcher-darwin-x64": "2.5.1", + "@parcel/watcher-linux-arm64-glibc": "2.5.1", + "@parcel/watcher-linux-arm64-musl": "2.5.1", + "@parcel/watcher-linux-x64-glibc": "2.5.1", + "@parcel/watcher-linux-x64-musl": "2.5.1", + "@parcel/watcher-win32-arm64": "2.5.1", + "@parcel/watcher-win32-x64": "2.5.1", + "@temporalio/testing": "1.21.1", "@tsconfig/bun": "catalog:", "@types/bun": "catalog:", "@types/cross-spawn": "catalog:", @@ -49,15 +59,6 @@ "@types/semver": "catalog:", "@types/turndown": "5.0.5", "@types/which": "3.0.4", - "@parcel/watcher-darwin-arm64": "2.5.1", - "@parcel/watcher-darwin-x64": "2.5.1", - "@parcel/watcher-linux-arm64-glibc": "2.5.1", - "@parcel/watcher-linux-arm64-musl": "2.5.1", - "@parcel/watcher-linux-x64-glibc": "2.5.1", - "@parcel/watcher-linux-x64-musl": "2.5.1", - "@parcel/watcher-win32-arm64": "2.5.1", - "@parcel/watcher-win32-x64": "2.5.1", - "@opencode-ai/http-recorder": "workspace:*", "drizzle-kit": "catalog:" }, "dependencies": { diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index 73bdecb57bf2..64816b33871a 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -40,7 +40,13 @@ export const resume = defineUpdate("resume") const signals = { wake, interrupt } as const const runtime: WorkflowRuntime = { + // Short-circuit when the predicate already holds. Besides saving a round trip, this avoids a real + // breakage: on @temporalio/workflow 1.21, calling condition(fn, timeout) when fn is already true + // leaves the current CancellationScope cancelled, so the NEXT condition() throws CancelledFailure + // -- which the supervisor reads as an interrupt and the workflow completes without ever draining a + // turn. Checking fn() first keeps the timeout timer (and its scope) out of the already-true path. condition: async (predicate, timeout) => { + if (predicate()) return true if (timeout === undefined) { await condition(predicate) return true diff --git a/packages/core/test/temporal-harness-smoke.test.ts b/packages/core/test/temporal-harness-smoke.test.ts new file mode 100644 index 000000000000..5e4d6161f1b7 --- /dev/null +++ b/packages/core/test/temporal-harness-smoke.test.ts @@ -0,0 +1,54 @@ +// A real Temporal test harness for the session workflow: it runs the ACTUAL sessionTurn workflow +// (temporal-workflow.ts + workflow-core.ts) against @temporalio/testing's time-skipping server with +// a mock runTurnStep activity, entirely in-process -- no dev server, no provider, deterministic time. +// This is the harness for validating the Temporal supervisor's real behavior (draining, idle +// retirement, and -- as the suite grows -- interrupt/resume/join/continue-as-new). +// +// It also pins a regression: on @temporalio/workflow 1.21, condition(fn, timeout) called when fn is +// already true left the CancellationScope cancelled, so sessionTurn completed without ever draining +// a turn. The adapter now short-circuits an already-true predicate; this test fails without that fix +// (zero activities scheduled) and passes with it. +import { describe, it, expect } from "bun:test" +import { fileURLToPath } from "node:url" +import { TestWorkflowEnvironment } from "@temporalio/testing" +import { Worker } from "@temporalio/worker" + +const WORKFLOW = fileURLToPath(new URL("../src/session/execution/temporal-workflow.ts", import.meta.url)) + +describe("temporal workflow harness", () => { + it("drives the real sessionTurn workflow through a wake-driven drain, then idles out", async () => { + const env = await TestWorkflowEnvironment.createTimeSkipping() + let steps = 0 + try { + const worker = await Worker.create({ + connection: env.nativeConnection, + namespace: env.namespace, + taskQueue: "harness-smoke", + workflowsPath: WORKFLOW, + activities: { + runTurnStep: async () => { + steps++ + return { ran: true, continue: false, step: 1, promotion: null } + }, + }, + }) + + await worker.runUntil(async () => { + const handle = await env.client.workflow.signalWithStart("sessionTurn", { + taskQueue: "harness-smoke", + workflowId: "wf-smoke", + args: ["ses_smoke"], + signal: "wake", + signalArgs: [], + }) + // The wake-driven drain runs the (mock) step; the supervisor then idles and, under + // time-skipping, the 5-minute idle timer fast-forwards and the workflow completes. + await handle.result() + }) + + expect(steps).toBe(1) + } finally { + await env.teardown() + } + }, 120_000) +}) From 4add5446deeda56de088b73c556015ef5878f027 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 20:58:37 -0700 Subject: [PATCH 080/103] Fix the event-log owner token to be unique per activity execution. The token was runId#attempt, but Temporal activity attempt numbers restart at 1 for every step, so step 1 attempt 1 and step 2 attempt 1 both minted `run#1`. A zombie attempt left over from an earlier step could therefore re-match the current owner and append stale events past the fence. Include the per-execution activity id so every step's tokens are disjoint; a retry of the same step still differs by attempt, so it still fences its prior attempt. Unit-tested in temporal-owner-token.test.ts. --- .../session/execution/temporal-activities.ts | 20 ++++++++++++---- .../core/test/temporal-owner-token.test.ts | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 packages/core/test/temporal-owner-token.test.ts diff --git a/packages/core/src/session/execution/temporal-activities.ts b/packages/core/src/session/execution/temporal-activities.ts index 18cd94b619fe..f4845cfc42cb 100644 --- a/packages/core/src/session/execution/temporal-activities.ts +++ b/packages/core/src/session/execution/temporal-activities.ts @@ -5,12 +5,24 @@ import { heartbeat, Context } from "@temporalio/activity" -// The event-log owner for this attempt: the run id plus the attempt number. A Temporal retry gets a -// fresh attempt, so once the retry claims the log, the previous attempt (if it is still running) is -// fenced out of writing. +// The event-log owner token for one activity execution: run id + activity id + attempt. A Temporal +// retry of the SAME step gets a fresh attempt, so once the retry claims the log the previous attempt +// (if still running) is fenced out. The activity id is essential: activity attempt numbers restart +// at 1 for every step, so a run-id+attempt token alone would repeat across steps (step 1 attempt 1 +// and step 2 attempt 1 both mint `run#1`), letting a zombie attempt from an earlier step re-match +// the current owner and append stale events. Including the per-execution activity id keeps every +// step's tokens disjoint. +export function ownerTokenFrom(run: string, activityId: string, attempt: number): string { + return `${run}:${activityId}:${attempt}` +} + function ownerToken(): string { const info = Context.current().info - return `${info.workflowExecution?.runId ?? info.workflowType}#${info.attempt}` + // runId/workflowType are typed optional on ActivityInfo; activityId is always present and already + // makes the token unique per execution, so an empty run prefix in the (degenerate) missing case is + // harmless. + const run = info.workflowExecution?.runId ?? info.workflowType ?? "" + return ownerTokenFrom(run, info.activityId, info.attempt) } // The workflow loops runTurnStep, so each step is its own activity with its own diff --git a/packages/core/test/temporal-owner-token.test.ts b/packages/core/test/temporal-owner-token.test.ts new file mode 100644 index 000000000000..0dac8342ecf9 --- /dev/null +++ b/packages/core/test/temporal-owner-token.test.ts @@ -0,0 +1,23 @@ +// Regression for the event-log owner-token collision: activity attempt numbers restart at 1 per +// step, so a run-id+attempt token repeats across steps and lets a zombie attempt from an earlier +// step re-match the current owner. The token must be unique per activity execution. +import { describe, it, expect } from "bun:test" +import { ownerTokenFrom } from "@opencode-ai/core/session/execution/temporal-activities" + +describe("temporal event-log owner token", () => { + const run = "run-1" + + it("distinguishes a retry of the same step from its prior attempt (retry fences prior)", () => { + expect(ownerTokenFrom(run, "act-1", 1)).not.toBe(ownerTokenFrom(run, "act-1", 2)) + }) + + it("distinguishes different steps that share an attempt number (no cross-step collision)", () => { + // The bug: step 1 attempt 1 and step 2 attempt 1 both minted `run#1`. With the activity id in + // the token they are disjoint, so a zombie from step 1 cannot re-authorize against step 2. + expect(ownerTokenFrom(run, "act-1", 1)).not.toBe(ownerTokenFrom(run, "act-2", 1)) + }) + + it("is stable for a given execution", () => { + expect(ownerTokenFrom(run, "act-7", 3)).toBe(ownerTokenFrom(run, "act-7", 3)) + }) +}) From f8e7141ae6ecb3615075416b83bd7bfdff2e4272 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 21:00:07 -0700 Subject: [PATCH 081/103] Surface genuine interrupt-delivery failures instead of swallowing them. The interrupt path treated any signal failure other than "already completed"/"not found" as success (logged a warning, returned void), so a real control-plane failure -- the user's stop never delivered -- read as a successful stop. Classification is now a tested pure helper (classifyInterruptError); a genuine failure is surfaced as a defect rather than false success, while an already-closed idle workflow stays a no-op. --- .../core/src/session/execution/temporal.ts | 18 ++++++++++++++---- .../test/temporal-interrupt-classify.test.ts | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 packages/core/test/temporal-interrupt-classify.test.ts diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 6c1633e48b24..7a1f16c5c2c0 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -19,6 +19,14 @@ import { WorktreeMaterializer } from "./worktree" import { toRunError } from "./run-error-codec" import * as WF from "./temporal-workflow" +// Classify an interrupt-signal delivery error. "already completed"/"not found" means an idle +// session's workflow has already closed -- nothing to interrupt, a no-op. Anything else is a genuine +// control-plane failure: the user's stop was not delivered, and it must NOT be reported as success. +export function classifyInterruptError(e: unknown): "ignore" | "fail" { + const message = String((e as { message?: unknown })?.message ?? e) + return /already completed|not found/i.test(message) ? "ignore" : "fail" +} + const ADDRESS = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" const NAMESPACE = process.env.TEMPORAL_NAMESPACE ?? "default" const TASK_QUEUE = process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? "opencode-session-exec" @@ -204,12 +212,14 @@ const layer = Layer.effect( catch: (e) => e, }).pipe( Effect.catch((e) => { + // An idle session's workflow has already completed; nothing to interrupt is fine. + if (classifyInterruptError(e) === "ignore") return Effect.void + // A genuine delivery failure must not read as success: the stop did not happen. The + // interface has no error channel, so surface it as a defect rather than a false success. const message = String((e as { message?: unknown })?.message ?? e) - // An idle session's workflow has already completed; nothing to interrupt is fine. A - // genuine delivery failure must not be silent: the user asked for a stop. - if (/already completed|not found/i.test(message)) return Effect.void - return Effect.logWarning("session interrupt signal failed").pipe( + return Effect.logError("session interrupt signal failed").pipe( Effect.annotateLogs({ sessionID: id, error: message }), + Effect.andThen(Effect.die(`session interrupt delivery failed for ${id}: ${message}`)), ) }), ), diff --git a/packages/core/test/temporal-interrupt-classify.test.ts b/packages/core/test/temporal-interrupt-classify.test.ts new file mode 100644 index 000000000000..93cbd447642b --- /dev/null +++ b/packages/core/test/temporal-interrupt-classify.test.ts @@ -0,0 +1,19 @@ +// The interrupt path must not report a genuine signal-delivery failure as success. Only an already +// closed workflow ("already completed"/"not found") is a benign no-op; everything else must fail. +import { describe, it, expect } from "bun:test" +import { SessionExecutionTemporal } from "@opencode-ai/core/session/execution/temporal" + +const classify = SessionExecutionTemporal.classifyInterruptError + +describe("temporal interrupt error classification", () => { + it("ignores an already-closed workflow", () => { + expect(classify(new Error("workflow execution already completed"))).toBe("ignore") + expect(classify(new Error("workflow not found"))).toBe("ignore") + }) + + it("fails a genuine delivery error", () => { + expect(classify(new Error("14 UNAVAILABLE: tcp connect error"))).toBe("fail") + expect(classify(new Error("DEADLINE_EXCEEDED"))).toBe("fail") + expect(classify("some non-error value")).toBe("fail") + }) +}) From f42e1f99bb848e24fe40c727b166e6f50f8fa536 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 21:05:07 -0700 Subject: [PATCH 082/103] Count all drains toward continue-as-new, including resume-driven ones. The bound only counted wake-loop drains, so a resume-heavy workflow never continued-as-new and its history grew until it hit Temporal's limit. The counter now increments inside drainTurn (every drain), and a `rolloverPending` flag lets the main loop trigger continueAsNew -- from the workflow's main method, never an update handler -- once the bound is crossed, even if the crossing drain came from a resume. Verified with a fake-runtime unit test (a resume drain crosses maxDrainsPerRun and rolls over). --- .../src/session/execution/workflow-core.ts | 20 ++++-- .../test/session-supervisor-rollover.test.ts | 71 +++++++++++++++++++ 2 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 packages/core/test/session-supervisor-rollover.test.ts diff --git a/packages/core/src/session/execution/workflow-core.ts b/packages/core/src/session/execution/workflow-core.ts index 8d8fbfd0c377..cebb41f32d51 100644 --- a/packages/core/src/session/execution/workflow-core.ts +++ b/packages/core/src/session/execution/workflow-core.ts @@ -52,6 +52,11 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => let stopping = false let draining = false let handlers = 0 + // Count EVERY drain (wake- and resume-driven) toward the continue-as-new bound. Counting only + // wake-loop drains let a resume-heavy session grow history without bound. `rolloverPending` + // lets the main loop trigger continue-as-new even when the drains came from resume handlers. + let drains = 0 + let rolloverPending = false const drainTurn = async (force: boolean) => { // Serialize drains, like the coordinator (one owner fiber per session at a time). Re-check @@ -63,6 +68,8 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => if (!draining) break } draining = true + drains++ + if (drains >= MAX_DRAINS_PER_RUN) rolloverPending = true try { let step = 1 let promotion: string | null = null @@ -99,11 +106,17 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => // interrupt cancels the whole scope, so a cancellation can surface at the idle wait itself, // not just inside a drain; treat it as a normal stop rather than a failure. - let drains = 0 try { for (;;) { - const gotWork = await rt.condition(() => pendingWake || stopping, IDLE_TIMEOUT) + // Wake on new work, a stop, OR a pending rollover (a resume drain may have crossed the + // bound while the main loop was parked). continue-as-new must be called from here, the + // workflow's main method -- never from an update handler. + const gotWork = await rt.condition(() => pendingWake || stopping || rolloverPending, IDLE_TIMEOUT) if (stopping) return + if (rt.continueAsNew && rolloverPending && !draining && handlers === 0) { + // A fresh run starts with a pending wake, so no queued work is lost across the boundary. + await rt.continueAsNew(sessionID) + } if (!gotWork) { // A wake can race the idle timer; without this re-check it would be dropped. if (pendingWake) continue @@ -118,9 +131,6 @@ export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => // wake tolerates run errors (the coordinator logs and moves on); only cancellation stops us. if (rt.isCancellation(e)) return } - drains++ - if (rt.continueAsNew && drains >= MAX_DRAINS_PER_RUN && handlers === 0) - await rt.continueAsNew(sessionID) } } catch (e) { if (rt.isCancellation(e)) return diff --git a/packages/core/test/session-supervisor-rollover.test.ts b/packages/core/test/session-supervisor-rollover.test.ts new file mode 100644 index 000000000000..2182124ae90d --- /dev/null +++ b/packages/core/test/session-supervisor-rollover.test.ts @@ -0,0 +1,71 @@ +// #4 regression: the continue-as-new bound must count EVERY drain, including resume-driven ones. +// Counting only wake-loop drains let a resume-heavy workflow accumulate history without ever rolling +// over. Driven with a fake WorkflowRuntime (no Temporal): a resume drain that crosses +// maxDrainsPerRun must trigger continueAsNew from the main loop. +import { describe, it, expect } from "bun:test" +import { makeWorkflows, type WorkflowRuntime } from "@opencode-ai/core/session/execution/workflow-core" +import type { StepDrainResult } from "@opencode-ai/core/session/execution/temporal-activities" + +class ContinuedAsNew extends Error {} +const DONE: StepDrainResult = { ran: true, continue: false, step: 1, promotion: null } +const settle = () => new Promise((r) => setTimeout(r, 0)) + +class FakeRuntime implements WorkflowRuntime { + steps = 0 + continued = 0 + private waiters: { predicate: () => boolean; resolve: (b: boolean) => void }[] = [] + private updates = new Map Promise>() + + condition = (predicate: () => boolean, _timeout?: string) => + new Promise((resolve) => { + this.waiters.push({ predicate, resolve }) + this.flush() + }) + setSignalHandler = () => {} + setUpdateHandler = (_name: "resume", handler: () => Promise) => this.updates.set("resume", handler) + runTurnStep = async () => { + this.steps++ + return DONE + } + cancelCurrentScope = () => {} + isCancellation = () => false // no interrupts in this test; continueAsNew propagates out + continueAsNew = async (): Promise => { + this.continued++ + throw new ContinuedAsNew() + } + + private flush() { + for (const w of [...this.waiters]) { + if (!w.predicate()) continue + this.waiters = this.waiters.filter((x) => x !== w) + w.resolve(true) + } + } + async resume(): Promise { + const p = this.updates.get("resume")!() + this.flush() + await p.catch(() => {}) + this.flush() + } +} + +describe("supervisor continue-as-new counting", () => { + it("counts a resume-driven drain toward the bound and rolls over from the main loop", async () => { + const rt = new FakeRuntime() + // Bound of 2: the initial wake drain is #1; a single resume drain is #2 and must roll over. + const supervisor = makeWorkflows(rt, { maxDrainsPerRun: 2 }).sessionTurn("ses_rollover") + let ended = false + supervisor.then( + () => (ended = true), + () => (ended = true), + ) + await settle() // initial wake drain (#1) + expect(rt.steps).toBe(1) + expect(rt.continued).toBe(0) + await rt.resume() // resume drain (#2) crosses the bound + await settle() + expect(rt.steps).toBe(2) + expect(rt.continued).toBe(1) // continue-as-new fired because the resume drain was counted + expect(ended).toBe(true) + }) +}) From b35d6277fe3cb84eeb5cfef29458a27e9add4628 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 21:06:52 -0700 Subject: [PATCH 083/103] Document the remaining Temporal supervisor coordination follow-ups. Records what's fixed (condition blocker, owner-token collision, interrupt failure reporting, continue-as-new counting) and the interlocking deep items left as one coherent pass (resume/wake lost at interrupt, concurrent resumes duplicating turns, fresh-resume spurious drain), each with a fix sketch and a harness test to validate it. --- .../docs/temporal-supervisor-followups.md | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 packages/temporal/docs/temporal-supervisor-followups.md diff --git a/packages/temporal/docs/temporal-supervisor-followups.md b/packages/temporal/docs/temporal-supervisor-followups.md new file mode 100644 index 000000000000..665a81351976 --- /dev/null +++ b/packages/temporal/docs/temporal-supervisor-followups.md @@ -0,0 +1,83 @@ +# Temporal supervisor: follow-ups + +Independent review (Codex, several rounds) enumerated correctness issues in the Temporal-mode +session supervisor (`packages/core/src/session/execution/workflow-core.ts` + `temporal.ts`). Local +mode is unaffected: it runs the proven `SessionRunCoordinator` (`execution/local.ts`). + +A deterministic test harness now exists for this code +(`packages/core/test/temporal-harness-smoke.test.ts`, `@temporalio/testing` time-skipping running the +real workflow with a mock activity), plus fake-runtime unit tests of the supervisor loop. Every item +below is reproducible/verifiable through one of those. + +## Fixed + +- **Blocker: workflow never drained a turn.** On `@temporalio/workflow` 1.21, `condition(fn, timeout)` + with `fn` already true left the `CancellationScope` cancelled, so the next `condition()` threw and + the workflow completed with zero activities. Fixed by short-circuiting an already-true predicate in + the `condition` adapter (`temporal-workflow.ts`). This was the real cause of the "0 activities / + turn never runs" symptom (not the HTTP `steer` delivery). +- **Owner-token collision (event-log fence).** Token was `runId#attempt`; activity attempts restart + per step, so tokens repeated across steps and a zombie attempt could re-authorize. Now + `runId:activityId:attempt` (`temporal-activities.ts`). +- **Interrupt delivery failure reported as success.** A genuine signal failure was swallowed to + `void`; now surfaced as a defect (`temporal.ts`, `classifyInterruptError`). +- **continue-as-new ignored resume drains.** Only wake-loop drains counted toward the bound, so a + resume-heavy workflow grew history without rolling over. Now every drain counts and the main loop + rolls over on `rolloverPending` (`workflow-core.ts`). + +## Open (deep coordination pass — interlocking, do together) + +These three entangle (start intent ↔ continue-as-new state ↔ resume join ↔ interrupt generations), +so they are best done as one coherent supervisor pass, mirroring `SessionRunCoordinator`'s proven +semantics, rather than piecemeal. + +### 1. Resume/wake lost at the interrupt boundary (critical) + +`interrupt` sets `stopping` and cancels the workflow's scope; `drainTurn` then returns without a +successor, and the `resume` update handler ignores `stopping`. With `USE_EXISTING`, a resume admitted +after the interrupt attaches to the doomed workflow and is cancelled/abandoned instead of awaiting a +successor; a wake in the same window is dropped. Reference: `run-coordinator.ts` `run` (stopping +branch awaits cleanup then starts a successor) and `settle` (a wake during cleanup starts a +successor). + +Fix sketch: keep the workflow root alive and cancel a per-drain `CancellationScope` instead; model +`running → stopping → retired` generations explicitly; a resume while stopping awaits cleanup then +forces exactly one successor; a wake while stopping registers one non-forced successor. Consider +making `interrupt` an Update so it acknowledges only after cleanup. + +Test: harness — start, `interrupt`, then `executeUpdateWithStart(resume)`; assert it runs on a +successor rather than receiving cancellation. + +### 2. Concurrent resumes serialize into duplicate forced turns (critical) + +Each `resume` handler calls `drainTurn(true)` independently; `drainTurn` serializes on `draining` but +does not JOIN the active drain. Two concurrent resumes (or a resume during a wake drain) therefore +produce two forced drains, and a forced first step bypasses the "no eligible work" check +(`runner/llm.ts`) and calls the provider anyway — duplicate provider turns, charges, transcript +entries, tool side effects. Reference: `run-coordinator.ts` `run` joins the existing `done`. + +Fix sketch: route all drains through a single in-flight promise; resume joins it (or forces one only +when idle). A wake that only joins a resume-started drain must still get one follow-up drain (the +`joined` re-arm). Both were prototyped earlier and unit-tested with a fake runtime; fold into the +generation pass. + +Test: harness — two concurrent `resume`s → assert exactly one `runTurnStep` activity. + +### 3. Fresh-resume spurious drain (medium) + +The supervisor always starts `pendingWake = true`, but a resume-with-start carries no wake, so a +fresh resume does its forced drain AND a second, no-op wake drain (extra activity/history; not +incorrect). Coupled to continue-as-new: the current `pendingWake = true` start is also what keeps a +continued-as-new run from losing queued work, so fixing this needs explicit start intent threaded +through the workflow args and `continueAsNew`. + +Fix sketch: start `pendingWake` from an explicit initial-intent arg (wake vs resume vs rollover); +carry pending state across `continueAsNew` rather than manufacturing a wake every run. + +Test: harness — `executeUpdateWithStart(resume)` on a fresh workflow → assert exactly one drain. + +## Also noted (lower priority) + +- `active` (visibility query) reports an idle-but-parked workflow as running for the idle window; a + workflow query or search attribute exposing `pendingWake || draining || handlers > 0` would be + more accurate. From a964bcab6cc2a52fe91d39bab07cee133f2266a0 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 23:00:49 -0700 Subject: [PATCH 084/103] Redesign the Temporal supervisor: interrupt keeps serving, resume joins. Addresses the deep coordination findings from independent review, for Temporal mode (local mode uses SessionRunCoordinator and is unaffected): - resume JOINS the single in-flight drain instead of queueing a second forced one (concurrent resumes no longer duplicate provider turns / tool side effects), mirroring SessionRunCoordinator.run. - interrupt stops the CURRENT turn, not the session: it cancels only the turn's child cancellation scope (runInDrainScope) and the long-lived workflow keeps serving, so a wake/resume that races the interrupt drives a fresh turn on the same workflow instead of being lost to a doomed one. - A real workflow (root) cancellation is detected via the root scope and stops the supervisor -- never keeps serving or continue-as-news. - Explicit start intent: resume-with-start passes startWithWake=false, so a fresh resume no longer does a spurious wake drain; carried across continue-as-new. - continue-as-new counts every drain and gates on allHandlersFinished() so an in-flight update's result is never abandoned; a resume-driven rollover carries no spurious wake. Also fixes a blocker uncovered while validating this: the SDK's condition(fn, timeout) on @temporalio/workflow 1.21 leaks its timer-scope cancellation into the root scope when it resolves, which poisoned the next drain -- a session could serve only one turn. The timed wait now races a no-timeout condition against a bare sleep and abandons the loser, cancelling no scope, so nothing leaks (and root-cancellation detection stays reliable). --- .../session/execution/temporal-workflow.ts | 50 ++++- .../core/src/session/execution/temporal.ts | 14 +- .../src/session/execution/workflow-core.ts | 191 ++++++++++-------- 3 files changed, 164 insertions(+), 91 deletions(-) diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index 64816b33871a..a93fa0a2b546 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -13,9 +13,11 @@ import { defineUpdate, setHandler, condition, + sleep, continueAsNew, CancellationScope, isCancellation, + allHandlersFinished, } from "@temporalio/workflow" import type { StepActivities } from "./temporal-activities" import { makeWorkflows, type WorkflowRuntime } from "./workflow-core" @@ -45,25 +47,61 @@ const runtime: WorkflowRuntime = { // leaves the current CancellationScope cancelled, so the NEXT condition() throws CancelledFailure // -- which the supervisor reads as an interrupt and the workflow completes without ever draining a // turn. Checking fn() first keeps the timeout timer (and its scope) out of the already-true path. + // A timed wait that does NOT use the SDK's condition(fn, timeout). On @temporalio/workflow 1.21 + // that variant cancels its internal timer scope on resolve and the cancellation LEAKS into the + // parent (root) scope, so the next drain's child scope is born cancelled and the turn never runs + // (a session could serve only one turn). Instead: short-circuit an already-true predicate; for a + // real wait, race a no-timeout condition against a bare timer and abandon the loser. Nothing here + // cancels a scope, so nothing leaks; an unfired timer / unresolved condition is harmless and a + // pending timer is cleaned up when the workflow closes. condition: async (predicate, timeout) => { if (predicate()) return true if (timeout === undefined) { await condition(predicate) return true } - // The runtime interface uses plain strings; the SDK's Duration is a branded string template. - return condition(predicate, timeout as never) + let timedOut = false + const timer = sleep(timeout as never).then(() => { + timedOut = true + }) + timer.catch(() => {}) + await Promise.race([condition(() => predicate() || timedOut), timer]) + return predicate() }, setSignalHandler: (name, handler) => setHandler(signals[name], handler), setUpdateHandler: (_name, handler) => setHandler(resume, handler), runTurnStep, - cancelCurrentScope: () => CancellationScope.current().cancel(), + // Run the turn's drain inside its own cancellable scope, tracked as the active one. An interrupt + // cancels this scope (aborting the in-flight activity) without touching the workflow root, so the + // supervisor survives to serve later turns. + runInDrainScope: (fn) => + CancellationScope.cancellable(async () => { + activeDrainScope = CancellationScope.current() + try { + return await fn() + } finally { + activeDrainScope = undefined + } + }), + cancelCurrentScope: () => activeDrainScope?.cancel(), isCancellation, - continueAsNew: (sessionID) => continueAsNew<(id: string) => Promise>(sessionID), + // A per-turn interrupt cancels only the child drain scope; a real workflow cancellation cancels + // the root. Reliable now that the timed wait no longer cancels any scope (nothing contaminates the + // root's consideredCancelled). + isRootCancelled: () => rootScope?.consideredCancelled ?? false, + allHandlersFinished, + continueAsNew: (sessionID, startWithWake) => + continueAsNew<(id: string, startWithWake: boolean) => Promise>(sessionID, startWithWake), } +// The scope of the drain currently running, so an interrupt signal can cancel exactly that turn. +let activeDrainScope: CancellationScope | undefined +// The workflow's root scope, captured at entry, to detect a whole-run cancellation. +let rootScope: CancellationScope | undefined + const workflows = makeWorkflows(runtime) -export async function sessionTurn(sessionID: string): Promise { - return workflows.sessionTurn(sessionID) +export async function sessionTurn(sessionID: string, startWithWake: boolean = true): Promise { + rootScope = CancellationScope.current() + return workflows.sessionTurn(sessionID, startWithWake) } diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index 7a1f16c5c2c0..b20783a1d488 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -47,10 +47,11 @@ const HOST_CLIENT = ROLE !== "worker" /** * A Temporal-backed SessionExecution. It makes each session a durable workflow: - * - wake -> signalWithStart(wake) (start while idle, or coalesce into the running run) - * - resume -> signalWithStart(force) (force one drain even with no eligible input) - * - interrupt -> signal(interrupt) (cancels the workflow's scope -> aborts the drain) - * - active -> the set of sessions this process has started + * - wake -> signalWithStart(wake) (start while idle, or coalesce into the run) + * - resume -> executeUpdateWithStart(resume) (force one drain and await its result) + * - interrupt -> signal(interrupt) (cancels the current turn's drain scope; the + * workflow keeps serving later wakes/resumes) + * - active -> the running per-session workflows (visibility-backed) * * The drain runs one step (SessionRunner.runStep) inside a Temporal activity, looped by the * workflow. Because turn state lives in the durable event log, a worker crash is recovered by @@ -185,7 +186,10 @@ const layer = Layer.effect( const startOp = new WithStartWorkflowOperation(WORKFLOW_TYPE, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), - args: [id], + // startWithWake=false: a fresh resume-with-start must not manufacture a wake drain; + // its forced drain comes from the resume update. Ignored when USE_EXISTING joins a + // running workflow (which keeps its own state). + args: [id, false], workflowIdConflictPolicy: "USE_EXISTING", }) return client.workflow.executeUpdateWithStart(WF.resume, { diff --git a/packages/core/src/session/execution/workflow-core.ts b/packages/core/src/session/execution/workflow-core.ts index cebb41f32d51..6e0a3d2a296b 100644 --- a/packages/core/src/session/execution/workflow-core.ts +++ b/packages/core/src/session/execution/workflow-core.ts @@ -1,20 +1,20 @@ -// The Temporal driver's per-session supervisor, expressed over a six-primitive runtime interface so -// the SDK's condition/signals/updates/activities plug in (temporal-workflow.ts) and it can be -// unit-tested off a live cluster. Local mode does NOT run this loop: it uses the proven -// SessionRunCoordinator directly (execution/local.ts). The two modes share SessionRunner and the -// durable event log, not this supervisor. +// The Temporal driver's per-session supervisor, expressed over a runtime interface so the SDK's +// condition/signals/updates/activities/cancellation plug in (temporal-workflow.ts) and it stays +// unit-testable off a live cluster (a fake runtime). Local mode does NOT run this loop: it uses the +// proven SessionRunCoordinator directly (execution/local.ts). The two modes share SessionRunner and +// the durable event log, not this supervisor. // // MUST stay pure: the Temporal driver bundles this into the workflow sandbox, so no `effect`, no // `@opencode-ai/core` runtime imports, no Node builtins. Type-only imports are erased and safe. // -// Semantics mirror the local coordinator (run-coordinator.ts): drains are serialized (one at a -// time), a `wake` drives a drain and is tolerant of errors, and `resume` (an update) drives a -// forced drain and returns its result to the caller (throwing the run's error). The supervisor -// stays alive to serve later wakes/resumes and terminates after an idle period. +// Semantics mirror SessionRunCoordinator (run-coordinator.ts): at most one drain runs at a time; a +// `wake` drives a drain and tolerates errors; `resume` JOINS the active drain (or forces one when +// idle) and surfaces its result; `interrupt` stops the CURRENT turn (not the session) and the +// long-lived supervisor keeps serving later wakes/resumes, terminating only after an idle period. import type { StepDrainInput, StepDrainResult } from "./temporal-activities" -/** What a driver must provide. Six primitives; everything else is supervisor logic. */ +/** What a driver must provide; everything else is supervisor logic. */ export interface WorkflowRuntime { /** Wait until the predicate is true. With a timeout, resolve false when it expires first. */ readonly condition: (predicate: () => boolean, timeout?: string) => Promise @@ -22,12 +22,29 @@ export interface WorkflowRuntime { readonly setUpdateHandler: (name: "resume", handler: () => Promise) => void /** One step of a turn (SessionRunner.runStep). The Temporal driver runs it as an activity. */ readonly runTurnStep: (input: StepDrainInput) => Promise - /** Cancel the in-flight drain and any parked condition (interrupt semantics). */ + /** + * Run one whole turn's drain inside a fresh cancellable scope. `cancelCurrentScope()` cancels the + * scope of the drain currently running, which aborts its in-flight step; the supervisor stays + * alive. Scoping the cancellation to the turn (not the workflow) is what lets an interrupt stop + * the current turn without killing the session. + */ + readonly runInDrainScope: (fn: () => Promise) => Promise + /** Cancel the drain scope currently running (interrupt the active turn). No-op when idle. */ readonly cancelCurrentScope: () => void /** Whether an error is the driver's cancellation (a normal stop, not a failure). */ readonly isCancellation: (error: unknown) => boolean - /** Restart the run with fresh history. Only meaningful for drivers that keep history. */ - readonly continueAsNew?: (sessionID: string) => Promise + /** Whether the WHOLE run (root scope) is cancelled -- a real workflow cancellation, as opposed to + * a per-turn interrupt (which cancels only the current drain's child scope). A root cancellation + * must stop the supervisor; it must never keep serving or continue-as-new. Reliable because the + * timed wait no longer cancels any scope (so it cannot contaminate the root). */ + readonly isRootCancelled: () => boolean + /** Whether every signal/update handler has fully finished (Temporal's own accounting). Gates + * completion and continue-as-new so an in-flight update's result is never abandoned. Optional: + * drivers without a handler protocol return true. */ + readonly allHandlersFinished?: () => boolean + /** Restart the run with fresh history, carrying whether work is still pending. History-keeping + * drivers only (Temporal). */ + readonly continueAsNew?: (sessionID: string, startWithWake: boolean) => Promise } export interface WorkflowOptions { @@ -40,99 +57,113 @@ export interface WorkflowOptions { export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => { const IDLE_TIMEOUT = options?.idleTimeout ?? "5 minutes" // A continuously busy session never hits the idle return, so without a bound its history grows - // until Temporal terminates the workflow. A fresh run starts with a pending wake, so no work is - // lost across the boundary. + // until Temporal terminates the workflow. continue-as-new carries the pending-wake state, so no + // queued work is lost across the boundary. const MAX_DRAINS_PER_RUN = options?.maxDrainsPerRun ?? 30 - // The turn is driven one step at a time: each step (one provider attempt + its tools) is its - // own drain call, and the step loop is supervisor control flow. The loop state - // (step / promotion / first) mirrors SessionRunner.run's loop. - async function sessionTurn(sessionID: string): Promise { - let pendingWake = true // started by a wake -> there is work to drain - let stopping = false - let draining = false - let handlers = 0 - // Count EVERY drain (wake- and resume-driven) toward the continue-as-new bound. Counting only - // wake-loop drains let a resume-heavy session grow history without bound. `rolloverPending` - // lets the main loop trigger continue-as-new even when the drains came from resume handlers. + // Each step (one provider attempt + its tools) is its own activity; the step loop is supervisor + // control flow (step / promotion / first mirror SessionRunner.run's loop). `startWithWake` is the + // explicit start intent: a wake-with-start begins with pending work, a resume-with-start does not + // (its forced drain comes from the resume update), so resume must not manufacture a spurious wake. + async function sessionTurn(sessionID: string, startWithWake: boolean = true): Promise { + let pendingWake = startWithWake let drains = 0 let rolloverPending = false + let resumers = 0 // in-flight resume handlers awaiting a drain + // The single in-flight drain, or null when idle. New callers JOIN it rather than starting a + // second, mirroring SessionRunCoordinator.run: one execution per session at a time, and a resume + // attaches to the running one instead of queueing a redundant forced drain. + let inFlight: Promise | null = null - const drainTurn = async (force: boolean) => { - // Serialize drains, like the coordinator (one owner fiber per session at a time). Re-check - // after every wakeup: two waiters parked on the same condition can both observe `!draining` - // in one activation, and without the loop both would start a drain. - for (;;) { - await rt.condition(() => !draining || stopping) - if (stopping) return - if (!draining) break - } - draining = true - drains++ - if (drains >= MAX_DRAINS_PER_RUN) rolloverPending = true - try { - let step = 1 - let promotion: string | null = null - let first = true - for (;;) { - const r: StepDrainResult = await rt.runTurnStep({ sessionID, step, promotion, first, force }) - if (!r.continue) break - step = r.step - promotion = r.promotion - first = false - } - } finally { - draining = false - } + const drive = (force: boolean): Promise => { + if (inFlight) return inFlight + const running = rt + .runInDrainScope(async () => { + drains++ + if (drains >= MAX_DRAINS_PER_RUN) rolloverPending = true + let step = 1 + let promotion: string | null = null + let first = true + for (;;) { + const r: StepDrainResult = await rt.runTurnStep({ sessionID, step, promotion, first, force }) + if (!r.continue) break + step = r.step + promotion = r.promotion + first = false + } + }) + .finally(() => { + inFlight = null + }) + inFlight = running + return running } rt.setSignalHandler("wake", () => { pendingWake = true }) + // interrupt stops the CURRENT turn, not the session: cancel the active drain's child scope. The + // supervisor keeps serving, so a later wake/resume drives a fresh turn on this same long-lived + // workflow and a prompt that races the interrupt is never stranded. A per-turn interrupt leaves + // the root scope untouched, so isRootCancelled() distinguishes it from a real cancellation. rt.setSignalHandler("interrupt", () => { - stopping = true rt.cancelCurrentScope() }) - // resume = coordinator.run: force one drain and surface its result (a run error rejects the - // update, so the caller observes it). + // resume = coordinator.run: join the active drain (or force one when idle) and surface its + // result. A run error, or an interruption of the joined drain, rejects the update so the caller + // observes it. rt.setUpdateHandler("resume", async () => { - handlers++ + resumers++ try { - await drainTurn(true) + await drive(true) } finally { - handlers-- + resumers-- } }) - // interrupt cancels the whole scope, so a cancellation can surface at the idle wait itself, - // not just inside a drain; treat it as a normal stop rather than a failure. + // "Idle" for completion/continue-as-new means: no drain in flight, no resume handler in our + // accounting, AND Temporal reports every handler finished (so an update's result is never + // abandoned by completing/continuing before the protocol records it). + const quiescent = () => !inFlight && resumers === 0 && (rt.allHandlersFinished?.() ?? true) + try { for (;;) { - // Wake on new work, a stop, OR a pending rollover (a resume drain may have crossed the - // bound while the main loop was parked). continue-as-new must be called from here, the - // workflow's main method -- never from an update handler. - const gotWork = await rt.condition(() => pendingWake || stopping || rolloverPending, IDLE_TIMEOUT) - if (stopping) return - if (rt.continueAsNew && rolloverPending && !draining && handlers === 0) { - // A fresh run starts with a pending wake, so no queued work is lost across the boundary. - await rt.continueAsNew(sessionID) + // Wake on a real wake, or on an ACTIONABLE rollover (bound crossed and quiescent). Gating + // the rollover on quiescence keeps the loop from spinning while a resume drain is still in + // flight, and -- crucially -- keeps a rollover from being mis-handled as a wake below. + const woke = await rt.condition(() => pendingWake || (rolloverPending && quiescent()), IDLE_TIMEOUT) + // A real root cancellation must stop the supervisor -- never keep serving or continue-as-new. + // (Checked here too so the rollover short-circuit path can't continue-as-new a cancelled run.) + if (rt.isRootCancelled()) return + // continue-as-new: only from the main method, only when quiescent. Carry the pending wake so + // queued work survives the boundary (a pure rollover carries false -> no spurious drain). + if (rt.continueAsNew && rolloverPending && quiescent()) { + await rt.continueAsNew(sessionID, pendingWake) } - if (!gotWork) { - // A wake can race the idle timer; without this re-check it would be dropped. - if (pendingWake) continue - // Idle: terminate only when nothing is in flight. A later wake/resume starts a fresh run. - if (!draining && handlers === 0) return + if (pendingWake) { + pendingWake = false + // If a resume already started the drain we take, we only JOIN it -- and it may be past the + // point where it could pick up the work this wake signals. Re-arm pendingWake for one + // follow-up drain; a fresh drain we start ourselves already covers the wake. + const joined = inFlight !== null + try { + await drive(false) + } catch (e) { + // A real root cancellation wins: propagate it to end the workflow. A per-turn interrupt + // or a run error (already recorded in the log) is tolerated and the supervisor keeps + // serving. + if (rt.isRootCancelled()) throw e + } + if (joined) pendingWake = true continue } - pendingWake = false - try { - await drainTurn(false) - } catch (e) { - // wake tolerates run errors (the coordinator logs and moves on); only cancellation stops us. - if (rt.isCancellation(e)) return - } + // No real wake. Retire only on a genuine idle timeout with nothing in flight; a rollover + // wakeup was either handled above (continue-as-new) or is waiting for the drain to finish. + if (!woke && quiescent()) return } } catch (e) { + // A real root cancellation (workflow cancelled) or a defect reaches here. End the workflow: a + // cancellation completes the run cleanly, a defect fails it. if (rt.isCancellation(e)) return throw e } From 82d5beb4d0868c1bdd6dfe04b8fc5d0495afd4bc Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 23:00:49 -0700 Subject: [PATCH 085/103] Test the supervisor redesign: fake-runtime + real-Temporal harness. Fake-runtime unit tests (session-supervisor.test.ts): resume joins one drain, concurrent resumes join, a wake that only joins a resume drain still gets a follow-up, interrupt keeps the supervisor serving, a root cancellation stops it, and a fresh resume-with-start does exactly one drain. Rollover test asserts a resume-driven rollover carries startWithWake=false. Real-Temporal harness (@temporalio/testing), each in its own file since two native servers per bun process segfault: - interrupt: a per-turn interrupt cancels the turn but a later wake runs a second turn on the same workflow. - multiturn: turn 1 completes, the supervisor parks in the idle timed wait (asserted via a TimerStarted history event), then a wake drives turn 2 -- the exact path the condition-leak broke. --- .../test/session-supervisor-rollover.test.ts | 17 +- packages/core/test/session-supervisor.test.ts | 169 ++++++++++++++++++ .../test/temporal-harness-interrupt.test.ts | 82 +++++++++ .../test/temporal-harness-multiturn.test.ts | 69 +++++++ 4 files changed, 331 insertions(+), 6 deletions(-) create mode 100644 packages/core/test/session-supervisor.test.ts create mode 100644 packages/core/test/temporal-harness-interrupt.test.ts create mode 100644 packages/core/test/temporal-harness-multiturn.test.ts diff --git a/packages/core/test/session-supervisor-rollover.test.ts b/packages/core/test/session-supervisor-rollover.test.ts index 2182124ae90d..cfafe24e4d71 100644 --- a/packages/core/test/session-supervisor-rollover.test.ts +++ b/packages/core/test/session-supervisor-rollover.test.ts @@ -1,7 +1,7 @@ -// #4 regression: the continue-as-new bound must count EVERY drain, including resume-driven ones. -// Counting only wake-loop drains let a resume-heavy workflow accumulate history without ever rolling -// over. Driven with a fake WorkflowRuntime (no Temporal): a resume drain that crosses -// maxDrainsPerRun must trigger continueAsNew from the main loop. +// #4 regression: continue-as-new must count EVERY drain, including resume-driven ones (counting only +// wake-loop drains let a resume-heavy workflow grow history without rolling over). Plus the fix for +// the rollover-manufactures-a-wake bug: a rollover triggered by a resume drain must carry +// startWithWake=false into the successor, not re-arm a spurious wake. Driven by a fake runtime. import { describe, it, expect } from "bun:test" import { makeWorkflows, type WorkflowRuntime } from "@opencode-ai/core/session/execution/workflow-core" import type { StepDrainResult } from "@opencode-ai/core/session/execution/temporal-activities" @@ -13,6 +13,7 @@ const settle = () => new Promise((r) => setTimeout(r, 0)) class FakeRuntime implements WorkflowRuntime { steps = 0 continued = 0 + continuedWith: boolean | undefined private waiters: { predicate: () => boolean; resolve: (b: boolean) => void }[] = [] private updates = new Map Promise>() @@ -27,10 +28,13 @@ class FakeRuntime implements WorkflowRuntime { this.steps++ return DONE } + runInDrainScope = (fn: () => Promise) => fn() cancelCurrentScope = () => {} isCancellation = () => false // no interrupts in this test; continueAsNew propagates out - continueAsNew = async (): Promise => { + isRootCancelled = () => false + continueAsNew = async (_sessionID: string, startWithWake: boolean): Promise => { this.continued++ + this.continuedWith = startWithWake throw new ContinuedAsNew() } @@ -50,7 +54,7 @@ class FakeRuntime implements WorkflowRuntime { } describe("supervisor continue-as-new counting", () => { - it("counts a resume-driven drain toward the bound and rolls over from the main loop", async () => { + it("counts a resume-driven drain toward the bound and rolls over carrying no spurious wake", async () => { const rt = new FakeRuntime() // Bound of 2: the initial wake drain is #1; a single resume drain is #2 and must roll over. const supervisor = makeWorkflows(rt, { maxDrainsPerRun: 2 }).sessionTurn("ses_rollover") @@ -66,6 +70,7 @@ describe("supervisor continue-as-new counting", () => { await settle() expect(rt.steps).toBe(2) expect(rt.continued).toBe(1) // continue-as-new fired because the resume drain was counted + expect(rt.continuedWith).toBe(false) // no wake was pending, so none is carried (no spurious drain) expect(ended).toBe(true) }) }) diff --git a/packages/core/test/session-supervisor.test.ts b/packages/core/test/session-supervisor.test.ts new file mode 100644 index 000000000000..7618966060cf --- /dev/null +++ b/packages/core/test/session-supervisor.test.ts @@ -0,0 +1,169 @@ +// Deterministic unit tests for the Temporal supervisor (execution/workflow-core.ts) driven by a fake +// WorkflowRuntime -- no Temporal, no DB. Covers the coordination redesign: resume joins the single +// in-flight drain (never a second forced drain), a wake that only joins a resume drain still gets a +// follow-up, an interrupt stops the current turn but the supervisor keeps serving, and a fresh +// resume-with-start (startWithWake=false) does exactly one drain (no spurious wake drain). +import { describe, it, expect } from "bun:test" +import { makeWorkflows, type WorkflowRuntime } from "@opencode-ai/core/session/execution/workflow-core" +import type { StepDrainResult } from "@opencode-ai/core/session/execution/temporal-activities" + +class FakeCancel extends Error {} +const DONE: StepDrainResult = { ran: true, continue: false, step: 1, promotion: null } +const settle = () => new Promise((r) => setTimeout(r, 0)) + +// runTurnStep either resolves immediately (ungated) or parks until released (gated), so a turn can +// be held in flight while resumes/wakes/interrupts are delivered. cancelCurrentScope rejects the +// in-flight step with FakeCancel, modelling an interrupt aborting the active drain's scope. +class FakeRuntime implements WorkflowRuntime { + steps = 0 + gated = false + private waiters: { predicate: () => boolean; resolve: (b: boolean) => void; isTimeout: boolean }[] = [] + private signals = new Map void>() + private updates = new Map Promise>() + private pending: { resolve: (r: StepDrainResult) => void; reject: (e: unknown) => void }[] = [] + + condition = (predicate: () => boolean, timeout?: string) => + new Promise((resolve) => { + this.waiters.push({ predicate, resolve, isTimeout: timeout !== undefined }) + this.flush() + }) + setSignalHandler = (name: "wake" | "interrupt", handler: () => void) => { + this.signals.set(name, handler) + } + setUpdateHandler = (_name: "resume", handler: () => Promise) => this.updates.set("resume", handler) + runTurnStep = () => { + this.steps++ + if (!this.gated) return Promise.resolve(DONE) + return new Promise((resolve, reject) => this.pending.push({ resolve, reject })) + } + runInDrainScope = (fn: () => Promise) => fn() + cancelCurrentScope = () => { + for (const p of this.pending.splice(0)) p.reject(new FakeCancel()) + } + isCancellation = (e: unknown) => e instanceof FakeCancel + rootCancelled = false + isRootCancelled = () => this.rootCancelled + + private flush() { + for (const w of [...this.waiters]) { + if (!w.predicate()) continue + this.waiters = this.waiters.filter((x) => x !== w) + w.resolve(true) + } + } + deliver(name: "wake" | "interrupt") { + this.signals.get(name)?.() + this.flush() + } + resume(): Promise { + const p = this.updates.get("resume")!() + this.flush() + return p + } + fireIdle() { + const expiring = [...this.waiters] + this.waiters = [] + for (const w of expiring) w.resolve(false) + } + releaseStep(result: StepDrainResult = DONE) { + this.pending.shift()?.resolve(result) + } + get pendingSteps() { + return this.pending.length + } +} + +const start = (rt: FakeRuntime, startWithWake = true) => { + let ended = false + makeWorkflows(rt) + .sessionTurn("ses_test", startWithWake) + .then( + () => (ended = true), + () => (ended = true), + ) + return { isDone: () => ended } +} + +describe("Temporal supervisor (workflow-core)", () => { + it("concurrent resumes join a single drain (no duplicate forced turns)", async () => { + const rt = new FakeRuntime() + rt.gated = true + start(rt, false) // fresh resume-with-start: no initial wake drain + const r1 = rt.resume() + const r2 = rt.resume() + await settle() + expect(rt.steps).toBe(1) // joined + rt.releaseStep(DONE) + await Promise.all([r1, r2]) + expect(rt.steps).toBe(1) + }) + + it("a fresh resume-with-start does exactly one drain (no spurious wake drain)", async () => { + const rt = new FakeRuntime() + rt.gated = true + start(rt, false) + const resumed = rt.resume() + await settle() + expect(rt.steps).toBe(1) + rt.releaseStep(DONE) + await resumed + await settle() + expect(rt.steps).toBe(1) // no extra drain manufactured by an initial pendingWake + }) + + it("interrupt stops the current turn but the supervisor keeps serving", async () => { + const rt = new FakeRuntime() + rt.gated = true + const sup = start(rt, true) // initial wake drain + await settle() + expect(rt.steps).toBe(1) + rt.deliver("interrupt") // cancels the in-flight drain + await settle() + expect(sup.isDone()).toBe(false) // supervisor did NOT exit + rt.deliver("wake") // a new prompt after the interrupt + await settle() + expect(rt.steps).toBe(2) // drives a fresh turn on the same supervisor + rt.releaseStep(DONE) + await settle() + rt.fireIdle() + await settle() + expect(sup.isDone()).toBe(true) // ends only when idle + }) + + it("a root cancellation (not a per-turn interrupt) stops the supervisor", async () => { + const rt = new FakeRuntime() + rt.gated = true + const sup = start(rt, true) + await settle() + expect(rt.steps).toBe(1) + // A real workflow (root) cancellation, not a per-turn interrupt: the root scope is cancelled and + // it propagates into the drain. The supervisor must end -- not keep serving (the per-turn- + // interrupt behavior tested above). + rt.rootCancelled = true + rt.cancelCurrentScope() + await settle() + expect(sup.isDone()).toBe(true) + rt.deliver("wake") // a wake after root cancellation must NOT drive a new turn + await settle() + expect(rt.steps).toBe(1) + }) + + it("a wake that only joins a resume drain still gets its own follow-up", async () => { + const rt = new FakeRuntime() + rt.gated = true + start(rt, false) + const resumed = rt.resume() + await settle() + expect(rt.steps).toBe(1) + rt.deliver("wake") // arrives while the resume drain is in flight -> can only join + await settle() + expect(rt.steps).toBe(1) + rt.releaseStep(DONE) // resume drain ends -> the joined wake triggers a follow-up + await resumed + await settle() + expect(rt.steps).toBe(2) + rt.releaseStep(DONE) + await settle() + expect(rt.steps).toBe(2) + }) +}) diff --git a/packages/core/test/temporal-harness-interrupt.test.ts b/packages/core/test/temporal-harness-interrupt.test.ts new file mode 100644 index 000000000000..c452d712ce74 --- /dev/null +++ b/packages/core/test/temporal-harness-interrupt.test.ts @@ -0,0 +1,82 @@ +// Real-Temporal validation that an interrupt cancels the CURRENT turn but the supervisor keeps +// serving (the redesigned interrupt semantics). Uses a real (non-time-skipping) local server so a +// blocking, heartbeating activity behaves in real time -- time-skipping would jump past its +// heartbeat timeout and spuriously retry it. Runs in its own file: two native Temporal servers in +// one bun process segfault the runtime. +import { describe, it, expect } from "bun:test" +import { fileURLToPath } from "node:url" +import { TestWorkflowEnvironment } from "@temporalio/testing" +import { Worker } from "@temporalio/worker" +import { Context, heartbeat, CancelledFailure } from "@temporalio/activity" + +const WORKFLOW = fileURLToPath(new URL("../src/session/execution/temporal-workflow.ts", import.meta.url)) +const poll = async (fn: () => boolean, ms = 20_000) => { + const deadline = Date.now() + ms + while (!fn()) { + if (Date.now() > deadline) throw new Error("condition not reached") + await new Promise((r) => setTimeout(r, 50)) + } +} + +describe("temporal workflow harness: interrupt", () => { + it("interrupt cancels the current turn but the workflow keeps serving the next", async () => { + const env = await TestWorkflowEnvironment.createLocal() + let steps = 0 + let firstStarted = false + try { + const worker = await Worker.create({ + connection: env.nativeConnection, + namespace: env.namespace, + taskQueue: "harness-interrupt", + workflowsPath: WORKFLOW, + activities: { + runTurnStep: async () => { + steps++ + if (steps === 1) { + // Turn 1 blocks until the interrupt cancels it, heartbeating so it isn't failed for + // liveness while it waits. + firstStarted = true + const beat = setInterval(() => { + try { + heartbeat() + } catch { + // no-op outside an activity context + } + }, 200) + try { + await new Promise((_resolve, reject) => { + Context.current().cancellationSignal.addEventListener("abort", () => + reject(new CancelledFailure("interrupted")), + ) + }) + } finally { + clearInterval(beat) + } + } + return { ran: true, continue: false, step: 1, promotion: null } + }, + }, + }) + + await worker.runUntil(async () => { + const handle = await env.client.workflow.signalWithStart("sessionTurn", { + taskQueue: "harness-interrupt", + workflowId: "wf-interrupt", + args: ["ses_interrupt"], + signal: "wake", + signalArgs: [], + }) + await poll(() => firstStarted) // turn 1's activity is running + await handle.signal("interrupt") // cancel the current turn's scope + // The workflow must still be alive to accept a new prompt; if the interrupt had ended it, + // this wake would target a completed workflow and the second turn would never run. + await handle.signal("wake") + await poll(() => steps === 2) // a second turn ran on the SAME workflow + }) + + expect(steps).toBe(2) + } finally { + await env.teardown() + } + }, 120_000) +}) diff --git a/packages/core/test/temporal-harness-multiturn.test.ts b/packages/core/test/temporal-harness-multiturn.test.ts new file mode 100644 index 000000000000..05917d446ee0 --- /dev/null +++ b/packages/core/test/temporal-harness-multiturn.test.ts @@ -0,0 +1,69 @@ +// Real-Temporal validation that a session serves MULTIPLE turns: turn 1 completes, the supervisor +// parks in its idle timed wait, and a later wake drives turn 2 on the SAME workflow. This is the +// exact path the timed-wait rewrite fixes -- the SDK's condition(fn, timeout) leaked its cancellation +// into the root scope when it resolved, so turn 2's drain scope was born cancelled and never ran (a +// session could serve only one turn). With the leaking condition this test fails (steps stays 1). +// +// Uses a real (non-time-skipping) local server so the idle timer doesn't fast-forward and retire the +// workflow before the second wake. Own file: two native Temporal servers in one bun process segfault. +import { describe, it, expect } from "bun:test" +import { fileURLToPath } from "node:url" +import { TestWorkflowEnvironment } from "@temporalio/testing" +import { Worker } from "@temporalio/worker" + +const WORKFLOW = fileURLToPath(new URL("../src/session/execution/temporal-workflow.ts", import.meta.url)) +const poll = async (fn: () => boolean, ms = 20_000) => { + const deadline = Date.now() + ms + while (!fn()) { + if (Date.now() > deadline) throw new Error("condition not reached") + await new Promise((r) => setTimeout(r, 50)) + } +} + +describe("temporal workflow harness: multi-turn", () => { + it("serves a second turn after parking in the idle wait", async () => { + const env = await TestWorkflowEnvironment.createLocal() + let steps = 0 + try { + const worker = await Worker.create({ + connection: env.nativeConnection, + namespace: env.namespace, + taskQueue: "harness-multiturn", + workflowsPath: WORKFLOW, + activities: { + runTurnStep: async () => { + steps++ + return { ran: true, continue: false, step: 1, promotion: null } + }, + }, + }) + + await worker.runUntil(async () => { + const handle = await env.client.workflow.signalWithStart("sessionTurn", { + taskQueue: "harness-multiturn", + workflowId: "wf-multiturn", + args: ["ses_multiturn"], + signal: "wake", + signalArgs: [], + }) + await poll(() => steps === 1) // turn 1 drained + // Deterministically wait until the supervisor has actually PARKED in the idle timed wait -- + // i.e. its sleep timer is recorded in history -- before waking it, so we genuinely exercise + // the timed-wait/park-then-wake path (not a race where the wake lands during turn 1). + const deadline = Date.now() + 20_000 + for (;;) { + const history = await handle.fetchHistory() + if ((history.events ?? []).some((e) => e.timerStartedEventAttributes)) break + if (Date.now() > deadline) throw new Error("supervisor never parked in the idle timed wait") + await new Promise((r) => setTimeout(r, 100)) + } + await handle.signal("wake") + await poll(() => steps === 2) // turn 2 drained on the SAME workflow (no leak poisoning it) + }) + + expect(steps).toBe(2) + } finally { + await env.teardown() + } + }, 120_000) +}) From 3ff520263c43e03f02befc49d82fc08b6f722ff6 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 23:01:47 -0700 Subject: [PATCH 086/103] Update the Temporal supervisor follow-ups doc: deep items now fixed. --- .../docs/temporal-supervisor-followups.md | 122 +++++++----------- 1 file changed, 50 insertions(+), 72 deletions(-) diff --git a/packages/temporal/docs/temporal-supervisor-followups.md b/packages/temporal/docs/temporal-supervisor-followups.md index 665a81351976..9e5facc8f432 100644 --- a/packages/temporal/docs/temporal-supervisor-followups.md +++ b/packages/temporal/docs/temporal-supervisor-followups.md @@ -1,83 +1,61 @@ -# Temporal supervisor: follow-ups +# Temporal supervisor: status and follow-ups -Independent review (Codex, several rounds) enumerated correctness issues in the Temporal-mode -session supervisor (`packages/core/src/session/execution/workflow-core.ts` + `temporal.ts`). Local -mode is unaffected: it runs the proven `SessionRunCoordinator` (`execution/local.ts`). +Independent review (Codex, several rounds) enumerated correctness issues in the Temporal-mode session +supervisor (`packages/core/src/session/execution/workflow-core.ts` + `temporal-workflow.ts` + +`temporal.ts`). Local mode is unaffected: it runs the proven `SessionRunCoordinator` +(`execution/local.ts`). -A deterministic test harness now exists for this code -(`packages/core/test/temporal-harness-smoke.test.ts`, `@temporalio/testing` time-skipping running the -real workflow with a mock activity), plus fake-runtime unit tests of the supervisor loop. Every item -below is reproducible/verifiable through one of those. +A deterministic test harness now exists for this code: fake-runtime unit tests of the supervisor loop +(`session-supervisor.test.ts`, `session-supervisor-rollover.test.ts`) and real-Temporal tests via +`@temporalio/testing` (`temporal-harness-{smoke,interrupt,multiturn}.test.ts`). Every item below is +reproducible/verifiable through one of those. ## Fixed -- **Blocker: workflow never drained a turn.** On `@temporalio/workflow` 1.21, `condition(fn, timeout)` - with `fn` already true left the `CancellationScope` cancelled, so the next `condition()` threw and - the workflow completed with zero activities. Fixed by short-circuiting an already-true predicate in - the `condition` adapter (`temporal-workflow.ts`). This was the real cause of the "0 activities / - turn never runs" symptom (not the HTTP `steer` delivery). +- **Blocker: the workflow never drained a turn (and a session could serve only one turn).** On + `@temporalio/workflow` 1.21, `condition(fn, timeout)` cancels its internal timer scope on resolve + and that cancellation LEAKS into the parent/root scope, so the next `condition()`/drain saw a + cancelled scope. The supervisor completed with zero activities, or (after the first turn) a second + turn's drain was born cancelled. Fixed by not using the SDK's timed `condition` at all: the adapter + short-circuits an already-true predicate and, for a real wait, races a no-timeout `condition` + against a bare `sleep` and abandons the loser -- cancelling no scope, so nothing leaks. (This, not + the HTTP `steer` delivery, was the real cause of the "0 activities" symptom.) +- **Resume/wake lost at the interrupt boundary.** `interrupt` used to end the workflow, so a + resume/wake admitted in the interrupt→completion window (USE_EXISTING) attached to a doomed run. + Now `interrupt` cancels only the current turn's child cancellation scope (`runInDrainScope`); the + long-lived workflow keeps serving, and a later wake/resume drives a fresh turn on the same + workflow. Verified: `temporal-harness-interrupt.test.ts`. +- **Concurrent resumes duplicated forced turns.** `drainTurn` serialized but did not JOIN. Now every + drain routes through one in-flight promise and resume joins it (no duplicate provider turns / tool + side effects), mirroring `SessionRunCoordinator.run`. +- **Fresh-resume spurious drain.** Explicit start intent: `sessionTurn(sessionID, startWithWake)`, + resume-with-start passes `[id, false]`, and `pendingWake` is carried across `continueAsNew`, so a + fresh resume does exactly one drain. +- **continue-as-new ignored resume drains / manufactured a wake.** Every drain now counts toward the + bound, the main loop rolls over on `rolloverPending`, a resume-driven rollover carries no spurious + wake, and completion/continue-as-new gate on `allHandlersFinished()` so an in-flight update result + is never abandoned. +- **Real workflow (root) cancellation.** Detected via the root scope's `consideredCancelled` + (reliable now that the timed wait cancels no scope): a root cancellation stops the supervisor and + never keeps serving or continue-as-news; a per-turn interrupt (child scope) does not trip it. - **Owner-token collision (event-log fence).** Token was `runId#attempt`; activity attempts restart per step, so tokens repeated across steps and a zombie attempt could re-authorize. Now `runId:activityId:attempt` (`temporal-activities.ts`). - **Interrupt delivery failure reported as success.** A genuine signal failure was swallowed to `void`; now surfaced as a defect (`temporal.ts`, `classifyInterruptError`). -- **continue-as-new ignored resume drains.** Only wake-loop drains counted toward the bound, so a - resume-heavy workflow grew history without rolling over. Now every drain counts and the main loop - rolls over on `rolloverPending` (`workflow-core.ts`). -## Open (deep coordination pass — interlocking, do together) - -These three entangle (start intent ↔ continue-as-new state ↔ resume join ↔ interrupt generations), -so they are best done as one coherent supervisor pass, mirroring `SessionRunCoordinator`'s proven -semantics, rather than piecemeal. - -### 1. Resume/wake lost at the interrupt boundary (critical) - -`interrupt` sets `stopping` and cancels the workflow's scope; `drainTurn` then returns without a -successor, and the `resume` update handler ignores `stopping`. With `USE_EXISTING`, a resume admitted -after the interrupt attaches to the doomed workflow and is cancelled/abandoned instead of awaiting a -successor; a wake in the same window is dropped. Reference: `run-coordinator.ts` `run` (stopping -branch awaits cleanup then starts a successor) and `settle` (a wake during cleanup starts a -successor). - -Fix sketch: keep the workflow root alive and cancel a per-drain `CancellationScope` instead; model -`running → stopping → retired` generations explicitly; a resume while stopping awaits cleanup then -forces exactly one successor; a wake while stopping registers one non-forced successor. Consider -making `interrupt` an Update so it acknowledges only after cleanup. - -Test: harness — start, `interrupt`, then `executeUpdateWithStart(resume)`; assert it runs on a -successor rather than receiving cancellation. - -### 2. Concurrent resumes serialize into duplicate forced turns (critical) - -Each `resume` handler calls `drainTurn(true)` independently; `drainTurn` serializes on `draining` but -does not JOIN the active drain. Two concurrent resumes (or a resume during a wake drain) therefore -produce two forced drains, and a forced first step bypasses the "no eligible work" check -(`runner/llm.ts`) and calls the provider anyway — duplicate provider turns, charges, transcript -entries, tool side effects. Reference: `run-coordinator.ts` `run` joins the existing `done`. - -Fix sketch: route all drains through a single in-flight promise; resume joins it (or forces one only -when idle). A wake that only joins a resume-started drain must still get one follow-up drain (the -`joined` re-arm). Both were prototyped earlier and unit-tested with a fake runtime; fold into the -generation pass. - -Test: harness — two concurrent `resume`s → assert exactly one `runTurnStep` activity. - -### 3. Fresh-resume spurious drain (medium) - -The supervisor always starts `pendingWake = true`, but a resume-with-start carries no wake, so a -fresh resume does its forced drain AND a second, no-op wake drain (extra activity/history; not -incorrect). Coupled to continue-as-new: the current `pendingWake = true` start is also what keeps a -continued-as-new run from losing queued work, so fixing this needs explicit start intent threaded -through the workflow args and `continueAsNew`. - -Fix sketch: start `pendingWake` from an explicit initial-intent arg (wake vs resume vs rollover); -carry pending state across `continueAsNew` rather than manufacturing a wake every run. - -Test: harness — `executeUpdateWithStart(resume)` on a fresh workflow → assert exactly one drain. - -## Also noted (lower priority) - -- `active` (visibility query) reports an idle-but-parked workflow as running for the idle window; a - workflow query or search attribute exposing `pendingWake || draining || handlers > 0` would be - more accurate. +## Remaining follow-ups (not correctness bugs on a fresh deployment) + +- **Replay / versioning for rolling deploys.** This revision changes command-producing behavior + (single-flight resume, the timed wait no longer cancels a timer, `continueAsNew` carries a second + arg). A workflow already running under the OLD code can replay nondeterministically under the new + code. Before a rolling deploy, use Temporal Worker Versioning / patching or drain old executions. A + fresh deployment is unaffected. +- **`active` reports idle-but-parked workflows.** The visibility query lists running `sessionTurn` + workflows, so a session that finished a turn but hasn't idled out yet still shows as active. A + workflow query or search attribute exposing `pendingWake || inFlight || resumers > 0` would be more + precise if the API needs it. +- **Broader real-harness coverage.** The harness proves draining, idle retirement, interrupt + keep-serving, and park-then-wake. Concurrent real updates → one activity, resume-with-start → one + drain, and a real root-cancel-during-drain are covered at the fake-runtime level; promoting them to + the real harness would add belt-and-suspenders confidence. From 5cde171c5e6db696bc57f49f0d3b88b05e43f69e Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 23:02:40 -0700 Subject: [PATCH 087/103] Add a branch summary describing the local pivot and Temporal-mode fixes. --- .../docs/native-local-driver-branch.md | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 packages/temporal/docs/native-local-driver-branch.md diff --git a/packages/temporal/docs/native-local-driver-branch.md b/packages/temporal/docs/native-local-driver-branch.md new file mode 100644 index 000000000000..e971509c584a --- /dev/null +++ b/packages/temporal/docs/native-local-driver-branch.md @@ -0,0 +1,99 @@ +# Branch: 2026/08/opencode-native-local-driver + +Base: `2026/08/opencode-temporal` (`8c140c0344`). This branch reworks how a v2 session's +`SessionExecution` runs, in two independent parts, backed by a new deterministic test harness. + +## Why + +The base branch runs BOTH execution modes on one hand-written per-session supervisor +(`execution/workflow-core.ts`, driven for local mode by `execution/local-driver.ts` and for Temporal +mode by `execution/temporal-workflow.ts`). Independent review (Codex, several rounds) found repeated +lifecycle races in that shared supervisor and in the local driver: concurrent successors during +interrupt cleanup, a completion barrier that did not cover resume-started drains, resume/wake lost at +the interrupt boundary, and duplicate provider turns on concurrent resume. Each fix uncovered +another. The pattern was clear: the coordination lifecycle is the dangerous part, and hand-rolling it +kept going wrong, while opencode already had a proven, tested implementation of exactly those +semantics (`SessionRunCoordinator`, `session/run-coordinator.ts`). + +## What changed + +### 1. Local mode runs on the proven SessionRunCoordinator + +`routes.ts` now selects `SessionExecutionLocal` (`execution/local.ts`) for the default mode. It maps +`active` / `wake` / `resume` / `interrupt` onto `SessionRunCoordinator` and drains whole turns with +`SessionRunner.run`, the same lifecycle the v1 server already uses. The hand-written +`local-driver.ts` supervisor path was removed. Local mode is now the well-exercised default and its +correctness comes from reused, tested code rather than a second coordination loop. + +`workflow-core.ts` is therefore Temporal-only. + +### 2. A Temporal test harness, and a Temporal-mode correctness pass + +Standing up a real Temporal test harness (`@temporalio/testing` time-skipping / local server running +the actual `sessionTurn` workflow with a mock activity) immediately caught a blocker and enabled a +correctness pass over the Temporal supervisor. Fixed: + +- **Blocker: the workflow never drained a turn, and a session could serve only one turn.** The SDK's + `condition(fn, timeout)` on `@temporalio/workflow` 1.21 leaks its timer-scope cancellation into the + root scope on resolve, poisoning the next drain. The timed wait now races a no-timeout `condition` + against a bare `sleep` and abandons the loser, cancelling no scope. (This, not the HTTP `steer` + delivery, was the real cause of the "0 activities / turn never runs" symptom.) +- **Interrupt keeps serving.** `interrupt` now cancels only the current turn's child cancellation + scope; the long-lived workflow keeps serving, so a wake/resume racing the interrupt drives a fresh + turn on the same workflow instead of being lost to a doomed one. +- **resume joins.** All drains route through one in-flight promise; a resume joins it instead of + queueing a second forced drain (no duplicate provider turns / tool side effects). +- **Explicit start intent.** `sessionTurn(sessionID, startWithWake)`; resume-with-start passes + `[id, false]`, so a fresh resume does exactly one drain; the flag is carried across + `continueAsNew`. +- **continue-as-new** counts every drain, gates on `allHandlersFinished()`, and a resume-driven + rollover carries no spurious wake. +- **Real (root) workflow cancellation** is detected via the root scope and stops the supervisor, + never keeps serving or continue-as-news. +- **Event-log owner token** is now unique per activity execution (`runId:activityId:attempt`), so a + zombie attempt from an earlier step can no longer re-authorize past the fence. +- **Interrupt delivery failure** is surfaced as a defect instead of being swallowed as success. + +Full status and the remaining (non-correctness) follow-ups are in +[temporal-supervisor-followups.md](./temporal-supervisor-followups.md). + +## Commits + +1. Run local sessions on the proven SessionRunCoordinator. +2. Add a Temporal test harness and fix a condition-cancellation blocker. +3. Fix the event-log owner token to be unique per activity execution. +4. Surface genuine interrupt-delivery failures instead of swallowing them. +5. Count all drains toward continue-as-new, including resume-driven ones. +6. Document the remaining Temporal supervisor coordination follow-ups. +7. Redesign the Temporal supervisor: interrupt keeps serving, resume joins. +8. Test the supervisor redesign: fake-runtime + real-Temporal harness. +9. Update the Temporal supervisor follow-ups doc: deep items now fixed. + +## Verification + +- `bun typecheck` (core + server): clean. +- Fake-runtime + regression suites: green (supervisor join/interrupt/rollover/root-cancel, owner + token, interrupt classification, local coordinator integration, run-coordinator, and the + session-runner suites). +- Real Temporal harness (run each file on its own; two native Temporal servers in one bun process + segfault the runtime): + - `temporal-harness-smoke.test.ts`: a wake drives a turn, then the workflow idle-retires. + - `temporal-harness-interrupt.test.ts`: an interrupt cancels the current turn; a later wake runs a + second turn on the same workflow. + - `temporal-harness-multiturn.test.ts`: turn 1 completes, the supervisor parks in the idle wait + (asserted via a `TimerStarted` history event), then a wake drives turn 2. +- Live: with the blocker fixed, a real `gpt-5-mini` turn completes end-to-end against a dev server in + Temporal mode (previously it recorded zero activities). + +Each substantive change was reviewed with Codex (twice per round); the redesign went through several +review-and-fix rounds until both runs returned no remaining correctness must-fix for a fresh +deployment. + +## Known limitations + +- **Rolling deploys need Temporal versioning/patching or draining old executions**, because the + Temporal workflow's command sequence changed. Fresh deployments are unaffected. See the follow-ups + doc. +- The full HTTP-stack end-to-end (create + prompt via `opencode serve`) has an unrelated `steer` + prompt-delivery quirk in this checkout that predates the branch; it does not affect the automated + tests, which drive the engine directly. From c52b9676d8b2749f0fa88562d77af6cb425da2c5 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Fri, 14 Aug 2026 23:06:54 -0700 Subject: [PATCH 088/103] Correct the branch doc: default prompt delivery is steer and works, not a quirk. The 0-activities symptom was the condition-timeout scope leak, not the delivery mode. A default (steer) prompt drives a turn to completion once the leak is fixed. --- packages/temporal/docs/native-local-driver-branch.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/temporal/docs/native-local-driver-branch.md b/packages/temporal/docs/native-local-driver-branch.md index e971509c584a..3df87f6f8efe 100644 --- a/packages/temporal/docs/native-local-driver-branch.md +++ b/packages/temporal/docs/native-local-driver-branch.md @@ -94,6 +94,10 @@ deployment. - **Rolling deploys need Temporal versioning/patching or draining old executions**, because the Temporal workflow's command sequence changed. Fresh deployments are unaffected. See the follow-ups doc. -- The full HTTP-stack end-to-end (create + prompt via `opencode serve`) has an unrelated `steer` - prompt-delivery quirk in this checkout that predates the branch; it does not affect the automated - tests, which drive the engine directly. +There is NO "steer prompt delivery" problem, contrary to an earlier mistaken note. `POST +/api/session/:id/prompt` without an explicit `delivery` defaults to `"steer"` (`session.ts:366`); +`steer` is a valid delivery that `SessionRunner.run` handles (`llm.ts:425`, it interjects into / leads +the current turn, versus `queue` which waits for the next turn). The "0 activities / turn never runs" +symptom seen end-to-end was entirely the `condition(fn, timeout)` leak described above, not the +delivery mode. With that fixed, a default (`steer`) prompt drives a turn to completion in Temporal +mode (verified live: assistant reply recorded, one `runTurnStep` activity). From 82e1b69ca5e9cf001b30a3342987d0831337cd18 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 15 Aug 2026 02:03:42 -0700 Subject: [PATCH 089/103] Carried the idle override and the contract suite through the redesign. The supervisor redesign reverted the OPENCODE_SESSION_IDLE_TIMEOUT forwarding and removed the file the contract lib typed against. The override rides as a third workflow argument now (continue-as-new keeps it), the lib types against the Temporal node, and the interrupt scenario expects idle retirement since an interrupted supervisor keeps serving. --- bun.lock | 12 +-- packages/cli/bin/lildax.cjs | 0 .../session/execution/temporal-workflow.ts | 22 ++++- .../core/src/session/execution/temporal.ts | 8 +- .../test/lib/session-execution-contract.ts | 85 +++++++++++-------- packages/temporal/README.md | 4 +- 6 files changed, 81 insertions(+), 50 deletions(-) mode change 100644 => 100755 packages/cli/bin/lildax.cjs diff --git a/bun.lock b/bun.lock index 64542ce239cf..8433f7eab602 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "opencode", @@ -947,13 +948,6 @@ "vite": "catalog:", }, }, - "packages/temporal": { - "name": "@opencode-ai/temporal", - "version": "0.0.0", - "dependencies": { - "@temporalio/client": "^1.11.0", - }, - }, "packages/tui": { "name": "@opencode-ai/tui", "version": "1.18.18", @@ -2081,8 +2075,6 @@ "@opencode-ai/storybook": ["@opencode-ai/storybook@workspace:packages/storybook"], - "@opencode-ai/temporal": ["@opencode-ai/temporal@workspace:packages/temporal"], - "@opencode-ai/tui": ["@opencode-ai/tui@workspace:packages/tui"], "@opencode-ai/ui": ["@opencode-ai/ui@workspace:packages/ui"], @@ -4015,7 +4007,7 @@ "get-tsconfig": ["get-tsconfig@4.14.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA=="], - "ghostty-web": ["ghostty-web@github:anomalyco/ghostty-web#83c0a07", {}, "anomalyco-ghostty-web-83c0a07"], + "ghostty-web": ["ghostty-web@github:anomalyco/ghostty-web#83c0a07", {}, "anomalyco-ghostty-web-83c0a07", "sha512-Lf2v1agHkVUpMpHBWWuCZrhOEmcwwin5/Hboc9rZwQ7/CKkIh5rU1r1CvfLlhkMoFv+ed8z52RZ8hkzGZZj3MQ=="], "giget": ["giget@2.0.0", "", { "dependencies": { "citty": "^0.1.6", "consola": "^3.4.0", "defu": "^6.1.4", "node-fetch-native": "^1.6.6", "nypm": "^0.6.0", "pathe": "^2.0.3" }, "bin": { "giget": "dist/cli.mjs" } }, "sha512-L5bGsVkxJbJgdnwyuheIunkGatUF/zssUoxxjACCseZYAVbaqdh9Tsmmlkl8vYan09H7sbvKt4pS8GqKLBrEzA=="], diff --git a/packages/cli/bin/lildax.cjs b/packages/cli/bin/lildax.cjs old mode 100644 new mode 100755 diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/core/src/session/execution/temporal-workflow.ts index a93fa0a2b546..9ffe0b18639c 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/core/src/session/execution/temporal-workflow.ts @@ -101,7 +101,25 @@ let rootScope: CancellationScope | undefined const workflows = makeWorkflows(runtime) -export async function sessionTurn(sessionID: string, startWithWake: boolean = true): Promise { +// The sandbox cannot read env, so the idle override arrives as a workflow argument (the client +// reads the same variable local mode honors) and a continue-as-new run keeps it. +export async function sessionTurn( + sessionID: string, + startWithWake: boolean = true, + idleTimeout?: string, +): Promise { rootScope = CancellationScope.current() - return workflows.sessionTurn(sessionID, startWithWake) + if (!idleTimeout) return workflows.sessionTurn(sessionID, startWithWake) + return makeWorkflows( + { + ...runtime, + continueAsNew: (id, wake) => + continueAsNew<(id: string, startWithWake: boolean, idleTimeout?: string) => Promise>( + id, + wake, + idleTimeout, + ), + }, + { idleTimeout }, + ).sessionTurn(sessionID, startWithWake) } diff --git a/packages/core/src/session/execution/temporal.ts b/packages/core/src/session/execution/temporal.ts index b20783a1d488..cb4f5104d168 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/core/src/session/execution/temporal.ts @@ -66,6 +66,10 @@ const layer = Layer.effect( // The app context the local drain runs in: providing it, then the per-location layer, supplies // SessionRunner and all of its dependencies. const ctx = yield* Effect.context() + // Same knob local mode honors. The workflow sandbox cannot read env, so the client forwards the + // override as a workflow argument. Read at layer build (not module load) so tests can set it + // before constructing the layer. + const IDLE_TIMEOUT = process.env.OPENCODE_SESSION_IDLE_TIMEOUT const events = yield* EventV2.Service const worktrees = yield* WorktreeMaterializer.Service @@ -138,7 +142,7 @@ const layer = Layer.effect( client.workflow.signalWithStart(WORKFLOW_TYPE, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), - args: [id], + args: [id, true, IDLE_TIMEOUT], signal: WF.wake, signalArgs: [], }), @@ -189,7 +193,7 @@ const layer = Layer.effect( // startWithWake=false: a fresh resume-with-start must not manufacture a wake drain; // its forced drain comes from the resume update. Ignored when USE_EXISTING joins a // running workflow (which keeps its own state). - args: [id, false], + args: [id, false, IDLE_TIMEOUT], workflowIdConflictPolicy: "USE_EXISTING", }) return client.workflow.executeUpdateWithStart(WF.resume, { diff --git a/packages/core/test/lib/session-execution-contract.ts b/packages/core/test/lib/session-execution-contract.ts index 5483afbda05c..1e48f3572b3c 100644 --- a/packages/core/test/lib/session-execution-contract.ts +++ b/packages/core/test/lib/session-execution-contract.ts @@ -1,9 +1,10 @@ -// The SessionExecution driver-contract suite, parameterized over the driver factory. Each driver -// (the in-process native coordinator, the Temporal workflow) registers the SAME scenarios through -// runContract; this is what guarantees the modes agree now that they share only the drain, not one -// loop. The contract: wake drives a turn to settlement then the idle coordinator retires, resume -// forces a healthy turn to completion, resume surfaces the exact tagged RunError (through the same -// encode/decode path the Temporal boundary uses), and interrupt cancels an in-flight turn. +// The SessionExecution driver-contract suite, parameterized over the driver factory. The Temporal +// contract run (session-execution-temporal-contract.test.ts) registers these scenarios against real +// workflows; local mode's SessionRunCoordinator wiring asserts the same verbs in its own suite +// (session-execution-local.test.ts). The contract: wake drives a turn to settlement then the idle +// executor retires, resume forces a healthy turn to completion, resume surfaces the exact tagged +// RunError (through the same encode/decode path the Temporal boundary uses), and interrupt cancels +// an in-flight turn and the session eventually leaves the active set. import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" import { LLMEvent } from "@opencode-ai/llm" import { Database } from "@opencode-ai/core/database/database" @@ -21,7 +22,7 @@ import { Snapshot } from "@opencode-ai/core/snapshot" import { SessionEvent } from "@opencode-ai/core/session/event" import { SessionProjector } from "@opencode-ai/core/session/projector" import { SessionExecution } from "@opencode-ai/core/session/execution" -import { SessionExecutionLocalDriver } from "@opencode-ai/core/session/execution/local-driver" +import type { SessionExecutionTemporal } from "@opencode-ai/core/session/execution/temporal" import { SessionRunnerModel, ModelNotSelectedError } from "@opencode-ai/core/session/runner/model" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { SessionTable } from "@opencode-ai/core/session/sql" @@ -77,7 +78,7 @@ const countingModel = () => { // serve process builds it), with the model/LLM mocked. Any SessionExecution node with the standard // dependency set (the local coordinator, the Temporal driver) plugs in here. export const makeExecutionFor = - (node: typeof SessionExecutionLocalDriver.node) => + (node: typeof SessionExecutionTemporal.node) => (stream: LLMClientShape["stream"], models = okModels) => AppNodeBuilder.build(node, [ [LayerNodePlatform.llmClient, mockClient(stream)], @@ -146,24 +147,37 @@ const until = (read: Effect.Effect, predicate: (value: A) => boolean } }) +// The idle override is read at the executor's layer build (the Temporal client forwards it as a +// workflow argument), so it must be set before makeExec's layer is built and restored afterwards so +// later layer builds in this process get the real default. +const withIdleOverride = (body: Effect.Effect) => + Effect.gen(function* () { + const previous = process.env.OPENCODE_SESSION_IDLE_TIMEOUT + process.env.OPENCODE_SESSION_IDLE_TIMEOUT = "2 seconds" + try { + return yield* body + } finally { + if (previous === undefined) delete process.env.OPENCODE_SESSION_IDLE_TIMEOUT + else process.env.OPENCODE_SESSION_IDLE_TIMEOUT = previous + } + }) + // The SessionExecution contract, parameterized over the driver factory. `makeExec` builds a // SessionExecution graph the same way serve does, with the model/LLM mocked. Running the identical // suite against a second factory is how the two modes are held to one behavior now that they no // longer share a single coordination loop -- only the drain. export const runContract = (label: string, makeExec: ReturnType) => { - const slug = label.replace(/[^a-z0-9]+/gi, "_") + // The nonce keeps workflow ids unique across runs: the Temporal driver derives durable workflow + // ids from session ids, and on a shared dev server USE_EXISTING would otherwise route this run's + // updates to a leftover workflow from an earlier one, parked on a task queue nobody polls. + const slug = `${label.replace(/[^a-z0-9]+/gi, "_")}_${crypto.randomUUID().slice(0, 8)}` describe(`SessionExecution contract: ${label}`, () => { { const { requests, stream } = countingModel() const sessionID = SessionV2.ID.make(`ses_${slug}_wake`) - it.live("wake drives a turn to settlement, then the idle coordinator retires", () => - Effect.gen(function* () { - // Both drivers read this at layer build: the local coordinator directly, the Temporal - // client to forward it as a workflow argument. Restore it so later layer builds in this - // process get the real default. - const previousIdle = process.env.OPENCODE_SESSION_IDLE_TIMEOUT - process.env.OPENCODE_SESSION_IDLE_TIMEOUT = "2 seconds" - try { + it.live("wake drives a turn to settlement, then the idle executor retires", () => + withIdleOverride( + Effect.gen(function* () { yield* seedSession(sessionID) yield* seedPrompt(sessionID) const exec = Context.get(yield* Layer.build(makeExec(stream)), SessionExecution.Service) @@ -175,13 +189,10 @@ export const runContract = (label: string, makeExec: ReturnType active.has(sessionID)) - // Idle self-termination: the coordinator retires without an interrupt. + // Idle self-termination: the executor retires without an interrupt. yield* until(exec.active, (active) => !active.has(sessionID)) - } finally { - if (previousIdle === undefined) delete process.env.OPENCODE_SESSION_IDLE_TIMEOUT - else process.env.OPENCODE_SESSION_IDLE_TIMEOUT = previousIdle - } - }), + }), + ), ) } @@ -225,18 +236,22 @@ export const runContract = (label: string, makeExec: ReturnType - Effect.gen(function* () { - yield* seedSession(sessionID) - yield* seedPrompt(sessionID) - // A model that never answers: the turn hangs until interrupted. - const exec = Context.get(yield* Layer.build(makeExec(() => Stream.never)), SessionExecution.Service) - yield* exec.wake(sessionID) - yield* Effect.sleep(200) - yield* until(exec.active, (active) => active.has(sessionID)) - yield* exec.interrupt(sessionID) - yield* until(exec.active, (active) => !active.has(sessionID)) - }), + it.live("interrupt cancels an in-flight turn and the session leaves the active set", () => + withIdleOverride( + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + // A model that never answers: the turn hangs until interrupted. + const exec = Context.get(yield* Layer.build(makeExec(() => Stream.never)), SessionExecution.Service) + yield* exec.wake(sessionID) + yield* Effect.sleep(200) + yield* until(exec.active, (active) => active.has(sessionID)) + yield* exec.interrupt(sessionID) + // The Temporal supervisor keeps serving after an interrupt (a racing wake/resume must + // not be lost); with nothing else queued it leaves the active set via idle retirement. + yield* until(exec.active, (active) => !active.has(sessionID)) + }), + ), ) } }) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 8f8681bb03e8..bd8969da021e 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -102,7 +102,9 @@ coordinator is the same one the v1 server uses and has direct lifecycle tests (`session-run-coordinator.test.ts`), so the default path reuses well-exercised code rather than a second hand-written loop. The local integration wiring is covered by `session-execution-local.test.ts`, and Temporal crash recovery by the crash test (in the -stacked scripts PR). Verified: +stacked scripts PR). The same wake/resume/interrupt contract also runs against the Temporal driver +through real workflows via the opt-in suite (`session-execution-temporal-contract.test.ts`). +Verified: a create-then-read-then-reply turn recorded three `runTurnStep` activities under a `sessionTurn` workflow and completed. (Earlier whole-turn-per-activity, stock-coordinator, and shared-supervisor local modes were folded away in favor of the coordinator for local.) From 1aee29e6aa4f4afd816c28c1ca09e53a0be8cb01 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 15 Aug 2026 15:17:44 -0700 Subject: [PATCH 090/103] Promoted the conformance suite and test harness into core's source. The suite is the executable definition of the executor seam, so it lives where any driver package can import it; the local run is now a one-line conformance call. The old test-lib path re-exports the moved harness so the rest of the test tree keeps its import. --- .../session/execution/conformance.ts} | 30 ++- packages/core/src/testing/effect.ts | 53 +++++ packages/core/test/lib/effect.ts | 56 +---- .../core/test/session-execution-local.test.ts | 209 +----------------- ...ession-execution-temporal-contract.test.ts | 2 +- 5 files changed, 81 insertions(+), 269 deletions(-) rename packages/core/{test/lib/session-execution-contract.ts => src/session/execution/conformance.ts} (90%) create mode 100644 packages/core/src/testing/effect.ts diff --git a/packages/core/test/lib/session-execution-contract.ts b/packages/core/src/session/execution/conformance.ts similarity index 90% rename from packages/core/test/lib/session-execution-contract.ts rename to packages/core/src/session/execution/conformance.ts index 1e48f3572b3c..ce9e01ed8b3c 100644 --- a/packages/core/test/lib/session-execution-contract.ts +++ b/packages/core/src/session/execution/conformance.ts @@ -1,10 +1,10 @@ -// The SessionExecution driver-contract suite, parameterized over the driver factory. The Temporal -// contract run (session-execution-temporal-contract.test.ts) registers these scenarios against real -// workflows; local mode's SessionRunCoordinator wiring asserts the same verbs in its own suite -// (session-execution-local.test.ts). The contract: wake drives a turn to settlement then the idle -// executor retires, resume forces a healthy turn to completion, resume surfaces the exact tagged -// RunError (through the same encode/decode path the Temporal boundary uses), and interrupt cancels -// an in-flight turn and the session eventually leaves the active set. +// The SessionExecution conformance suite: the executable definition of what an executor must do. +// Any driver behind the SessionExecution seam registers the same scenarios through runContract; the +// built-in local executor runs it in core's tests, and the Temporal executor runs it against real +// workflows. The contract: wake drives a turn to settlement then the idle executor retires, resume +// forces a healthy turn to completion, resume surfaces the exact tagged RunError (through the same +// encode/decode path a process boundary uses), and interrupt cancels an in-flight turn and the +// session eventually leaves the active set. import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" import { LLMEvent } from "@opencode-ai/llm" import { Database } from "@opencode-ai/core/database/database" @@ -22,7 +22,7 @@ import { Snapshot } from "@opencode-ai/core/snapshot" import { SessionEvent } from "@opencode-ai/core/session/event" import { SessionProjector } from "@opencode-ai/core/session/projector" import { SessionExecution } from "@opencode-ai/core/session/execution" -import type { SessionExecutionTemporal } from "@opencode-ai/core/session/execution/temporal" +import { SessionExecutionLocal } from "@opencode-ai/core/session/execution/local" import { SessionRunnerModel, ModelNotSelectedError } from "@opencode-ai/core/session/runner/model" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { SessionTable } from "@opencode-ai/core/session/sql" @@ -40,7 +40,7 @@ import { describe, expect } from "bun:test" import { realpathSync } from "node:fs" import { tmpdir } from "node:os" import { Cause, Context, DateTime, Effect, Exit, Layer, Stream } from "effect" -import { testEffect } from "./effect" +import { testEffect } from "../../testing/effect" // The per-location service build resolves the session directory on disk, so it must exist. const WORKSPACE = AbsolutePath.make(realpathSync(tmpdir())) @@ -78,7 +78,7 @@ const countingModel = () => { // serve process builds it), with the model/LLM mocked. Any SessionExecution node with the standard // dependency set (the local coordinator, the Temporal driver) plugs in here. export const makeExecutionFor = - (node: typeof SessionExecutionTemporal.node) => + (node: typeof SessionExecutionLocal.node) => (stream: LLMClientShape["stream"], models = okModels) => AppNodeBuilder.build(node, [ [LayerNodePlatform.llmClient, mockClient(stream)], @@ -188,11 +188,14 @@ export const runContract = (label: string, makeExec: ReturnType active.has(sessionID)) - // Idle self-termination: the executor retires without an interrupt. + // Idle self-termination: the executor retires without an interrupt. How long a settled + // session lingers in `active` is the executor's business (the local coordinator retires + // on settlement, the Temporal workflow serves until its idle timeout); the contract only + // demands it eventually leaves. yield* until(exec.active, (active) => !active.has(sessionID)) }), ), + 60000, ) } @@ -213,6 +216,7 @@ export const runContract = (label: string, makeExec: ReturnType message.type === "assistant") expect(assistant?.type === "assistant" && Boolean(assistant.time.completed)).toBe(true) }), + 60000, ) } @@ -231,6 +235,7 @@ export const runContract = (label: string, makeExec: ReturnType !active.has(sessionID)) }), ), + 60000, ) } }) diff --git a/packages/core/src/testing/effect.ts b/packages/core/src/testing/effect.ts new file mode 100644 index 000000000000..131ec5cc6bc2 --- /dev/null +++ b/packages/core/src/testing/effect.ts @@ -0,0 +1,53 @@ +import { test, type TestOptions } from "bun:test" +import { Cause, Effect, Exit, Layer } from "effect" +import type * as Scope from "effect/Scope" +import * as TestClock from "effect/testing/TestClock" +import * as TestConsole from "effect/testing/TestConsole" + +type Body = Effect.Effect | (() => Effect.Effect) + +const body = (value: Body) => Effect.suspend(() => (typeof value === "function" ? value() : value)) + +const run = (value: Body, layer: Layer.Layer) => + Effect.gen(function* () { + const exit = yield* body(value).pipe(Effect.scoped, Effect.provide(layer), Effect.exit) + if (Exit.isFailure(exit)) { + for (const err of Cause.prettyErrors(exit.cause)) { + yield* Effect.logError(err) + } + } + return yield* exit + }).pipe(Effect.runPromise) + +const make = (testLayer: Layer.Layer, liveLayer: Layer.Layer) => { + const effect = (name: string, value: Body, opts?: number | TestOptions) => + test(name, () => run(value, testLayer), opts) + + effect.only = (name: string, value: Body, opts?: number | TestOptions) => + test.only(name, () => run(value, testLayer), opts) + + effect.skip = (name: string, value: Body, opts?: number | TestOptions) => + test.skip(name, () => run(value, testLayer), opts) + + const live = (name: string, value: Body, opts?: number | TestOptions) => + test(name, () => run(value, liveLayer), opts) + + live.only = (name: string, value: Body, opts?: number | TestOptions) => + test.only(name, () => run(value, liveLayer), opts) + + live.skip = (name: string, value: Body, opts?: number | TestOptions) => + test.skip(name, () => run(value, liveLayer), opts) + + return { effect, live } +} + +// Test environment with TestClock and TestConsole +const testEnv = Layer.mergeAll(TestConsole.layer, TestClock.layer()) + +// Live environment - uses real clock, but keeps TestConsole for output capture +const liveEnv = TestConsole.layer + +export const it = make(testEnv, liveEnv) + +export const testEffect = (layer: Layer.Layer) => + make(Layer.provideMerge(layer, testEnv), Layer.provideMerge(layer, liveEnv)) diff --git a/packages/core/test/lib/effect.ts b/packages/core/test/lib/effect.ts index 131ec5cc6bc2..ad94c5fe615d 100644 --- a/packages/core/test/lib/effect.ts +++ b/packages/core/test/lib/effect.ts @@ -1,53 +1,3 @@ -import { test, type TestOptions } from "bun:test" -import { Cause, Effect, Exit, Layer } from "effect" -import type * as Scope from "effect/Scope" -import * as TestClock from "effect/testing/TestClock" -import * as TestConsole from "effect/testing/TestConsole" - -type Body = Effect.Effect | (() => Effect.Effect) - -const body = (value: Body) => Effect.suspend(() => (typeof value === "function" ? value() : value)) - -const run = (value: Body, layer: Layer.Layer) => - Effect.gen(function* () { - const exit = yield* body(value).pipe(Effect.scoped, Effect.provide(layer), Effect.exit) - if (Exit.isFailure(exit)) { - for (const err of Cause.prettyErrors(exit.cause)) { - yield* Effect.logError(err) - } - } - return yield* exit - }).pipe(Effect.runPromise) - -const make = (testLayer: Layer.Layer, liveLayer: Layer.Layer) => { - const effect = (name: string, value: Body, opts?: number | TestOptions) => - test(name, () => run(value, testLayer), opts) - - effect.only = (name: string, value: Body, opts?: number | TestOptions) => - test.only(name, () => run(value, testLayer), opts) - - effect.skip = (name: string, value: Body, opts?: number | TestOptions) => - test.skip(name, () => run(value, testLayer), opts) - - const live = (name: string, value: Body, opts?: number | TestOptions) => - test(name, () => run(value, liveLayer), opts) - - live.only = (name: string, value: Body, opts?: number | TestOptions) => - test.only(name, () => run(value, liveLayer), opts) - - live.skip = (name: string, value: Body, opts?: number | TestOptions) => - test.skip(name, () => run(value, liveLayer), opts) - - return { effect, live } -} - -// Test environment with TestClock and TestConsole -const testEnv = Layer.mergeAll(TestConsole.layer, TestClock.layer()) - -// Live environment - uses real clock, but keeps TestConsole for output capture -const liveEnv = TestConsole.layer - -export const it = make(testEnv, liveEnv) - -export const testEffect = (layer: Layer.Layer) => - make(Layer.provideMerge(layer, testEnv), Layer.provideMerge(layer, liveEnv)) +// Re-export so the test tree keeps its historical import path; the implementation lives in src so +// packages outside core (and the conformance suite) can use the same harness. +export * from "@opencode-ai/core/testing/effect" diff --git a/packages/core/test/session-execution-local.test.ts b/packages/core/test/session-execution-local.test.ts index 7e0f57b35dee..1b5930fe2e5e 100644 --- a/packages/core/test/session-execution-local.test.ts +++ b/packages/core/test/session-execution-local.test.ts @@ -1,205 +1,8 @@ -// Integration tests for the in-process SessionExecution (execution/local.ts), which delegates the -// wake/resume/interrupt lifecycle to the proven SessionRunCoordinator and drains with SessionRunner -// over the shared event log -- no Temporal, no server. The contract: wake drives a turn to -// settlement and the coordinator retires it, resume surfaces the RunError, and interrupt cancels an -// in-flight turn. (The coordinator's own lifecycle races are covered by session-run-coordinator.test.ts.) -import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" -import { LLMEvent } from "@opencode-ai/llm" -import { Database } from "@opencode-ai/core/database/database" -import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" -import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" -import { LayerNode } from "@opencode-ai/core/effect/layer-node" -import { EventV2 } from "@opencode-ai/core/event" -import { PermissionV2 } from "@opencode-ai/core/permission" -import { Config } from "@opencode-ai/core/config" -import { Project } from "@opencode-ai/core/project" -import { ProjectTable } from "@opencode-ai/core/project/sql" -import { AbsolutePath } from "@opencode-ai/core/schema" -import { SessionV2 } from "@opencode-ai/core/session" -import { Snapshot } from "@opencode-ai/core/snapshot" -import { SessionEvent } from "@opencode-ai/core/session/event" -import { SessionProjector } from "@opencode-ai/core/session/projector" -import { SessionExecution } from "@opencode-ai/core/session/execution" +// The conformance suite run against the built-in local executor: SessionRunCoordinator behind the +// SessionExecution seam, whole turns through SessionRunner.run, no server. The same scenarios run +// against the Temporal executor in packages/temporal; the shared suite is what holds any executor +// to one behavior. import { SessionExecutionLocal } from "@opencode-ai/core/session/execution/local" -import { SessionRunnerModel, ModelNotSelectedError } from "@opencode-ai/core/session/runner/model" -import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" -import { SessionTable } from "@opencode-ai/core/session/sql" -import { SessionStore } from "@opencode-ai/core/session/store" -import { SessionMessage } from "@opencode-ai/core/session/message" -import { Prompt } from "@opencode-ai/core/session/prompt" -import { Location } from "@opencode-ai/core/location" -import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" -import { SystemContext } from "@opencode-ai/core/system-context" -import { SkillGuidance } from "@opencode-ai/core/skill/guidance" -import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" -import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" -import { Auth } from "@opencode-ai/llm/route" -import { describe, expect } from "bun:test" -import { realpathSync } from "node:fs" -import { tmpdir } from "node:os" -import { Cause, Context, DateTime, Effect, Exit, Layer, Stream } from "effect" -import { testEffect } from "./lib/effect" +import { makeExecutionFor, runContract } from "@opencode-ai/core/session/execution/conformance" -// The per-location service build resolves the session directory on disk, so it must exist. -const WORKSPACE = AbsolutePath.make(realpathSync(tmpdir())) - -const model = OpenAIChat.route - .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) - .model({ id: "gpt-4o-mini" }) -const okModels = SessionRunnerModel.layerWith(() => Effect.succeed(model)) -const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) -const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) -const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) -const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) -const permission = Layer.mock(PermissionV2.Service, {}) - -const mockClient = (stream: LLMClientShape["stream"]) => - Layer.succeed( - LLMClient.Service, - LLMClient.Service.of({ - prepare: () => Effect.die("unused"), - generate: () => Effect.die("unused"), - stream, - }), - ) - -const countingModel = () => { - const requests: number[] = [] - const stream: LLMClientShape["stream"] = () => { - requests.push(1) - return Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })]) - } - return { requests, stream } -} - -// The executor under test, built as its own graph over the shared database file (the same way the -// serve process builds it), with the model/LLM mocked. -const makeExecution = (stream: LLMClientShape["stream"], models = okModels) => - AppNodeBuilder.build(SessionExecutionLocal.node, [ - [LayerNodePlatform.llmClient, mockClient(stream)], - [PermissionV2.node, permission], - [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], - [SessionRunnerModel.node, models], - [SystemContextRegistry.node, systemContext], - [Location.node, Location.boundNode({ directory: WORKSPACE })], - [SkillGuidance.node, skillGuidance], - [ReferenceGuidance.node, referenceGuidance], - [Config.node, config], - [Snapshot.node, Snapshot.noopLayer], - ]) - -// Reads and seeds go through a separate graph sharing the same database file. -const it = testEffect( - AppNodeBuilder.build(LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node])), -) - -const seedSession = (sessionID: SessionV2.ID) => - Effect.gen(function* () { - const { db } = yield* Database.Service - yield* db - .insert(ProjectTable) - .values({ id: Project.ID.global, worktree: WORKSPACE, sandboxes: [] }) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie) - yield* db - .insert(SessionTable) - .values({ - id: sessionID, - project_id: Project.ID.global, - slug: "t", - directory: WORKSPACE, - title: "t", - version: "t", - }) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie) - }) - -const seedPrompt = (sessionID: SessionV2.ID) => - Effect.gen(function* () { - const events = yield* EventV2.Service - yield* events.publish(SessionEvent.Prompted, { - sessionID, - timestamp: yield* DateTime.now, - messageID: SessionMessage.ID.create(), - prompt: Prompt.make({ text: "do the thing" }), - delivery: "queue", - }) - }) - -const until = (read: Effect.Effect, predicate: (value: A) => boolean, timeoutMs = 8000) => - Effect.gen(function* () { - const deadline = Date.now() + timeoutMs - for (;;) { - const value = yield* read - if (predicate(value)) return value - if (Date.now() > deadline) throw new Error("condition not reached in time") - yield* Effect.sleep(50) - } - }) - -describe("SessionExecution local (coordinator)", () => { - { - const { requests, stream } = countingModel() - const sessionID = SessionV2.ID.make("ses_local_wake") - it.live("wake drives a turn to settlement, then the coordinator retires it", () => - Effect.gen(function* () { - yield* seedSession(sessionID) - yield* seedPrompt(sessionID) - const exec = Context.get(yield* Layer.build(makeExecution(stream)), SessionExecution.Service) - yield* exec.wake(sessionID) - const store = yield* SessionStore.Service - yield* until(store.context(sessionID), (context) => { - const assistant = context.findLast((message) => message.type === "assistant") - return assistant?.type === "assistant" && Boolean(assistant.time.completed) - }) - expect(requests).toHaveLength(1) - // The coordinator holds no idle timer: once the drain settles with no follow-up, the entry - // is dropped, so the session leaves the active set on its own. - yield* until(exec.active, (active) => !active.has(sessionID)) - }), - ) - } - - { - const sessionID = SessionV2.ID.make("ses_local_error") - const failingModels = SessionRunnerModel.layerWith(() => - Effect.fail(new ModelNotSelectedError({ sessionID })), - ) - it.live("resume surfaces the tagged RunError to the caller", () => - Effect.gen(function* () { - yield* seedSession(sessionID) - const { stream } = countingModel() - const exec = Context.get(yield* Layer.build(makeExecution(stream, failingModels)), SessionExecution.Service) - const exit = yield* exec.resume(sessionID).pipe(Effect.exit) - expect(Exit.isFailure(exit)).toBe(true) - const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined - // resume = coordinator.run: the run's error propagates natively as the tagged RunError, no - // encode/decode boundary in local mode. - expect(error).toBeInstanceOf(ModelNotSelectedError) - }), - ) - } - - { - const sessionID = SessionV2.ID.make("ses_local_interrupt") - it.live("interrupt cancels an in-flight turn and the coordinator retires it", () => - Effect.gen(function* () { - yield* seedSession(sessionID) - yield* seedPrompt(sessionID) - // A model that never answers: the turn hangs until interrupted. - const exec = Context.get( - yield* Layer.build(makeExecution(() => Stream.never)), - SessionExecution.Service, - ) - yield* exec.wake(sessionID) - yield* Effect.sleep(200) - expect((yield* exec.active).has(sessionID)).toBe(true) - yield* exec.interrupt(sessionID) - yield* until(exec.active, (active) => !active.has(sessionID), 4000) - }), - ) - } -}) +runContract("local executor", makeExecutionFor(SessionExecutionLocal.node)) diff --git a/packages/core/test/session-execution-temporal-contract.test.ts b/packages/core/test/session-execution-temporal-contract.test.ts index a8f11d35923b..2ade9c351e32 100644 --- a/packages/core/test/session-execution-temporal-contract.test.ts +++ b/packages/core/test/session-execution-temporal-contract.test.ts @@ -9,7 +9,7 @@ // OPENCODE_CONTRACT_TEMPORAL=1 bun test --timeout 120000 test/session-execution-temporal-contract.test.ts // // Without the opt-in the file registers nothing, so a plain `bun test` stays server-free. -import { makeExecutionFor, runContract } from "./lib/session-execution-contract" +import { makeExecutionFor, runContract } from "@opencode-ai/core/session/execution/conformance" if (process.env.OPENCODE_CONTRACT_TEMPORAL === "1") { // One task queue per run: a stale worker from an earlier run against the same dev server would From 621951bee204f7abe50b2630ea85dc5549c779f5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 15 Aug 2026 15:23:36 -0700 Subject: [PATCH 091/103] Moved the Temporal executor into its own package. Core now carries only the executor seam, the built-in local executor, and the executor-agnostic toolkit (step runner, event fencing, error codec, worktree materializer); everything Temporal, including the per-step activity drain, lives in @opencode-ai/temporal, which the server wires in as one dependency. The step contract types moved into the drain where they belonged. --- bun.lock | 23 ++++++++--- packages/core/package.json | 5 --- packages/server/package.json | 1 + packages/server/src/routes.ts | 2 +- packages/temporal/package.json | 24 +++++++++++ .../src/activities.ts} | 23 ++--------- .../execution => temporal/src}/drain.ts | 41 +++++++++++++------ .../temporal.ts => temporal/src/executor.ts} | 22 +++++----- .../src/supervisor.ts} | 2 +- .../src/workflow.ts} | 4 +- ...ession-execution-temporal-contract.test.ts | 2 +- .../test/session-supervisor-rollover.test.ts | 4 +- .../test/session-supervisor.test.ts | 4 +- .../test/temporal-harness-interrupt.test.ts | 2 +- .../test/temporal-harness-multiturn.test.ts | 2 +- .../test/temporal-harness-smoke.test.ts | 2 +- .../test/temporal-interrupt-classify.test.ts | 2 +- .../test/temporal-owner-token.test.ts | 2 +- packages/temporal/tsconfig.json | 8 ++++ 19 files changed, 109 insertions(+), 66 deletions(-) create mode 100644 packages/temporal/package.json rename packages/{core/src/session/execution/temporal-activities.ts => temporal/src/activities.ts} (79%) rename packages/{core/src/session/execution => temporal/src}/drain.ts (77%) rename packages/{core/src/session/execution/temporal.ts => temporal/src/executor.ts} (94%) rename packages/{core/src/session/execution/workflow-core.ts => temporal/src/supervisor.ts} (99%) rename packages/{core/src/session/execution/temporal-workflow.ts => temporal/src/workflow.ts} (97%) rename packages/{core => temporal}/test/session-execution-temporal-contract.test.ts (97%) rename packages/{core => temporal}/test/session-supervisor-rollover.test.ts (93%) rename packages/{core => temporal}/test/session-supervisor.test.ts (96%) rename packages/{core => temporal}/test/temporal-harness-interrupt.test.ts (96%) rename packages/{core => temporal}/test/temporal-harness-multiturn.test.ts (96%) rename packages/{core => temporal}/test/temporal-harness-smoke.test.ts (95%) rename packages/{core => temporal}/test/temporal-interrupt-classify.test.ts (90%) rename packages/{core => temporal}/test/temporal-owner-token.test.ts (92%) create mode 100644 packages/temporal/tsconfig.json diff --git a/bun.lock b/bun.lock index 8433f7eab602..c22cc13d0e24 100644 --- a/bun.lock +++ b/bun.lock @@ -333,10 +333,6 @@ "@opentelemetry/sdk-trace-base": "2.6.1", "@parcel/watcher": "2.5.1", "@silvia-odwyer/photon-node": "0.3.4", - "@temporalio/activity": "^1.21.0", - "@temporalio/client": "^1.21.0", - "@temporalio/worker": "^1.21.0", - "@temporalio/workflow": "^1.21.0", "ai-gateway-provider": "3.2.0", "bun-pty": "0.4.8", "cross-spawn": "catalog:", @@ -372,7 +368,6 @@ "@parcel/watcher-linux-x64-musl": "2.5.1", "@parcel/watcher-win32-arm64": "2.5.1", "@parcel/watcher-win32-x64": "2.5.1", - "@temporalio/testing": "1.21.1", "@tsconfig/bun": "catalog:", "@types/bun": "catalog:", "@types/cross-spawn": "catalog:", @@ -791,6 +786,7 @@ "dependencies": { "@opencode-ai/core": "workspace:*", "@opencode-ai/protocol": "workspace:*", + "@opencode-ai/temporal": "workspace:*", "drizzle-orm": "catalog:", "effect": "catalog:", }, @@ -948,6 +944,21 @@ "vite": "catalog:", }, }, + "packages/temporal": { + "name": "@opencode-ai/temporal", + "version": "0.0.1", + "dependencies": { + "@opencode-ai/core": "workspace:*", + "@temporalio/activity": "^1.21.0", + "@temporalio/client": "^1.21.0", + "@temporalio/worker": "^1.21.0", + "@temporalio/workflow": "^1.21.0", + "effect": "catalog:", + }, + "devDependencies": { + "@temporalio/testing": "1.21.1", + }, + }, "packages/tui": { "name": "@opencode-ai/tui", "version": "1.18.18", @@ -2075,6 +2086,8 @@ "@opencode-ai/storybook": ["@opencode-ai/storybook@workspace:packages/storybook"], + "@opencode-ai/temporal": ["@opencode-ai/temporal@workspace:packages/temporal"], + "@opencode-ai/tui": ["@opencode-ai/tui@workspace:packages/tui"], "@opencode-ai/ui": ["@opencode-ai/ui@workspace:packages/ui"], diff --git a/packages/core/package.json b/packages/core/package.json index c61512087a96..ca8eac20a713 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -49,7 +49,6 @@ "@parcel/watcher-linux-x64-musl": "2.5.1", "@parcel/watcher-win32-arm64": "2.5.1", "@parcel/watcher-win32-x64": "2.5.1", - "@temporalio/testing": "1.21.1", "@tsconfig/bun": "catalog:", "@types/bun": "catalog:", "@types/cross-spawn": "catalog:", @@ -103,10 +102,6 @@ "@opentelemetry/sdk-trace-base": "2.6.1", "@parcel/watcher": "2.5.1", "@silvia-odwyer/photon-node": "0.3.4", - "@temporalio/activity": "^1.21.0", - "@temporalio/client": "^1.21.0", - "@temporalio/worker": "^1.21.0", - "@temporalio/workflow": "^1.21.0", "ai-gateway-provider": "3.2.0", "bun-pty": "0.4.8", "cross-spawn": "catalog:", diff --git a/packages/server/package.json b/packages/server/package.json index 83eca9036b2a..671cfc774c98 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "@opencode-ai/core": "workspace:*", + "@opencode-ai/temporal": "workspace:*", "@opencode-ai/protocol": "workspace:*", "drizzle-orm": "catalog:", "effect": "catalog:" diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index 5b47fef50009..71b750fb878f 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -10,7 +10,7 @@ import { SessionV2 } from "@opencode-ai/core/session" import { SessionExecution } from "@opencode-ai/core/session/execution" import { LocationServiceMap } from "@opencode-ai/core/location-service-map" import { SessionExecutionLocal } from "@opencode-ai/core/session/execution/local" -import { SessionExecutionTemporal } from "@opencode-ai/core/session/execution/temporal" +import { SessionExecutionTemporal } from "@opencode-ai/temporal/executor" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { HttpRouter, HttpServer } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" diff --git a/packages/temporal/package.json b/packages/temporal/package.json new file mode 100644 index 000000000000..c660ce4c7eca --- /dev/null +++ b/packages/temporal/package.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json.schemastore.org/package", + "name": "@opencode-ai/temporal", + "version": "0.0.1", + "private": true, + "type": "module", + "exports": { + "./*": "./src/*.ts" + }, + "scripts": { + "typecheck": "tsgo --noEmit" + }, + "dependencies": { + "@opencode-ai/core": "workspace:*", + "@temporalio/activity": "^1.21.0", + "@temporalio/client": "^1.21.0", + "@temporalio/worker": "^1.21.0", + "@temporalio/workflow": "^1.21.0", + "effect": "catalog:" + }, + "devDependencies": { + "@temporalio/testing": "1.21.1" + } +} diff --git a/packages/core/src/session/execution/temporal-activities.ts b/packages/temporal/src/activities.ts similarity index 79% rename from packages/core/src/session/execution/temporal-activities.ts rename to packages/temporal/src/activities.ts index f4845cfc42cb..68a37c8d8b03 100644 --- a/packages/core/src/session/execution/temporal-activities.ts +++ b/packages/temporal/src/activities.ts @@ -25,25 +25,10 @@ function ownerToken(): string { return ownerTokenFrom(run, info.activityId, info.attempt) } -// The workflow loops runTurnStep, so each step is its own activity with its own -// retry/timeout/visibility. `promotion` is null (not undefined) so it serializes cleanly. -export interface StepDrainInput { - sessionID: string - step: number - promotion: string | null - first: boolean - force: boolean - // The attempt that owns the event log while this drain runs. Set activity-side (see - // ownerToken), so it stays out of the workflow's deterministic input. - owner?: string -} - -export interface StepDrainResult { - ran: boolean - continue: boolean - step: number - promotion: string | null -} +// The step contract lives with the drain in core; re-exported here so the workflow and its tests +// keep one import site inside this package. +import type { StepDrainInput, StepDrainResult } from "./drain" +export type { StepDrainInput, StepDrainResult } export type StepActivities = { runTurnStep(input: StepDrainInput): Promise diff --git a/packages/core/src/session/execution/drain.ts b/packages/temporal/src/drain.ts similarity index 77% rename from packages/core/src/session/execution/drain.ts rename to packages/temporal/src/drain.ts index eb699f03b3e3..db10397030d0 100644 --- a/packages/core/src/session/execution/drain.ts +++ b/packages/temporal/src/drain.ts @@ -6,18 +6,35 @@ import { Cause, Context, Effect, Exit, type LayerMap } from "effect" import { ApplicationFailure } from "@temporalio/activity" -import type { LocationServiceMap } from "../../location-service-map" -import type { Location } from "../../location" -import type { LocationError, LocationServices } from "../../location-services" -import { EventV2 } from "../../event" -import { WorktreeMaterializer } from "./worktree" -import { SessionRunner } from "../runner" -import { SessionSchema } from "../schema" -import { SessionStore } from "../store" -import { SessionRunDeclinedError } from "../error" -import type { SessionInput } from "../input" -import { encodeRunError } from "./run-error-codec" -import type { StepDrainInput, StepDrainResult } from "./temporal-activities" +import type { LocationServiceMap } from "@opencode-ai/core/location-service-map" +import type { Location } from "@opencode-ai/core/location" +import type { LocationError, LocationServices } from "@opencode-ai/core/location-services" +import { EventV2 } from "@opencode-ai/core/event" +import { WorktreeMaterializer } from "@opencode-ai/core/session/execution/worktree" +import { SessionRunner } from "@opencode-ai/core/session/runner" +import { SessionSchema } from "@opencode-ai/core/session/schema" +import { SessionStore } from "@opencode-ai/core/session/store" +import { SessionRunDeclinedError } from "@opencode-ai/core/session/error" +import type { SessionInput } from "@opencode-ai/core/session/input" +import { encodeRunError } from "@opencode-ai/core/session/execution/run-error-codec" +// One step of a turn, as any executor drives it. `promotion` is null (not undefined) so it +// serializes cleanly across an executor's process boundary. +export interface StepDrainInput { + sessionID: string + step: number + promotion: string | null + first: boolean + force: boolean + /** The attempt that owns the event log while this drain runs; set by the executor. */ + owner?: string +} + +export interface StepDrainResult { + ran: boolean + continue: boolean + step: number + promotion: string | null +} export interface DrainDeps { readonly store: SessionStore.Interface diff --git a/packages/core/src/session/execution/temporal.ts b/packages/temporal/src/executor.ts similarity index 94% rename from packages/core/src/session/execution/temporal.ts rename to packages/temporal/src/executor.ts index cb4f5104d168..9debee5bcad1 100644 --- a/packages/core/src/session/execution/temporal.ts +++ b/packages/temporal/src/executor.ts @@ -1,4 +1,4 @@ -export * as SessionExecutionTemporal from "./temporal" +export * as SessionExecutionTemporal from "./executor" import { fileURLToPath } from "node:url" import { Effect, Layer } from "effect" @@ -7,17 +7,17 @@ import { Client, Connection, WithStartWorkflowOperation } from "@temporalio/clie // the workflow from source at startup), which a compiled binary can neither bundle nor run. A // packaged serve runs OPENCODE_TEMPORAL_ROLE=client next to standalone workers instead. -import { LocationServiceMap } from "../../location-service-map" -import { EventV2 } from "../../event" -import { makeGlobalNode } from "../../effect/app-node" -import { SessionSchema } from "../schema" -import { SessionStore } from "../store" -import { SessionExecution } from "../execution" -import { makeStepActivities } from "./temporal-activities" +import { LocationServiceMap } from "@opencode-ai/core/location-service-map" +import { EventV2 } from "@opencode-ai/core/event" +import { makeGlobalNode } from "@opencode-ai/core/effect/app-node" +import { SessionSchema } from "@opencode-ai/core/session/schema" +import { SessionStore } from "@opencode-ai/core/session/store" +import { SessionExecution } from "@opencode-ai/core/session/execution" +import { makeStepActivities } from "./activities" import { makeDrains } from "./drain" -import { WorktreeMaterializer } from "./worktree" -import { toRunError } from "./run-error-codec" -import * as WF from "./temporal-workflow" +import { WorktreeMaterializer } from "@opencode-ai/core/session/execution/worktree" +import { toRunError } from "@opencode-ai/core/session/execution/run-error-codec" +import * as WF from "./workflow" // Classify an interrupt-signal delivery error. "already completed"/"not found" means an idle // session's workflow has already closed -- nothing to interrupt, a no-op. Anything else is a genuine diff --git a/packages/core/src/session/execution/workflow-core.ts b/packages/temporal/src/supervisor.ts similarity index 99% rename from packages/core/src/session/execution/workflow-core.ts rename to packages/temporal/src/supervisor.ts index 6e0a3d2a296b..8a48f019587d 100644 --- a/packages/core/src/session/execution/workflow-core.ts +++ b/packages/temporal/src/supervisor.ts @@ -12,7 +12,7 @@ // idle) and surfaces its result; `interrupt` stops the CURRENT turn (not the session) and the // long-lived supervisor keeps serving later wakes/resumes, terminating only after an idle period. -import type { StepDrainInput, StepDrainResult } from "./temporal-activities" +import type { StepDrainInput, StepDrainResult } from "./activities" /** What a driver must provide; everything else is supervisor logic. */ export interface WorkflowRuntime { diff --git a/packages/core/src/session/execution/temporal-workflow.ts b/packages/temporal/src/workflow.ts similarity index 97% rename from packages/core/src/session/execution/temporal-workflow.ts rename to packages/temporal/src/workflow.ts index 9ffe0b18639c..b7378378fe5e 100644 --- a/packages/core/src/session/execution/temporal-workflow.ts +++ b/packages/temporal/src/workflow.ts @@ -19,8 +19,8 @@ import { isCancellation, allHandlersFinished, } from "@temporalio/workflow" -import type { StepActivities } from "./temporal-activities" -import { makeWorkflows, type WorkflowRuntime } from "./workflow-core" +import type { StepActivities } from "./activities" +import { makeWorkflows, type WorkflowRuntime } from "./supervisor" const activityOptions = { // The heartbeat is the liveness bound (it stops within seconds of a worker death and Temporal diff --git a/packages/core/test/session-execution-temporal-contract.test.ts b/packages/temporal/test/session-execution-temporal-contract.test.ts similarity index 97% rename from packages/core/test/session-execution-temporal-contract.test.ts rename to packages/temporal/test/session-execution-temporal-contract.test.ts index 2ade9c351e32..bb9142008a62 100644 --- a/packages/core/test/session-execution-temporal-contract.test.ts +++ b/packages/temporal/test/session-execution-temporal-contract.test.ts @@ -16,6 +16,6 @@ if (process.env.OPENCODE_CONTRACT_TEMPORAL === "1") { // otherwise steal activities and answer with its own (differently mocked) graph. process.env.OPENCODE_TEMPORAL_TASK_QUEUE ??= `contract-${crypto.randomUUID()}` // Imported dynamically because the driver reads its connection config at module load. - const { SessionExecutionTemporal } = await import("@opencode-ai/core/session/execution/temporal") + const { SessionExecutionTemporal } = await import("@opencode-ai/temporal/executor") runContract("temporal driver", makeExecutionFor(SessionExecutionTemporal.node)) } diff --git a/packages/core/test/session-supervisor-rollover.test.ts b/packages/temporal/test/session-supervisor-rollover.test.ts similarity index 93% rename from packages/core/test/session-supervisor-rollover.test.ts rename to packages/temporal/test/session-supervisor-rollover.test.ts index cfafe24e4d71..cbbf7b20fde3 100644 --- a/packages/core/test/session-supervisor-rollover.test.ts +++ b/packages/temporal/test/session-supervisor-rollover.test.ts @@ -3,8 +3,8 @@ // the rollover-manufactures-a-wake bug: a rollover triggered by a resume drain must carry // startWithWake=false into the successor, not re-arm a spurious wake. Driven by a fake runtime. import { describe, it, expect } from "bun:test" -import { makeWorkflows, type WorkflowRuntime } from "@opencode-ai/core/session/execution/workflow-core" -import type { StepDrainResult } from "@opencode-ai/core/session/execution/temporal-activities" +import { makeWorkflows, type WorkflowRuntime } from "../src/supervisor" +import type { StepDrainResult } from "../src/activities" class ContinuedAsNew extends Error {} const DONE: StepDrainResult = { ran: true, continue: false, step: 1, promotion: null } diff --git a/packages/core/test/session-supervisor.test.ts b/packages/temporal/test/session-supervisor.test.ts similarity index 96% rename from packages/core/test/session-supervisor.test.ts rename to packages/temporal/test/session-supervisor.test.ts index 7618966060cf..f52e525cd965 100644 --- a/packages/core/test/session-supervisor.test.ts +++ b/packages/temporal/test/session-supervisor.test.ts @@ -4,8 +4,8 @@ // follow-up, an interrupt stops the current turn but the supervisor keeps serving, and a fresh // resume-with-start (startWithWake=false) does exactly one drain (no spurious wake drain). import { describe, it, expect } from "bun:test" -import { makeWorkflows, type WorkflowRuntime } from "@opencode-ai/core/session/execution/workflow-core" -import type { StepDrainResult } from "@opencode-ai/core/session/execution/temporal-activities" +import { makeWorkflows, type WorkflowRuntime } from "../src/supervisor" +import type { StepDrainResult } from "../src/activities" class FakeCancel extends Error {} const DONE: StepDrainResult = { ran: true, continue: false, step: 1, promotion: null } diff --git a/packages/core/test/temporal-harness-interrupt.test.ts b/packages/temporal/test/temporal-harness-interrupt.test.ts similarity index 96% rename from packages/core/test/temporal-harness-interrupt.test.ts rename to packages/temporal/test/temporal-harness-interrupt.test.ts index c452d712ce74..729b9a746e7e 100644 --- a/packages/core/test/temporal-harness-interrupt.test.ts +++ b/packages/temporal/test/temporal-harness-interrupt.test.ts @@ -9,7 +9,7 @@ import { TestWorkflowEnvironment } from "@temporalio/testing" import { Worker } from "@temporalio/worker" import { Context, heartbeat, CancelledFailure } from "@temporalio/activity" -const WORKFLOW = fileURLToPath(new URL("../src/session/execution/temporal-workflow.ts", import.meta.url)) +const WORKFLOW = fileURLToPath(new URL("../src/workflow.ts", import.meta.url)) const poll = async (fn: () => boolean, ms = 20_000) => { const deadline = Date.now() + ms while (!fn()) { diff --git a/packages/core/test/temporal-harness-multiturn.test.ts b/packages/temporal/test/temporal-harness-multiturn.test.ts similarity index 96% rename from packages/core/test/temporal-harness-multiturn.test.ts rename to packages/temporal/test/temporal-harness-multiturn.test.ts index 05917d446ee0..2b8f7c93ccfd 100644 --- a/packages/core/test/temporal-harness-multiturn.test.ts +++ b/packages/temporal/test/temporal-harness-multiturn.test.ts @@ -11,7 +11,7 @@ import { fileURLToPath } from "node:url" import { TestWorkflowEnvironment } from "@temporalio/testing" import { Worker } from "@temporalio/worker" -const WORKFLOW = fileURLToPath(new URL("../src/session/execution/temporal-workflow.ts", import.meta.url)) +const WORKFLOW = fileURLToPath(new URL("../src/workflow.ts", import.meta.url)) const poll = async (fn: () => boolean, ms = 20_000) => { const deadline = Date.now() + ms while (!fn()) { diff --git a/packages/core/test/temporal-harness-smoke.test.ts b/packages/temporal/test/temporal-harness-smoke.test.ts similarity index 95% rename from packages/core/test/temporal-harness-smoke.test.ts rename to packages/temporal/test/temporal-harness-smoke.test.ts index 5e4d6161f1b7..c0d5b5747ce7 100644 --- a/packages/core/test/temporal-harness-smoke.test.ts +++ b/packages/temporal/test/temporal-harness-smoke.test.ts @@ -13,7 +13,7 @@ import { fileURLToPath } from "node:url" import { TestWorkflowEnvironment } from "@temporalio/testing" import { Worker } from "@temporalio/worker" -const WORKFLOW = fileURLToPath(new URL("../src/session/execution/temporal-workflow.ts", import.meta.url)) +const WORKFLOW = fileURLToPath(new URL("../src/workflow.ts", import.meta.url)) describe("temporal workflow harness", () => { it("drives the real sessionTurn workflow through a wake-driven drain, then idles out", async () => { diff --git a/packages/core/test/temporal-interrupt-classify.test.ts b/packages/temporal/test/temporal-interrupt-classify.test.ts similarity index 90% rename from packages/core/test/temporal-interrupt-classify.test.ts rename to packages/temporal/test/temporal-interrupt-classify.test.ts index 93cbd447642b..281935ae32d3 100644 --- a/packages/core/test/temporal-interrupt-classify.test.ts +++ b/packages/temporal/test/temporal-interrupt-classify.test.ts @@ -1,7 +1,7 @@ // The interrupt path must not report a genuine signal-delivery failure as success. Only an already // closed workflow ("already completed"/"not found") is a benign no-op; everything else must fail. import { describe, it, expect } from "bun:test" -import { SessionExecutionTemporal } from "@opencode-ai/core/session/execution/temporal" +import { SessionExecutionTemporal } from "../src/executor" const classify = SessionExecutionTemporal.classifyInterruptError diff --git a/packages/core/test/temporal-owner-token.test.ts b/packages/temporal/test/temporal-owner-token.test.ts similarity index 92% rename from packages/core/test/temporal-owner-token.test.ts rename to packages/temporal/test/temporal-owner-token.test.ts index 0dac8342ecf9..7a2015068cba 100644 --- a/packages/core/test/temporal-owner-token.test.ts +++ b/packages/temporal/test/temporal-owner-token.test.ts @@ -2,7 +2,7 @@ // step, so a run-id+attempt token repeats across steps and lets a zombie attempt from an earlier // step re-match the current owner. The token must be unique per activity execution. import { describe, it, expect } from "bun:test" -import { ownerTokenFrom } from "@opencode-ai/core/session/execution/temporal-activities" +import { ownerTokenFrom } from "../src/activities" describe("temporal event-log owner token", () => { const run = "run-1" diff --git a/packages/temporal/tsconfig.json b/packages/temporal/tsconfig.json new file mode 100644 index 000000000000..00ef12546856 --- /dev/null +++ b/packages/temporal/tsconfig.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "@tsconfig/bun/tsconfig.json", + "compilerOptions": { + "lib": ["ESNext", "DOM", "DOM.Iterable"], + "noUncheckedIndexedAccess": false + } +} From 34f6f83e2fb62a5819e7a32b10417e22570caaaf Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 15 Aug 2026 15:27:28 -0700 Subject: [PATCH 092/103] Collected the executor's protocol and config into their own modules. protocol.ts is the one source of truth for the workflow type, signal and update names, the workflow-id scheme, and the connection defaults; config.ts is a service read at layer build, so an embedder or a test injects settings instead of racing module-load env reads. --- packages/temporal/src/config.ts | 31 +++++++++++++++++++++++ packages/temporal/src/executor.ts | 41 +++++++++++-------------------- packages/temporal/src/protocol.ts | 18 ++++++++++++++ packages/temporal/src/workflow.ts | 7 +++--- 4 files changed, 67 insertions(+), 30 deletions(-) create mode 100644 packages/temporal/src/config.ts create mode 100644 packages/temporal/src/protocol.ts diff --git a/packages/temporal/src/config.ts b/packages/temporal/src/config.ts new file mode 100644 index 000000000000..6d4c7c70de3e --- /dev/null +++ b/packages/temporal/src/config.ts @@ -0,0 +1,31 @@ +export * as TemporalConfig from "./config" + +// Connection and behavior settings for the Temporal executor. The executor reads them at layer +// build: an embedder or a test provides the service to override, and absent that the values come +// from env. Nothing reads env at module load, so import order carries no configuration. +import { Context } from "effect" +import { DEFAULTS } from "./protocol" + +// `both` (default) hosts the activity worker AND the workflow client in one process (the serve +// process). `client` drives workflows without hosting a worker (a packaged binary cannot carry the +// worker's bundler); `worker` runs a standalone activity worker with no HTTP surface. +export type Role = "both" | "client" | "worker" + +export interface Interface { + readonly address: string + readonly namespace: string + readonly taskQueue: string + readonly role: Role + /** Override for the supervisor's idle self-termination; local mode honors the same variable. */ + readonly idleTimeout?: string +} + +export class Service extends Context.Service()("@opencode/temporal/Config") {} + +export const fromEnv = (): Interface => ({ + address: process.env.TEMPORAL_ADDRESS ?? DEFAULTS.address, + namespace: process.env.TEMPORAL_NAMESPACE ?? DEFAULTS.namespace, + taskQueue: process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? DEFAULTS.taskQueue, + role: (process.env.OPENCODE_TEMPORAL_ROLE as Role | undefined) ?? "both", + idleTimeout: process.env.OPENCODE_SESSION_IDLE_TIMEOUT, +}) diff --git a/packages/temporal/src/executor.ts b/packages/temporal/src/executor.ts index 9debee5bcad1..66d093ba9c19 100644 --- a/packages/temporal/src/executor.ts +++ b/packages/temporal/src/executor.ts @@ -1,7 +1,7 @@ export * as SessionExecutionTemporal from "./executor" import { fileURLToPath } from "node:url" -import { Effect, Layer } from "effect" +import { Effect, Layer, Option } from "effect" import { Client, Connection, WithStartWorkflowOperation } from "@temporalio/client" // Imported lazily inside the worker branch: the worker package drags webpack and swc (it bundles // the workflow from source at startup), which a compiled binary can neither bundle nor run. A @@ -18,6 +18,8 @@ import { makeDrains } from "./drain" import { WorktreeMaterializer } from "@opencode-ai/core/session/execution/worktree" import { toRunError } from "@opencode-ai/core/session/execution/run-error-codec" import * as WF from "./workflow" +import { TemporalConfig } from "./config" +import { WORKFLOW_TYPE, WORKFLOW_ID_PREFIX, workflowId } from "./protocol" // Classify an interrupt-signal delivery error. "already completed"/"not found" means an idle // session's workflow has already closed -- nothing to interrupt, a no-op. Anything else is a genuine @@ -27,24 +29,6 @@ export function classifyInterruptError(e: unknown): "ignore" | "fail" { return /already completed|not found/i.test(message) ? "ignore" : "fail" } -const ADDRESS = process.env.TEMPORAL_ADDRESS ?? "127.0.0.1:7237" -const NAMESPACE = process.env.TEMPORAL_NAMESPACE ?? "default" -const TASK_QUEUE = process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? "opencode-session-exec" -const workflowId = (id: string) => `session-exec-${id}` - -// One activity per step (the model call + its tools), with the step loop as workflow control flow. -// Workflows start by the string type, never the function: a minified (packaged) client would -// otherwise register the mangled function name as the type and no worker would match it. -const WORKFLOW_TYPE = "sessionTurn" - -// Role split so the worker fleet can run separately from the HTTP server. `both` (default) hosts the -// activity worker AND the workflow client in one process (the serve process). `client` makes serve -// drive workflows without hosting a worker; `worker` runs a standalone activity worker with no HTTP -// surface (see packages/server/src/worker.ts). -const ROLE = process.env.OPENCODE_TEMPORAL_ROLE ?? "both" -const HOST_WORKER = ROLE !== "client" -const HOST_CLIENT = ROLE !== "worker" - /** * A Temporal-backed SessionExecution. It makes each session a durable workflow: * - wake -> signalWithStart(wake) (start while idle, or coalesce into the run) @@ -66,10 +50,14 @@ const layer = Layer.effect( // The app context the local drain runs in: providing it, then the per-location layer, supplies // SessionRunner and all of its dependencies. const ctx = yield* Effect.context() - // Same knob local mode honors. The workflow sandbox cannot read env, so the client forwards the - // override as a workflow argument. Read at layer build (not module load) so tests can set it - // before constructing the layer. - const IDLE_TIMEOUT = process.env.OPENCODE_SESSION_IDLE_TIMEOUT + // Provided by an embedder or a test, env otherwise; nothing is read at module load. + const config = Option.getOrElse(yield* Effect.serviceOption(TemporalConfig.Service), TemporalConfig.fromEnv) + const { address: ADDRESS, namespace: NAMESPACE, taskQueue: TASK_QUEUE } = config + const HOST_WORKER = config.role !== "client" + const HOST_CLIENT = config.role !== "worker" + // Same knob local mode honors; the workflow sandbox cannot read env, so the client forwards the + // override as a workflow argument. + const IDLE_TIMEOUT = config.idleTimeout const events = yield* EventV2.Service const worktrees = yield* WorktreeMaterializer.Service @@ -113,13 +101,12 @@ const layer = Layer.effect( ) } - const SESSION_PREFIX = "session-exec-" // Worker-only process: it hosts activities but drives no workflows, so the client methods are // unused. Return a service whose driving methods fail loudly if something unexpectedly calls them. if (!HOST_CLIENT) { yield* Effect.logInfo("SessionExecutionTemporal worker ready").pipe( - Effect.annotateLogs({ address: ADDRESS, taskQueue: TASK_QUEUE, workflow: WORKFLOW_TYPE, role: ROLE }), + Effect.annotateLogs({ address: ADDRESS, taskQueue: TASK_QUEUE, workflow: WORKFLOW_TYPE, role: config.role }), ) const clientOnly = Effect.die("SessionExecution client is not hosted when OPENCODE_TEMPORAL_ROLE=worker") return SessionExecution.Service.of({ @@ -165,8 +152,8 @@ const layer = Layer.effect( for await (const wf of client.workflow.list({ query: `WorkflowType = 'sessionTurn' AND ExecutionStatus = 'Running'`, })) { - if (wf.workflowId.startsWith(SESSION_PREFIX)) { - ids.push(SessionSchema.ID.make(wf.workflowId.slice(SESSION_PREFIX.length))) + if (wf.workflowId.startsWith(WORKFLOW_ID_PREFIX)) { + ids.push(SessionSchema.ID.make(wf.workflowId.slice(WORKFLOW_ID_PREFIX.length))) } } return ids diff --git a/packages/temporal/src/protocol.ts b/packages/temporal/src/protocol.ts new file mode 100644 index 000000000000..3f7807fef94f --- /dev/null +++ b/packages/temporal/src/protocol.ts @@ -0,0 +1,18 @@ +export * as TemporalProtocol from "./protocol" + +// One source of truth for the names the client, the worker, and the tests must agree on. Workflows +// start by the string type, never the function: a minified (packaged) client would otherwise +// register the mangled function name as the type and no worker would match it. +export const WORKFLOW_TYPE = "sessionTurn" + +export const SIGNALS = { wake: "wake", interrupt: "interrupt" } as const +export const RESUME_UPDATE = "resume" + +export const WORKFLOW_ID_PREFIX = "session-exec-" +export const workflowId = (sessionID: string) => `${WORKFLOW_ID_PREFIX}${sessionID}` + +export const DEFAULTS = { + address: "127.0.0.1:7237", + namespace: "default", + taskQueue: "opencode-session-exec", +} as const diff --git a/packages/temporal/src/workflow.ts b/packages/temporal/src/workflow.ts index b7378378fe5e..c5426f011e6c 100644 --- a/packages/temporal/src/workflow.ts +++ b/packages/temporal/src/workflow.ts @@ -20,6 +20,7 @@ import { allHandlersFinished, } from "@temporalio/workflow" import type { StepActivities } from "./activities" +import { SIGNALS, RESUME_UPDATE } from "./protocol" import { makeWorkflows, type WorkflowRuntime } from "./supervisor" const activityOptions = { @@ -36,9 +37,9 @@ const activityOptions = { const { runTurnStep } = proxyActivities(activityOptions) -export const wake = defineSignal("wake") -export const interrupt = defineSignal("interrupt") -export const resume = defineUpdate("resume") +export const wake = defineSignal(SIGNALS.wake) +export const interrupt = defineSignal(SIGNALS.interrupt) +export const resume = defineUpdate(RESUME_UPDATE) const signals = { wake, interrupt } as const const runtime: WorkflowRuntime = { From ca618b99a7602eb6e45ccd5d6fa1d574e949632d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 15 Aug 2026 15:28:27 -0700 Subject: [PATCH 093/103] Folded the workflow's inputs into one options record. Positional arguments turn signature evolution into a breaking change mid-history; a single record lets new settings ride along, and a continue-as-new run carries it forward unchanged. --- packages/temporal/src/executor.ts | 4 ++-- packages/temporal/src/workflow.ts | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/temporal/src/executor.ts b/packages/temporal/src/executor.ts index 66d093ba9c19..de1ba995f967 100644 --- a/packages/temporal/src/executor.ts +++ b/packages/temporal/src/executor.ts @@ -129,7 +129,7 @@ const layer = Layer.effect( client.workflow.signalWithStart(WORKFLOW_TYPE, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), - args: [id, true, IDLE_TIMEOUT], + args: [id, { startWithWake: true, idleTimeout: IDLE_TIMEOUT } satisfies WF.SessionTurnOptions], signal: WF.wake, signalArgs: [], }), @@ -180,7 +180,7 @@ const layer = Layer.effect( // startWithWake=false: a fresh resume-with-start must not manufacture a wake drain; // its forced drain comes from the resume update. Ignored when USE_EXISTING joins a // running workflow (which keeps its own state). - args: [id, false, IDLE_TIMEOUT], + args: [id, { startWithWake: false, idleTimeout: IDLE_TIMEOUT } satisfies WF.SessionTurnOptions], workflowIdConflictPolicy: "USE_EXISTING", }) return client.workflow.executeUpdateWithStart(WF.resume, { diff --git a/packages/temporal/src/workflow.ts b/packages/temporal/src/workflow.ts index c5426f011e6c..28dc278f1c84 100644 --- a/packages/temporal/src/workflow.ts +++ b/packages/temporal/src/workflow.ts @@ -92,7 +92,7 @@ const runtime: WorkflowRuntime = { isRootCancelled: () => rootScope?.consideredCancelled ?? false, allHandlersFinished, continueAsNew: (sessionID, startWithWake) => - continueAsNew<(id: string, startWithWake: boolean) => Promise>(sessionID, startWithWake), + continueAsNew(sessionID, { startWithWake }), } // The scope of the drain currently running, so an interrupt signal can cancel exactly that turn. @@ -102,24 +102,24 @@ let rootScope: CancellationScope | undefined const workflows = makeWorkflows(runtime) -// The sandbox cannot read env, so the idle override arrives as a workflow argument (the client -// reads the same variable local mode honors) and a continue-as-new run keeps it. -export async function sessionTurn( - sessionID: string, - startWithWake: boolean = true, - idleTimeout?: string, -): Promise { +// One evolvable input record instead of positional arguments: new settings ride along without a +// signature change, and a continue-as-new run carries the record forward. The sandbox cannot read +// env, so the idle override arrives here from the client (which reads the same variable local mode +// honors). +export interface SessionTurnOptions { + readonly startWithWake?: boolean + readonly idleTimeout?: string +} + +export async function sessionTurn(sessionID: string, options?: SessionTurnOptions): Promise { rootScope = CancellationScope.current() + const startWithWake = options?.startWithWake ?? true + const idleTimeout = options?.idleTimeout if (!idleTimeout) return workflows.sessionTurn(sessionID, startWithWake) return makeWorkflows( { ...runtime, - continueAsNew: (id, wake) => - continueAsNew<(id: string, startWithWake: boolean, idleTimeout?: string) => Promise>( - id, - wake, - idleTimeout, - ), + continueAsNew: (id, wake) => continueAsNew(id, { startWithWake: wake, idleTimeout }), }, { idleTimeout }, ).sessionTurn(sessionID, startWithWake) From f5e8d315016ca0a1f641bfe6062a109977d27bed Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 15 Aug 2026 15:29:13 -0700 Subject: [PATCH 094/103] Renamed the supervisor pieces for what they are now. workflow-core became supervisor.ts when local mode stopped driving it; SupervisorRuntime and makeSupervisor say what the interface and the factory actually build. --- packages/temporal/src/executor.ts | 2 +- packages/temporal/src/supervisor.ts | 6 +++--- packages/temporal/src/workflow.ts | 12 ++++++------ .../test/session-supervisor-rollover.test.ts | 6 +++--- packages/temporal/test/session-supervisor.test.ts | 12 ++++++------ .../temporal/test/temporal-harness-smoke.test.ts | 2 +- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/temporal/src/executor.ts b/packages/temporal/src/executor.ts index de1ba995f967..1d3e7042a0e6 100644 --- a/packages/temporal/src/executor.ts +++ b/packages/temporal/src/executor.ts @@ -87,7 +87,7 @@ const layer = Layer.effect( connection: nativeConn, namespace: NAMESPACE, taskQueue: TASK_QUEUE, - workflowsPath: fileURLToPath(new URL("./temporal-workflow.ts", import.meta.url)), + workflowsPath: fileURLToPath(new URL("./workflow.ts", import.meta.url)), activities: makeStepActivities(stepDrain), }), ) diff --git a/packages/temporal/src/supervisor.ts b/packages/temporal/src/supervisor.ts index 8a48f019587d..d279dcfc5164 100644 --- a/packages/temporal/src/supervisor.ts +++ b/packages/temporal/src/supervisor.ts @@ -1,5 +1,5 @@ // The Temporal driver's per-session supervisor, expressed over a runtime interface so the SDK's -// condition/signals/updates/activities/cancellation plug in (temporal-workflow.ts) and it stays +// condition/signals/updates/activities/cancellation plug in (workflow.ts) and it stays // unit-testable off a live cluster (a fake runtime). Local mode does NOT run this loop: it uses the // proven SessionRunCoordinator directly (execution/local.ts). The two modes share SessionRunner and // the durable event log, not this supervisor. @@ -15,7 +15,7 @@ import type { StepDrainInput, StepDrainResult } from "./activities" /** What a driver must provide; everything else is supervisor logic. */ -export interface WorkflowRuntime { +export interface SupervisorRuntime { /** Wait until the predicate is true. With a timeout, resolve false when it expires first. */ readonly condition: (predicate: () => boolean, timeout?: string) => Promise readonly setSignalHandler: (name: "wake" | "interrupt", handler: () => void) => void @@ -54,7 +54,7 @@ export interface WorkflowOptions { readonly maxDrainsPerRun?: number } -export const makeWorkflows = (rt: WorkflowRuntime, options?: WorkflowOptions) => { +export const makeSupervisor = (rt: SupervisorRuntime, options?: WorkflowOptions) => { const IDLE_TIMEOUT = options?.idleTimeout ?? "5 minutes" // A continuously busy session never hits the idle return, so without a bound its history grows // until Temporal terminates the workflow. continue-as-new carries the pending-wake state, so no diff --git a/packages/temporal/src/workflow.ts b/packages/temporal/src/workflow.ts index 28dc278f1c84..d19ac7df8228 100644 --- a/packages/temporal/src/workflow.ts +++ b/packages/temporal/src/workflow.ts @@ -1,6 +1,6 @@ -// The Temporal driver for the session supervisor. The supervisor loop lives in workflow-core.ts; +// The Temporal driver for the session supervisor. The supervisor loop lives in supervisor.ts; // this file adapts the real SDK's primitives (condition, signal/update handlers, activity proxies, -// cancellation) to the WorkflowRuntime interface and exports the workflow function the worker +// cancellation) to the SupervisorRuntime interface and exports the workflow function the worker // registers. Local mode does not use this loop -- it runs the proven SessionRunCoordinator directly // (execution/local.ts); the two modes share SessionRunner and the durable event log. // @@ -21,7 +21,7 @@ import { } from "@temporalio/workflow" import type { StepActivities } from "./activities" import { SIGNALS, RESUME_UPDATE } from "./protocol" -import { makeWorkflows, type WorkflowRuntime } from "./supervisor" +import { makeSupervisor, type SupervisorRuntime } from "./supervisor" const activityOptions = { // The heartbeat is the liveness bound (it stops within seconds of a worker death and Temporal @@ -42,7 +42,7 @@ export const interrupt = defineSignal(SIGNALS.interrupt) export const resume = defineUpdate(RESUME_UPDATE) const signals = { wake, interrupt } as const -const runtime: WorkflowRuntime = { +const runtime: SupervisorRuntime = { // Short-circuit when the predicate already holds. Besides saving a round trip, this avoids a real // breakage: on @temporalio/workflow 1.21, calling condition(fn, timeout) when fn is already true // leaves the current CancellationScope cancelled, so the NEXT condition() throws CancelledFailure @@ -100,7 +100,7 @@ let activeDrainScope: CancellationScope | undefined // The workflow's root scope, captured at entry, to detect a whole-run cancellation. let rootScope: CancellationScope | undefined -const workflows = makeWorkflows(runtime) +const workflows = makeSupervisor(runtime) // One evolvable input record instead of positional arguments: new settings ride along without a // signature change, and a continue-as-new run carries the record forward. The sandbox cannot read @@ -116,7 +116,7 @@ export async function sessionTurn(sessionID: string, options?: SessionTurnOption const startWithWake = options?.startWithWake ?? true const idleTimeout = options?.idleTimeout if (!idleTimeout) return workflows.sessionTurn(sessionID, startWithWake) - return makeWorkflows( + return makeSupervisor( { ...runtime, continueAsNew: (id, wake) => continueAsNew(id, { startWithWake: wake, idleTimeout }), diff --git a/packages/temporal/test/session-supervisor-rollover.test.ts b/packages/temporal/test/session-supervisor-rollover.test.ts index cbbf7b20fde3..29bf46d9d662 100644 --- a/packages/temporal/test/session-supervisor-rollover.test.ts +++ b/packages/temporal/test/session-supervisor-rollover.test.ts @@ -3,14 +3,14 @@ // the rollover-manufactures-a-wake bug: a rollover triggered by a resume drain must carry // startWithWake=false into the successor, not re-arm a spurious wake. Driven by a fake runtime. import { describe, it, expect } from "bun:test" -import { makeWorkflows, type WorkflowRuntime } from "../src/supervisor" +import { makeSupervisor, type SupervisorRuntime } from "../src/supervisor" import type { StepDrainResult } from "../src/activities" class ContinuedAsNew extends Error {} const DONE: StepDrainResult = { ran: true, continue: false, step: 1, promotion: null } const settle = () => new Promise((r) => setTimeout(r, 0)) -class FakeRuntime implements WorkflowRuntime { +class FakeRuntime implements SupervisorRuntime { steps = 0 continued = 0 continuedWith: boolean | undefined @@ -57,7 +57,7 @@ describe("supervisor continue-as-new counting", () => { it("counts a resume-driven drain toward the bound and rolls over carrying no spurious wake", async () => { const rt = new FakeRuntime() // Bound of 2: the initial wake drain is #1; a single resume drain is #2 and must roll over. - const supervisor = makeWorkflows(rt, { maxDrainsPerRun: 2 }).sessionTurn("ses_rollover") + const supervisor = makeSupervisor(rt, { maxDrainsPerRun: 2 }).sessionTurn("ses_rollover") let ended = false supervisor.then( () => (ended = true), diff --git a/packages/temporal/test/session-supervisor.test.ts b/packages/temporal/test/session-supervisor.test.ts index f52e525cd965..0855bb97cd50 100644 --- a/packages/temporal/test/session-supervisor.test.ts +++ b/packages/temporal/test/session-supervisor.test.ts @@ -1,10 +1,10 @@ -// Deterministic unit tests for the Temporal supervisor (execution/workflow-core.ts) driven by a fake -// WorkflowRuntime -- no Temporal, no DB. Covers the coordination redesign: resume joins the single +// Deterministic unit tests for the Temporal supervisor (execution/supervisor.ts) driven by a fake +// SupervisorRuntime -- no Temporal, no DB. Covers the coordination redesign: resume joins the single // in-flight drain (never a second forced drain), a wake that only joins a resume drain still gets a // follow-up, an interrupt stops the current turn but the supervisor keeps serving, and a fresh // resume-with-start (startWithWake=false) does exactly one drain (no spurious wake drain). import { describe, it, expect } from "bun:test" -import { makeWorkflows, type WorkflowRuntime } from "../src/supervisor" +import { makeSupervisor, type SupervisorRuntime } from "../src/supervisor" import type { StepDrainResult } from "../src/activities" class FakeCancel extends Error {} @@ -14,7 +14,7 @@ const settle = () => new Promise((r) => setTimeout(r, 0)) // runTurnStep either resolves immediately (ungated) or parks until released (gated), so a turn can // be held in flight while resumes/wakes/interrupts are delivered. cancelCurrentScope rejects the // in-flight step with FakeCancel, modelling an interrupt aborting the active drain's scope. -class FakeRuntime implements WorkflowRuntime { +class FakeRuntime implements SupervisorRuntime { steps = 0 gated = false private waiters: { predicate: () => boolean; resolve: (b: boolean) => void; isTimeout: boolean }[] = [] @@ -75,7 +75,7 @@ class FakeRuntime implements WorkflowRuntime { const start = (rt: FakeRuntime, startWithWake = true) => { let ended = false - makeWorkflows(rt) + makeSupervisor(rt) .sessionTurn("ses_test", startWithWake) .then( () => (ended = true), @@ -84,7 +84,7 @@ const start = (rt: FakeRuntime, startWithWake = true) => { return { isDone: () => ended } } -describe("Temporal supervisor (workflow-core)", () => { +describe("Temporal supervisor", () => { it("concurrent resumes join a single drain (no duplicate forced turns)", async () => { const rt = new FakeRuntime() rt.gated = true diff --git a/packages/temporal/test/temporal-harness-smoke.test.ts b/packages/temporal/test/temporal-harness-smoke.test.ts index c0d5b5747ce7..59615eec6f2b 100644 --- a/packages/temporal/test/temporal-harness-smoke.test.ts +++ b/packages/temporal/test/temporal-harness-smoke.test.ts @@ -1,5 +1,5 @@ // A real Temporal test harness for the session workflow: it runs the ACTUAL sessionTurn workflow -// (temporal-workflow.ts + workflow-core.ts) against @temporalio/testing's time-skipping server with +// (workflow.ts + supervisor.ts) against @temporalio/testing's time-skipping server with // a mock runTurnStep activity, entirely in-process -- no dev server, no provider, deterministic time. // This is the harness for validating the Temporal supervisor's real behavior (draining, idle // retirement, and -- as the suite grows -- interrupt/resume/join/continue-as-new). From 1d23e4403808b4200e96f19013591c8771ccd44f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 15 Aug 2026 15:30:59 -0700 Subject: [PATCH 095/103] Retold the README around the executor seam. The package reads as what it is now: one executor behind core's seam, held to the conformance suite, with a porting recipe for engines that want the same shape. --- packages/server/src/routes.ts | 3 ++- packages/temporal/README.md | 41 +++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index 71b750fb878f..94117d000399 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -54,7 +54,8 @@ export function createEmbeddedRoutes() { // exact same context. export function createServiceLayer() { // The factory: two modes. "temporal" runs each session as a per-step Temporal workflow - // (execution/temporal.ts); anything else runs it in-process on the proven SessionRunCoordinator + // (@opencode-ai/temporal, wired here as one dependency); anything else runs it in-process on the + // proven SessionRunCoordinator // (execution/local.ts) -- no server, no ports. Both drive SessionRunner over the same durable // event log; the local coordinator owns the wake/resume/interrupt lifecycle and is shared with // the v1 server path, so it is the well-exercised default. diff --git a/packages/temporal/README.md b/packages/temporal/README.md index bd8969da021e..1de02f76b2b8 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -1,9 +1,11 @@ # @opencode-ai/temporal -A Temporal durable-execution layer for opencode. It makes an opencode **session** a durable -Temporal workflow, so a coding session survives worker loss, can run detached or in the -background, and can be driven from anywhere by signal. It is a drop-in layer: opencode's loop, -tools, model, storage, and HTTP API are untouched. +The Temporal executor for opencode's `SessionExecution` seam, packaged as a plugin: core carries +the seam, the built-in local executor, and the executor-agnostic toolkit; this package is one +dependency that makes an opencode **session** a durable Temporal workflow. A coding session +survives worker loss, can run detached or in the background, and can be driven from anywhere by +signal. opencode's loop, tools, model, storage, and HTTP API are untouched, and nothing Temporal +exists in core. A lighter increment exists as its own change: the `2026/08/opencode-temporal-http` branch wraps the **shipping** `opencode serve` over its HTTP API, for the agent-as-black-box case. It makes the @@ -16,12 +18,14 @@ One design decision carries the change: durability is a choice of executor behin `SessionExecution` service, and both executors drive the same `SessionRunner` over the same durable event log. Everything else here is a consequence of taking at-least-once execution seriously. -One env var picks the executor. `temporal` runs each session as a per-session Temporal workflow with -one activity per step (`workflow-core.ts` + `temporal.ts`). The default runs it in-process on the -proven `SessionRunCoordinator` (`execution/local.ts`) -- the same lifecycle the v1 server uses -- with -no server and no ports (see [Two modes, one runner](#two-modes-one-runner)). The coordinator owns the -local wake/resume/interrupt lifecycle; the Temporal supervisor mirrors its semantics inside the -workflow sandbox. +One env var picks the executor. `temporal` runs each session as a per-session Temporal workflow +with one activity per step (this package: `executor.ts` wires the client and worker, `supervisor.ts` +is the loop, `workflow.ts` adapts it to the sandbox, `drain.ts` is the step body). The default runs +in-process on the proven `SessionRunCoordinator` (core's `execution/local.ts`), the same lifecycle +the v1 server uses, with no server and no ports (see [Two modes, one runner](#two-modes-one-runner)). +What an executor must do is defined executably: core's conformance suite +(`session/execution/conformance.ts`) runs the same wake/resume/interrupt scenarios against the local +executor in core's tests and against this package through real workflows. That forces six things: @@ -93,7 +97,7 @@ session runs as a Temporal workflow `session-exec-`. The factory has exactly two modes, both driving the same `SessionRunner` over the same durable event log. `OPENCODE_SESSION_EXECUTION=temporal` runs each session as a per-session Temporal workflow: the -`sessionTurn` supervisor (`workflow-core.ts`) loops a `runTurnStep` drain, so each step (one provider +`sessionTurn` supervisor (`supervisor.ts`) loops a `runTurnStep` drain, so each step (one provider attempt + its tools) is its own activity with its own retry/timeout/visibility, reusing `SessionRunner.runStep` (one iteration of `run`'s loop). Anything else (the default) runs in-process on the proven `SessionRunCoordinator` (`execution/local.ts`) -- no server, no worker, no ports -- which @@ -268,3 +272,18 @@ Host-local state that does NOT ride the DB, so it is not reconstructed on a diff best-effort (`Effect.catch` to `undefined`), and the model sees the bounded tool-output preview, not the file. They only affect the diff/restore/revert features and full-output viewing. Point `${data}` (the XDG data dir) at shared storage to make them portable. + +## Porting this pattern + +The shape transfers to any agent engine; Temporal is one executor behind a seam the engine owns. + +1. Find the engine's coordination seam and name it: here, four verbs (`active`, `wake`, `resume`, + `interrupt`) behind one substitutable service, with the in-process coordinator as the default. +2. Make the turn body an idempotent, fenced step function: claim the log with an owner token, + reuse recorded results on re-drive, encode errors so they survive a process boundary. +3. Write the executor as a thin workflow that loops the step as activities; keep the loop free of + engine imports so it stays deterministic and sandbox-safe. +4. Forward settings as one evolvable input record; read configuration at layer build, never at + module load. +5. Hold every executor to one conformance suite. Parity between the default and the durable path + is a test, not a promise. From 777e95083ee04d80d0e0ae443bf741d6b386925e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 15 Aug 2026 18:36:13 -0700 Subject: [PATCH 096/103] Made the conformance suite's fixture endpoint self-evidently inert. The suite mocks the LLM client, so the model descriptor's endpoint and token are never used for I/O; a real-looking OpenAI URL invited the wrong conclusion. An RFC 2606 .invalid host cannot be mistaken for a live dependency and cannot resolve if the mock is ever removed. --- packages/core/src/session/execution/conformance.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/core/src/session/execution/conformance.ts b/packages/core/src/session/execution/conformance.ts index ce9e01ed8b3c..da3c0e99d577 100644 --- a/packages/core/src/session/execution/conformance.ts +++ b/packages/core/src/session/execution/conformance.ts @@ -45,8 +45,11 @@ import { testEffect } from "../../testing/effect" // The per-location service build resolves the session directory on disk, so it must exist. const WORKSPACE = AbsolutePath.make(realpathSync(tmpdir())) +// Inert fixture data: the suite replaces LLMClient.Service with an injected stream, so nothing +// dials this endpoint or sends this token. The runner only needs a well-formed model descriptor +// to select and record; the .invalid TLD (RFC 2606) makes the inertness visible. const model = OpenAIChat.route - .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) + .with({ endpoint: { baseURL: "https://llm.fixture.invalid/v1" }, auth: Auth.bearer("fixture") }) .model({ id: "gpt-4o-mini" }) const okModels = SessionRunnerModel.layerWith(() => Effect.succeed(model)) const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) From df0583117a7079617eda1116ee384f27b9d38414 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 15 Aug 2026 19:01:38 -0700 Subject: [PATCH 097/103] Retired the conformance resume sessions before their run ends. The resume scenarios left a durable executor's session running forever: the test's task queue dies with the process, so nothing ever processes the idle timer's task. Each scenario now waits out the retirement while its worker still exists. --- .../core/src/session/execution/conformance.ts | 58 +++++++++++-------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/packages/core/src/session/execution/conformance.ts b/packages/core/src/session/execution/conformance.ts index da3c0e99d577..a9386a7ed724 100644 --- a/packages/core/src/session/execution/conformance.ts +++ b/packages/core/src/session/execution/conformance.ts @@ -206,19 +206,25 @@ export const runContract = (label: string, makeExec: ReturnType - Effect.gen(function* () { - yield* seedSession(sessionID) - yield* seedPrompt(sessionID) - const exec = Context.get(yield* Layer.build(makeExec(stream)), SessionExecution.Service) - // resume is request/response: it awaits the forced drain and resolves on success. - const exit = yield* exec.resume(sessionID).pipe(Effect.exit) - expect(Exit.isSuccess(exit)).toBe(true) - expect(requests).toHaveLength(1) - const store = yield* SessionStore.Service - const context = yield* store.context(sessionID) - const assistant = context.findLast((message) => message.type === "assistant") - expect(assistant?.type === "assistant" && Boolean(assistant.time.completed)).toBe(true) - }), + withIdleOverride( + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + const exec = Context.get(yield* Layer.build(makeExec(stream)), SessionExecution.Service) + // resume is request/response: it awaits the forced drain and resolves on success. + const exit = yield* exec.resume(sessionID).pipe(Effect.exit) + expect(Exit.isSuccess(exit)).toBe(true) + expect(requests).toHaveLength(1) + const store = yield* SessionStore.Service + const context = yield* store.context(sessionID) + const assistant = context.findLast((message) => message.type === "assistant") + expect(assistant?.type === "assistant" && Boolean(assistant.time.completed)).toBe(true) + // Wait out the retirement while this run's worker still exists. A durable executor's + // session would otherwise linger forever: the test's task queue dies with the process, + // so nothing ever processes the idle timer's task. + yield* until(exec.active, (active) => !active.has(sessionID)) + }), + ), 60000, ) } @@ -227,17 +233,21 @@ export const runContract = (label: string, makeExec: ReturnType Effect.fail(new ModelNotSelectedError({ sessionID }))) it.live("resume surfaces the exact tagged RunError through the shared codec", () => - Effect.gen(function* () { - yield* seedSession(sessionID) - const { stream } = countingModel() - const exec = Context.get(yield* Layer.build(makeExec(stream, failingModels)), SessionExecution.Service) - const exit = yield* exec.resume(sessionID).pipe(Effect.exit) - expect(Exit.isFailure(exit)).toBe(true) - const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined - // The same encode -> details -> decode path the Temporal boundary uses, so the caller gets - // the identical tagged instance in both modes. - expect(error).toBeInstanceOf(ModelNotSelectedError) - }), + withIdleOverride( + Effect.gen(function* () { + yield* seedSession(sessionID) + const { stream } = countingModel() + const exec = Context.get(yield* Layer.build(makeExec(stream, failingModels)), SessionExecution.Service) + const exit = yield* exec.resume(sessionID).pipe(Effect.exit) + expect(Exit.isFailure(exit)).toBe(true) + const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined + // The same encode -> details -> decode path the Temporal boundary uses, so the caller gets + // the identical tagged instance in both modes. + expect(error).toBeInstanceOf(ModelNotSelectedError) + // Same reason as the healthy resume: retire before this run's task queue dies. + yield* until(exec.active, (active) => !active.has(sessionID)) + }), + ), 60000, ) } From 86dac84bb43bacd811f9f184ac9f9e9827815a5b Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 17 Aug 2026 01:23:55 -0700 Subject: [PATCH 098/103] Followed cross-process turns in the TUI's event projector. With OPENCODE_TEMPORAL_ROLE=client the turn runs in a separate worker process, and its session.next.* events live on that process's bus; the TUI only observes its own daemon's admission. A reply therefore rendered one prompt late, when the next admission triggered a re-read. A followed session now re-reads until its latest assistant message settles. --- packages/cli/src/tui.ts | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/tui.ts b/packages/cli/src/tui.ts index cd9161398025..dba9e404c5cc 100644 --- a/packages/cli/src/tui.ts +++ b/packages/cli/src/tui.ts @@ -455,8 +455,18 @@ async function globalEventStream(origin: string, directory: string | null, heade // its session.next.* stream into them: admissions and text deltas map directly for low latency, // and every other session.next.* event schedules a debounced re-read of the session so the store // converges on what the daemon persisted (tools, reasoning, tokens, titles). +// +// Cross-process turns stream nothing here: with OPENCODE_TEMPORAL_ROLE=client the turn runs in a +// separate worker process, and its session.next.* events live on that process's bus. The only +// locally observable moment is this daemon's own admission, so a followed session keeps re-reading +// until its latest assistant message settles; without that, a reply would only render when the +// NEXT local event happened to trigger a re-read. function createSessionProjector(origin: string, headers: HeadersInit | undefined, emit: (dir: string, payload: unknown) => void) { const timers = new Map>() + const FOLLOW_INTERVAL = 1200 + // A turn longer than this stops refreshing a cross-process TUI until the next admission. + const FOLLOW_DEADLINE = 15 * 60 * 1000 + const follows = new Map() // The SDK validates every frame against the event schema; a nonconforming frame kills the // stream, so synthetic events carry the required id and full property sets. let counter = 0 @@ -465,27 +475,39 @@ function createSessionProjector(origin: string, headers: HeadersInit | undefined type, properties, }) - const resync = (sessionID: string, dir: string) => { + const resync = (sessionID: string, dir: string, delay = 250) => { clearTimeout(timers.get(sessionID)) timers.set( sessionID, setTimeout(async () => { timers.delete(sessionID) + let settled = false try { const [info, items] = await Promise.all([ v2(origin, `/api/session/${sessionID}`, null, headers), v2(origin, `/api/session/${sessionID}/message`, null, headers), ]) emit(dir, legacyEvent("session.updated", { sessionID, info: v1Session(info) })) - for (const message of v1Messages(sessionID, items as any[])) { + const messages = v1Messages(sessionID, items as any[]) + for (const message of messages) { emit(dir, legacyEvent("message.updated", { sessionID, info: message.info })) for (const part of message.parts) emit(dir, legacyEvent("message.part.updated", { sessionID, part, time: Date.now() })) } + const latest = messages.at(-1)?.info + settled = latest?.role === "assistant" && Boolean(latest.time?.completed) } catch {} - }, 250), + const deadline = follows.get(sessionID) + if (deadline === undefined) return + if (settled || Date.now() > deadline) follows.delete(sessionID) + else resync(sessionID, dir, FOLLOW_INTERVAL) + }, delay), ) } + const follow = (sessionID: string, dir: string) => { + follows.set(sessionID, Date.now() + FOLLOW_DEADLINE) + resync(sessionID, dir) + } const handle = (event: any) => { const type = event?.type if (typeof type !== "string") return @@ -517,6 +539,7 @@ function createSessionProjector(origin: string, headers: HeadersInit | undefined time: Date.now(), }), ) + follow(sessionID, dir) return } if (type === "session.next.step.started" && data.assistantMessageID) { @@ -557,12 +580,13 @@ function createSessionProjector(origin: string, headers: HeadersInit | undefined ) return } - resync(sessionID, dir) + follow(sessionID, dir) } return Object.assign(handle, { dispose() { for (const timer of timers.values()) clearTimeout(timer) timers.clear() + follows.clear() }, }) } From f85e59a8fb788713b6f396f13ce7f7b3f4eddd52 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 17 Aug 2026 01:28:08 -0700 Subject: [PATCH 099/103] Stopped fabricating an agent and model on projected user messages. The prompt bar syncs its selection from the last user message; the invented values clobbered the user's choice and the empty model rendered a 'Model / is not valid' toast on every turn. An empty agent short-circuits that sync. --- packages/cli/src/tui.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/tui.ts b/packages/cli/src/tui.ts index dba9e404c5cc..fe6fe5ae6e70 100644 --- a/packages/cli/src/tui.ts +++ b/packages/cli/src/tui.ts @@ -213,13 +213,15 @@ function v1AssistantInfo(sessionID: string, item: any) { } function v1UserInfo(sessionID: string, item: any) { + // No fabricated agent or model: the prompt bar syncs its selection from the last user message, + // so an invented value would clobber the user's choice (and an empty model renders a + // "Model / is not valid" toast). An empty agent short-circuits that sync. return { id: item.id, sessionID, role: "user", time: { created: epoch(item.time?.created) }, - agent: "build", - model: { providerID: "", modelID: "" }, + agent: "", } } From f3cd27a34ded79b952dff72a24c974235ec80c8b Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 17 Aug 2026 11:18:11 -0700 Subject: [PATCH 100/103] Confirmed settlement with a second read; removed coalesced parts. The read model can show an assistant's completed flag before the settled row's rewritten content, so a follow that trusted one read stopped with the reply's tail missing. It now stops only after two identical settled reads, and it emits part removals when the settled rewrite coalesces streamed parts under new ids, so no stale text lingers. --- packages/cli/src/tui.ts | 46 ++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/tui.ts b/packages/cli/src/tui.ts index fe6fe5ae6e70..17cde7afa44f 100644 --- a/packages/cli/src/tui.ts +++ b/packages/cli/src/tui.ts @@ -468,7 +468,13 @@ function createSessionProjector(origin: string, headers: HeadersInit | undefined const FOLLOW_INTERVAL = 1200 // A turn longer than this stops refreshing a cross-process TUI until the next admission. const FOLLOW_DEADLINE = 15 * 60 * 1000 - const follows = new Map() + // deadline plus the signature of the last settled read: the read model can show the completed + // flag before the settled row's rewritten content, so a follow only stops after two identical + // settled reads. + const follows = new Map() + // Part ids emitted per message, so a settled rewrite that coalesces parts under new ids also + // removes the stale ones from the store instead of leaving duplicated text. + const emittedParts = new Map>>() // The SDK validates every frame against the event schema; a nonconforming frame kills the // stream, so synthetic events carry the required id and full property sets. let counter = 0 @@ -484,6 +490,7 @@ function createSessionProjector(origin: string, headers: HeadersInit | undefined setTimeout(async () => { timers.delete(sessionID) let settled = false + let signature = "" try { const [info, items] = await Promise.all([ v2(origin, `/api/session/${sessionID}`, null, headers), @@ -491,23 +498,47 @@ function createSessionProjector(origin: string, headers: HeadersInit | undefined ]) emit(dir, legacyEvent("session.updated", { sessionID, info: v1Session(info) })) const messages = v1Messages(sessionID, items as any[]) + const seen = emittedParts.get(sessionID) ?? new Map>() + emittedParts.set(sessionID, seen) for (const message of messages) { emit(dir, legacyEvent("message.updated", { sessionID, info: message.info })) - for (const part of message.parts) + const ids = new Set() + for (const part of message.parts) { + ids.add(part.id) emit(dir, legacyEvent("message.part.updated", { sessionID, part, time: Date.now() })) + } + for (const stale of seen.get(message.info.id) ?? []) { + if (!ids.has(stale)) + emit( + dir, + legacyEvent("message.part.removed", { sessionID, messageID: message.info.id, partID: stale }), + ) + } + seen.set(message.info.id, ids) } const latest = messages.at(-1)?.info settled = latest?.role === "assistant" && Boolean(latest.time?.completed) + signature = messages + .map((m) => `${m.info.id}:${m.parts.map((part) => `${part.id}=${part.text?.length ?? part.state?.status ?? ""}`).join(",")}`) + .join(";") } catch {} - const deadline = follows.get(sessionID) - if (deadline === undefined) return - if (settled || Date.now() > deadline) follows.delete(sessionID) - else resync(sessionID, dir, FOLLOW_INTERVAL) + const follow = follows.get(sessionID) + if (follow === undefined) return + if (Date.now() > follow.deadline) { + follows.delete(sessionID) + return + } + if (settled && follow.confirmed === signature) { + follows.delete(sessionID) + return + } + follow.confirmed = settled ? signature : null + resync(sessionID, dir, FOLLOW_INTERVAL) }, delay), ) } const follow = (sessionID: string, dir: string) => { - follows.set(sessionID, Date.now() + FOLLOW_DEADLINE) + follows.set(sessionID, { deadline: Date.now() + FOLLOW_DEADLINE, confirmed: null }) resync(sessionID, dir) } const handle = (event: any) => { @@ -589,6 +620,7 @@ function createSessionProjector(origin: string, headers: HeadersInit | undefined for (const timer of timers.values()) clearTimeout(timer) timers.clear() follows.clear() + emittedParts.clear() }, }) } From 6370983c9e813f649f62905e6c81cf4c7d010e61 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 17 Aug 2026 11:32:07 -0700 Subject: [PATCH 101/103] Updated the branch docs' supervisor paths after the package move. --- packages/temporal/docs/native-local-driver-branch.md | 2 +- packages/temporal/docs/temporal-supervisor-followups.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/temporal/docs/native-local-driver-branch.md b/packages/temporal/docs/native-local-driver-branch.md index 3df87f6f8efe..67317539e10b 100644 --- a/packages/temporal/docs/native-local-driver-branch.md +++ b/packages/temporal/docs/native-local-driver-branch.md @@ -25,7 +25,7 @@ semantics (`SessionRunCoordinator`, `session/run-coordinator.ts`). `local-driver.ts` supervisor path was removed. Local mode is now the well-exercised default and its correctness comes from reused, tested code rather than a second coordination loop. -`workflow-core.ts` is therefore Temporal-only. +That supervisor is therefore Temporal-only; it now lives at `packages/temporal/src/supervisor.ts`. ### 2. A Temporal test harness, and a Temporal-mode correctness pass diff --git a/packages/temporal/docs/temporal-supervisor-followups.md b/packages/temporal/docs/temporal-supervisor-followups.md index 9e5facc8f432..ede4e77e4888 100644 --- a/packages/temporal/docs/temporal-supervisor-followups.md +++ b/packages/temporal/docs/temporal-supervisor-followups.md @@ -1,8 +1,8 @@ # Temporal supervisor: status and follow-ups Independent review (Codex, several rounds) enumerated correctness issues in the Temporal-mode session -supervisor (`packages/core/src/session/execution/workflow-core.ts` + `temporal-workflow.ts` + -`temporal.ts`). Local mode is unaffected: it runs the proven `SessionRunCoordinator` +supervisor (now `packages/temporal/src/supervisor.ts` + `workflow.ts` + `executor.ts`). +Local mode is unaffected: it runs the proven `SessionRunCoordinator` (`execution/local.ts`). A deterministic test harness now exists for this code: fake-runtime unit tests of the supervisor loop From 259a0f96ee57a6980e43afc7db5887848d9242d3 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 17 Aug 2026 13:12:34 -0700 Subject: [PATCH 102/103] Made question asks durable rows, mirroring permission requests. A question raised inside an activity on one worker was parked on an in-memory deferred, so no other process could list or answer it and the step stayed blocked. Deterministic ids let a re-driven attempt adopt the same row or return answers that already landed, which also makes the question tool safe to mark idempotent for crash-resume. --- packages/core/schema.json | 119 +++++++++- packages/core/src/database/migration.gen.ts | 1 + .../20260817200644_question_request.ts | 25 ++ packages/core/src/database/schema.gen.ts | 15 ++ packages/core/src/question.ts | 214 +++++++++++++++--- packages/core/src/question/sql.ts | 23 ++ packages/core/src/tool/question.ts | 4 + packages/core/test/question-durable.test.ts | 207 +++++++++++++++++ packages/core/test/question.test.ts | 24 +- 9 files changed, 586 insertions(+), 46 deletions(-) create mode 100644 packages/core/src/database/migration/20260817200644_question_request.ts create mode 100644 packages/core/src/question/sql.ts create mode 100644 packages/core/test/question-durable.test.ts diff --git a/packages/core/schema.json b/packages/core/schema.json index 01aeeffb9540..df1c4c21a05d 100644 --- a/packages/core/schema.json +++ b/packages/core/schema.json @@ -1,9 +1,9 @@ { "version": "7", "dialect": "sqlite", - "id": "c609b4b3-8061-4cef-a339-c44d54121462", + "id": "c1e14245-625d-4a2c-b3dd-aa10a72f5c0c", "prevIds": [ - "69462d47-737f-4005-83ff-1f43cf647276" + "c609b4b3-8061-4cef-a339-c44d54121462" ], "ddl": [ { @@ -54,6 +54,10 @@ "name": "project", "entityType": "tables" }, + { + "name": "question_request", + "entityType": "tables" + }, { "name": "message", "entityType": "tables" @@ -860,6 +864,76 @@ "entityType": "columns", "table": "project" }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "question_request" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "question_request" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "payload", + "entityType": "columns", + "table": "question_request" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'pending'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "question_request" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "answers", + "entityType": "columns", + "table": "question_request" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "question_request" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "question_request" + }, { "type": "text", "notNull": false, @@ -1965,6 +2039,15 @@ "table": "project", "entityType": "pks" }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "question_request_pk", + "table": "question_request", + "entityType": "pks" + }, { "columns": [ "id" @@ -2131,6 +2214,38 @@ "entityType": "indexes", "table": "permission" }, + { + "columns": [ + { + "value": "session_id", + "isExpression": false + }, + { + "value": "status", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "question_request_session_status_idx", + "entityType": "indexes", + "table": "question_request" + }, + { + "columns": [ + { + "value": "status", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "question_request_status_idx", + "entityType": "indexes", + "table": "question_request" + }, { "columns": [ { diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index c1cfbf9a0c7e..159c9c6660b2 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -43,5 +43,6 @@ export const migrations = ( import("./migration/20260810092511_permission_request"), import("./migration/20260812000509_permission_request_status_idx"), import("./migration/20260812070622_snapshot_pack"), + import("./migration/20260817200644_question_request"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20260817200644_question_request.ts b/packages/core/src/database/migration/20260817200644_question_request.ts new file mode 100644 index 000000000000..35732d24f8c8 --- /dev/null +++ b/packages/core/src/database/migration/20260817200644_question_request.ts @@ -0,0 +1,25 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260817200644_question_request", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`question_request\` ( + \`id\` text PRIMARY KEY, + \`session_id\` text NOT NULL, + \`payload\` text NOT NULL, + \`status\` text DEFAULT 'pending' NOT NULL, + \`answers\` text, + \`time_created\` integer NOT NULL, + \`time_updated\` integer NOT NULL + ); + `) + yield* tx.run( + `CREATE INDEX \`question_request_session_status_idx\` ON \`question_request\` (\`session_id\`,\`status\`);`, + ) + yield* tx.run(`CREATE INDEX \`question_request_status_idx\` ON \`question_request\` (\`status\`);`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/schema.gen.ts b/packages/core/src/database/schema.gen.ts index 60a72dacac89..9a9027a4c149 100644 --- a/packages/core/src/database/schema.gen.ts +++ b/packages/core/src/database/schema.gen.ts @@ -136,6 +136,17 @@ export default { \`commands\` text ); `) + yield* tx.run(` + CREATE TABLE \`question_request\` ( + \`id\` text PRIMARY KEY, + \`session_id\` text NOT NULL, + \`payload\` text NOT NULL, + \`status\` text DEFAULT 'pending' NOT NULL, + \`answers\` text, + \`time_created\` integer NOT NULL, + \`time_updated\` integer NOT NULL + ); + `) yield* tx.run(` CREATE TABLE \`message\` ( \`id\` text PRIMARY KEY, @@ -269,6 +280,10 @@ export default { yield* tx.run( `CREATE UNIQUE INDEX \`permission_project_action_resource_idx\` ON \`permission\` (\`project_id\`,\`action\`,\`resource\`);`, ) + yield* tx.run( + `CREATE INDEX \`question_request_session_status_idx\` ON \`question_request\` (\`session_id\`,\`status\`);`, + ) + yield* tx.run(`CREATE INDEX \`question_request_status_idx\` ON \`question_request\` (\`status\`);`) yield* tx.run( `CREATE INDEX \`message_session_time_created_id_idx\` ON \`message\` (\`session_id\`,\`time_created\`,\`id\`);`, ) diff --git a/packages/core/src/question.ts b/packages/core/src/question.ts index 79e0ea5e0384..73abb6aa07a4 100644 --- a/packages/core/src/question.ts +++ b/packages/core/src/question.ts @@ -1,9 +1,13 @@ export * as QuestionV2 from "./question" +import { createHash } from "node:crypto" +import { and, eq } from "drizzle-orm" import { makeLocationNode } from "./effect/app-node" import { Context, Deferred, Effect, Layer, Schema } from "effect" import { Question } from "@opencode-ai/schema/question" +import { Database } from "./database/database" import { EventV2 } from "./event" +import { QuestionRequestTable } from "./question/sql" import { SessionSchema } from "./session/schema" export const ID = Question.ID @@ -32,6 +36,13 @@ export type Reply = typeof Reply.Type export const Event = Question.Event +// A pending question older than this is treated as abandoned (the turn that raised it was +// interrupted or crashed, so nothing is waiting for the answer). Long enough that a human +// deliberating never trips it. +const PENDING_TTL_MS = 24 * 60 * 60 * 1000 + +const Answers = Schema.Array(Question.Answer) + export class RejectedError extends Schema.TaggedErrorClass()("QuestionV2.RejectedError", {}) { override get message() { return "The user dismissed this question" @@ -67,21 +78,89 @@ interface Pending { readonly deferred: Deferred.Deferred, RejectedError> } -/** - * Location-owned pending prompts. The Location layer map must materialize this - * layer once per embedded Location so replies cannot settle another Location's - * deferred request. - */ +// A tool-originated ask gets a DETERMINISTIC id (session + callID + question texts), so a re-driven +// activity resolves to the same durable row instead of filing a duplicate: answers that landed +// while the asker was dead are honored on the retry, and a still-pending row is adopted rather +// than re-asked. Asks without a tool source keep random ids. +function deterministicID(input: AskInput) { + if (!input.tool?.callID) return undefined + const digest = createHash("sha256") + .update([input.sessionID, input.tool.callID, ...input.questions.map((item) => item.question)].join("\u0000")) + .digest("hex") + return ID.ascending(`que_${digest.slice(0, 26)}`) +} + const layer = Layer.effect( Service, Effect.gen(function* () { const events = yield* EventV2.Service + const db = (yield* Database.Service).db const pending = new Map() + // Pending questions are durable rows in the shared store, so an ask raised in one process (a + // standalone worker's activity) is visible and answerable from another (the HTTP server), and + // an answer lands even after the asking process restarted. The in-memory deferred stays as the + // same-process fast path; a cross-process answer is observed by polling the row. + const readRow = (id: string) => + db + .select() + .from(QuestionRequestTable) + .where(eq(QuestionRequestTable.id, id)) + .all() + .pipe( + Effect.orDie, + Effect.map((rows) => rows[0]), + ) + // Status moves are compare-and-set on the stated `from`, so a reply that lost a race, or a + // shutdown racing an answer, cannot overwrite a landed outcome. + const transitionRow = (id: string, from: string, to: string, answers?: string) => + db + .update(QuestionRequestTable) + .set({ status: to, answers: answers ?? null }) + .where(and(eq(QuestionRequestTable.id, id), eq(QuestionRequestTable.status, from))) + .run() + .pipe(Effect.orDie) + // An interrupted turn leaves its questions pending with nothing waiting for the answers; a row + // untouched past the TTL is treated as abandoned. Reads sweep them lazily. + const pendingRows = () => + Effect.gen(function* () { + const cutoff = Date.now() - PENDING_TTL_MS + const rows = yield* db + .select() + .from(QuestionRequestTable) + .where(eq(QuestionRequestTable.status, "pending")) + .all() + .pipe(Effect.orDie) + const fresh: typeof rows = [] + for (const row of rows) { + if (row.time_updated < cutoff) yield* transitionRow(row.id, "pending", "expired") + else fresh.push(row) + } + return fresh + }) + const decodeRow = (row: { payload: string }) => + Schema.decodeUnknownSync(Request)(JSON.parse(row.payload)) as Request + const decodeAnswers = (raw: string | null): ReadonlyArray => + raw ? (Schema.decodeUnknownSync(Answers)(JSON.parse(raw)) as ReadonlyArray) : [] + yield* Effect.addFinalizer(() => - Effect.forEach(pending.values(), (item) => Deferred.fail(item.deferred, new RejectedError()), { - discard: true, - }).pipe( + Effect.forEach( + pending.values(), + (item) => + // Graceful shutdown: an answer may have landed on the row before the local poll saw it. + // Honor it, or the shutdown rejects a question the user already answered. Only a row that + // is still pending gets retired. + Effect.gen(function* () { + const row = yield* readRow(item.request.id) + if (row?.status === "answered") { + yield* Deferred.succeed(item.deferred, decodeAnswers(row.answers)) + return + } + yield* transitionRow(item.request.id, "pending", "expired") + yield* Deferred.fail(item.deferred, new RejectedError()) + }).pipe(Effect.catch(() => Effect.void)), + { discard: true }, + ).pipe( Effect.ensuring( Effect.sync(() => { pending.clear() @@ -90,18 +169,75 @@ const layer = Layer.effect( ), ) + const create = (request: Request) => + Effect.uninterruptible( + Effect.gen(function* () { + // A retry can land in this process while the prior attempt's waiter is still parked + // (deterministic ids make them the same ask). Share the waiter instead of dying. + const parked = pending.get(request.id) + if (parked) return parked + const deferred = yield* Deferred.make, RejectedError>() + const item: Pending = { request, deferred } + pending.set(request.id, item) + yield* db + .insert(QuestionRequestTable) + .values({ + id: request.id, + session_id: request.sessionID, + payload: JSON.stringify(Schema.encodeSync(Request)(request)), + }) + .onConflictDoNothing() + .run() + .pipe( + Effect.orDie, + Effect.onError(() => Effect.sync(() => pending.delete(request.id))), + ) + // A deterministic id can collide with its own expired row (a prior attempt shut down + // gracefully); revive it so the reply path and pollers see one pending ask again. + yield* db + .update(QuestionRequestTable) + .set({ status: "pending", answers: null }) + .where(and(eq(QuestionRequestTable.id, request.id), eq(QuestionRequestTable.status, "expired"))) + .run() + .pipe(Effect.orDie) + yield* events + .publish(Event.Asked, request) + .pipe(Effect.onError(() => Effect.sync(() => pending.delete(request.id)))) + return item + }), + ) + + // Observe a cross-process answer: the replying process updates the row, not our deferred. + const awaitRow = (id: ID): Effect.Effect, RejectedError> => + Effect.gen(function* () { + for (;;) { + const row = yield* readRow(id) + if (row && row.status !== "pending") { + if (row.status === "answered") return decodeAnswers(row.answers) + return yield* new RejectedError() + } + yield* Effect.sleep(500) + } + }) + const ask = Effect.fn("QuestionV2.ask")((input: AskInput) => Effect.uninterruptibleMask((restore) => Effect.gen(function* () { - const id = ID.ascending() - const deferred = yield* Deferred.make, RejectedError>() - const request: Request = { id, ...input } - pending.set(id, { request, deferred }) - return yield* events.publish(Event.Asked, request).pipe( - Effect.andThen(restore(Deferred.await(deferred))), + const request: Request = { id: deterministicID(input) ?? ID.ascending(), ...input } + // A deterministic id may already have a settled or in-flight row from a prior attempt of + // the same call: honor answers that landed while the asker was dead, and adopt a pending + // row instead of duplicating the ask. + const existing = yield* readRow(request.id) + if (existing) { + if (existing.status === "answered") return decodeAnswers(existing.answers) + if (existing.status === "rejected") return yield* new RejectedError() + // pending or expired: fall through; create adopts (insert no-ops) or revives the row. + } + const item = yield* create(request) + return yield* restore(Effect.raceFirst(Deferred.await(item.deferred), awaitRow(item.request.id))).pipe( Effect.ensuring( Effect.sync(() => { - pending.delete(id) + pending.delete(item.request.id) }), ), ) @@ -109,18 +245,36 @@ const layer = Layer.effect( ), ) + // Complete the local waiter if the ask was raised in this process; a cross-process waiter + // observes the row update through its poll. + const settleLocal = (id: ID, complete: (deferred: Pending["deferred"]) => Effect.Effect) => + Effect.suspend(() => { + const item = pending.get(id) + if (!item) return Effect.void + pending.delete(id) + return Effect.asVoid(complete(item.deferred)) + }) + + // The durable row is the source of truth, so a reply works from any process (the HTTP server + // answering a question raised inside a standalone worker's activity), not just the asking one. const reply = Effect.fn("QuestionV2.reply")((input: ReplyInput) => Effect.uninterruptible( Effect.gen(function* () { - const existing = pending.get(input.requestID) - if (!existing) return yield* new NotFoundError({ requestID: input.requestID }) + const row = yield* readRow(input.requestID) + if (!row || row.status !== "pending") return yield* new NotFoundError({ requestID: input.requestID }) + const existing = decodeRow(row) yield* events.publish(Event.Replied, { - sessionID: existing.request.sessionID, - requestID: existing.request.id, + sessionID: existing.sessionID, + requestID: existing.id, answers: input.answers.map((answer) => [...answer]), }) - yield* Deferred.succeed(existing.deferred, input.answers) - pending.delete(input.requestID) + yield* transitionRow( + existing.id, + "pending", + "answered", + JSON.stringify(Schema.encodeSync(Answers)(input.answers)), + ) + yield* settleLocal(existing.id, (deferred) => Deferred.succeed(deferred, input.answers)) }), ), ) @@ -128,20 +282,22 @@ const layer = Layer.effect( const reject = Effect.fn("QuestionV2.reject")((requestID: ID) => Effect.uninterruptible( Effect.gen(function* () { - const existing = pending.get(requestID) - if (!existing) return yield* new NotFoundError({ requestID }) + const row = yield* readRow(requestID) + if (!row || row.status !== "pending") return yield* new NotFoundError({ requestID }) + const existing = decodeRow(row) yield* events.publish(Event.Rejected, { - sessionID: existing.request.sessionID, - requestID: existing.request.id, + sessionID: existing.sessionID, + requestID: existing.id, }) - yield* Deferred.fail(existing.deferred, new RejectedError()) - pending.delete(requestID) + yield* transitionRow(existing.id, "pending", "rejected") + yield* settleLocal(existing.id, (deferred) => Deferred.fail(deferred, new RejectedError())) }), ), ) + // Reads come from the durable rows, so serve can list questions raised by any worker. const list = Effect.fn("QuestionV2.list")(function* () { - return Array.from(pending.values(), (item) => item.request) + return (yield* pendingRows()).map(decodeRow) }) return Service.of({ ask, reply, reject, list }) @@ -150,4 +306,4 @@ const layer = Layer.effect( export const locationLayer = layer -export const node = makeLocationNode({ service: Service, layer, deps: [EventV2.node] }) +export const node = makeLocationNode({ service: Service, layer, deps: [EventV2.node, Database.node] }) diff --git a/packages/core/src/question/sql.ts b/packages/core/src/question/sql.ts new file mode 100644 index 000000000000..6dc08a6e0890 --- /dev/null +++ b/packages/core/src/question/sql.ts @@ -0,0 +1,23 @@ +import { index, sqliteTable, text } from "drizzle-orm/sqlite-core" +import { Timestamps } from "../database/schema.sql" + +// A durable pending-question record, so an ask raised by one process (a standalone worker's +// activity) can be listed and answered from another (the HTTP server) via the shared store, and +// survives the asking process. `payload` is the JSON-encoded QuestionV2.Request; `answers` holds +// the JSON-encoded reply once one lands; status transitions pending -> answered | rejected | expired. +export const QuestionRequestTable = sqliteTable( + "question_request", + { + id: text().primaryKey(), + session_id: text().notNull(), + payload: text().notNull(), + status: text().notNull().default("pending"), + answers: text(), + ...Timestamps, + }, + (table) => [ + index("question_request_session_status_idx").on(table.session_id, table.status), + // The no-session list() and the TTL sweep both filter by status alone. + index("question_request_status_idx").on(table.status), + ], +) diff --git a/packages/core/src/tool/question.ts b/packages/core/src/tool/question.ts index e5ae0d7426a6..44b995f8fe34 100644 --- a/packages/core/src/tool/question.ts +++ b/packages/core/src/tool/question.ts @@ -56,6 +56,10 @@ const layer = Layer.effectDiscard( description, input: Input, output: Output, + // Re-running the ask is safe: the durable question row has a deterministic id, so a + // crash-resume adopts the pending row (or returns answers that already landed) instead + // of failing the user's open questions. + idempotent: true, toModelOutput: ({ input, output }) => [ { type: "text", text: toModelOutput(input.questions, output.answers) }, ], diff --git a/packages/core/test/question-durable.test.ts b/packages/core/test/question-durable.test.ts new file mode 100644 index 000000000000..cbfbb89537bf --- /dev/null +++ b/packages/core/test/question-durable.test.ts @@ -0,0 +1,207 @@ +// Durable question asks: a pending question is a row in the shared store, so an ask raised by one +// process (a standalone worker's activity) can be listed and answered from another (the HTTP +// server), and the blocked ask observes the cross-process answer by polling the row. Two fully +// independent service stacks share one DB file to simulate the two processes. +import { describe, expect } from "bun:test" +import path from "path" +import { createClient } from "@libsql/client" +import { Cause, Context, Effect, Exit, Fiber, Layer, Scope } from "effect" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { Database } from "@opencode-ai/core/database/database" +import { QuestionV2 } from "@opencode-ai/core/question" +import { SessionV2 } from "@opencode-ai/core/session" +import { testEffect } from "./lib/effect" +import { tmpdir } from "./fixture/tmpdir" + +const sessionID = SessionV2.ID.make("ses_question_durable") +const question: QuestionV2.Info = { + question: "Which option?", + header: "Option", + options: [{ label: "One", description: "First option" }], +} + +const stack = (file: string) => + AppNodeBuilder.build(QuestionV2.node, [[Database.node, Database.layerFromPath(file)]]) + +const it = testEffect(Layer.empty) + +const awaitAsk = (service: QuestionV2.Interface) => + Effect.gen(function* () { + for (;;) { + const asks = yield* service.list() + const ask = asks[0] + if (ask) return ask + yield* Effect.sleep(50) + } + }) + +describe("QuestionV2 durable asks", () => { + it.live("unblocks an ask via an answer from a second process sharing the store", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + const A = yield* Layer.build(stack(file)) + const B = yield* Layer.build(stack(file)) + const questionA = Context.get(A, QuestionV2.Service) + const questionB = Context.get(B, QuestionV2.Service) + + // A: a tool blocks on the questions. B: a different process sees the durable ask and answers. + const blocked = yield* questionA.ask({ sessionID, questions: [question] }).pipe(Effect.forkChild) + const ask = yield* awaitAsk(questionB) + expect(ask.sessionID).toBe(sessionID) + expect(ask.questions).toEqual([question]) + yield* questionB.reply({ requestID: ask.id, answers: [["One"]] }) + const exit = yield* Fiber.await(blocked) + expect(Exit.isSuccess(exit)).toBe(true) + if (Exit.isSuccess(exit)) expect(exit.value).toEqual([["One"]]) + // The row is settled everywhere: no pending asks remain on either side. + expect(yield* questionA.list()).toEqual([]) + expect(yield* questionB.list()).toEqual([]) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + it.live("delivers a cross-process rejection as the typed RejectedError", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + const A = yield* Layer.build(stack(file)) + const B = yield* Layer.build(stack(file)) + const questionA = Context.get(A, QuestionV2.Service) + const questionB = Context.get(B, QuestionV2.Service) + + const blocked = yield* questionA.ask({ sessionID, questions: [question] }).pipe(Effect.forkChild) + const ask = yield* awaitAsk(questionB) + yield* questionB.reject(ask.id) + const exit = yield* Fiber.await(blocked) + expect(Exit.isFailure(exit)).toBe(true) + const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined + expect(error).toBeInstanceOf(QuestionV2.RejectedError) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + it.live("a re-drive adopts the same pending ask instead of duplicating it", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + const A = yield* Layer.build(stack(file)) + const B = yield* Layer.build(stack(file)) + const questionA = Context.get(A, QuestionV2.Service) + const questionB = Context.get(B, QuestionV2.Service) + const input: QuestionV2.AskInput = { + sessionID, + questions: [question], + tool: { messageID: "msg_1", callID: "call_redrive" }, + } + + // Attempt 1 blocks, then dies (a crashed activity): the row stays pending. + const first = yield* questionA.ask(input).pipe(Effect.forkChild) + const ask = yield* awaitAsk(questionB) + yield* Fiber.interrupt(first) + // Attempt 2 (the Temporal retry) files the same deterministic ask: one row, same id. + const second = yield* questionA.ask(input).pipe(Effect.forkChild) + yield* Effect.sleep(100) + const asks = yield* questionB.list() + expect(asks).toHaveLength(1) + expect(asks[0]?.id).toBe(ask.id) + yield* questionB.reply({ requestID: ask.id, answers: [["One"]] }) + const exit = yield* Fiber.await(second) + expect(Exit.isSuccess(exit)).toBe(true) + if (Exit.isSuccess(exit)) expect(exit.value).toEqual([["One"]]) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + it.live("honors answers that landed while the asker was dead", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + const A = yield* Layer.build(stack(file)) + const B = yield* Layer.build(stack(file)) + const questionA = Context.get(A, QuestionV2.Service) + const questionB = Context.get(B, QuestionV2.Service) + const input: QuestionV2.AskInput = { + sessionID, + questions: [question], + tool: { messageID: "msg_2", callID: "call_dead_asker" }, + } + + const first = yield* questionA.ask(input).pipe(Effect.forkChild) + const ask = yield* awaitAsk(questionB) + yield* Fiber.interrupt(first) + // The human answers after the asker died; the retry short-circuits on the answered row. + yield* questionB.reply({ requestID: ask.id, answers: [["One"]] }) + expect(yield* questionA.ask(input)).toEqual([["One"]]) + expect(yield* questionB.list()).toEqual([]) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + it.live("shutdown honors an answer that landed before the poll saw it", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + const B = yield* Layer.build(stack(file)) + const questionB = Context.get(B, QuestionV2.Service) + // A lives in its own scope so the test can shut it down while the waiter is parked. + const scope = yield* Scope.make() + const A = yield* Layer.build(stack(file)).pipe(Effect.provideService(Scope.Scope, scope)) + const questionA = Context.get(A, QuestionV2.Service) + const blocked = yield* questionA.ask({ sessionID, questions: [question] }).pipe(Effect.forkChild) + const ask = yield* awaitAsk(questionB) + yield* questionB.reply({ requestID: ask.id, answers: [["One"]] }) + // Shut A down inside the poll window: the finalizer must honor the answer on the row, not + // reject a question the user already answered. + yield* Scope.close(scope, Exit.void) + const exit = yield* Fiber.await(blocked) + expect(Exit.isSuccess(exit)).toBe(true) + if (Exit.isSuccess(exit)) expect(exit.value).toEqual([["One"]]) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + it.live("sweeps an abandoned pending ask out of the list on read", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + const A = yield* Layer.build(stack(file)) + const questionA = Context.get(A, QuestionV2.Service) + const blocked = yield* questionA.ask({ sessionID, questions: [question] }).pipe(Effect.forkChild) + const ask = yield* awaitAsk(questionA) + // Backdate the row past the TTL: the turn that raised it is gone. + yield* Effect.promise(async () => { + const raw = createClient({ url: `file:${file}` }) + await raw.execute({ + sql: "UPDATE question_request SET time_updated = ? WHERE id = ?", + args: [Date.now() - 25 * 60 * 60 * 1000, ask.id], + }) + raw.close() + }) + expect(yield* questionA.list()).toEqual([]) + yield* Fiber.interrupt(blocked) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + + it.live("expires locally-pending asks on shutdown so they do not linger as pending rows", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const file = path.join(tmp.path, "shared.db") + // A raises an ask, then its scope closes (a graceful shutdown) before anyone answers. + const scope = yield* Scope.make() + const A = yield* Layer.build(stack(file)).pipe(Effect.provideService(Scope.Scope, scope)) + const questionA = Context.get(A, QuestionV2.Service) + const blocked = yield* questionA.ask({ sessionID, questions: [question] }).pipe(Effect.forkChild) + yield* awaitAsk(questionA) + yield* Scope.close(scope, Exit.void) + const exit = yield* Fiber.await(blocked) + expect(Exit.isFailure(exit)).toBe(true) + const B = yield* Layer.build(stack(file)) + const questionB = Context.get(B, QuestionV2.Service) + // The waiter died with A, so the ask is retired, not stuck pending forever. + expect(yield* questionB.list()).toEqual([]) + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) +}) diff --git a/packages/core/test/question.test.ts b/packages/core/test/question.test.ts index 03d61a956507..64d87be80297 100644 --- a/packages/core/test/question.test.ts +++ b/packages/core/test/question.test.ts @@ -89,26 +89,20 @@ describe("QuestionV2", () => { }), ) - it.effect("isolates pending requests by location-layer instance and rejects them on finalization", () => + // Cross-instance visibility now goes through the durable rows (question-durable.test.ts); what + // finalization still owes a parked asker is a rejection, not silence. + it.effect("rejects pending asks when the owning layer finalizes", () => Effect.gen(function* () { - const firstScope = yield* Scope.make() - const secondScope = yield* Scope.make() - const first = Context.get(yield* Layer.buildWithScope(Layer.fresh(questions), firstScope), QuestionV2.Service) - const second = Context.get(yield* Layer.buildWithScope(Layer.fresh(questions), secondScope), QuestionV2.Service) - const fiber = yield* first.ask({ sessionID, questions: [question] }).pipe(Effect.forkScoped) - yield* Effect.yieldNow - const request = (yield* first.list())[0]! + const scope = yield* Scope.make() + const service = Context.get(yield* Layer.buildWithScope(Layer.fresh(questions), scope), QuestionV2.Service) + const fiber = yield* service.ask({ sessionID, questions: [question] }).pipe(Effect.forkScoped) + for (let i = 0; i < 100 && (yield* service.list()).length === 0; i++) yield* Effect.yieldNow + expect(yield* service.list()).toHaveLength(1) - expect(yield* second.list()).toEqual([]) - expect(yield* second.reply({ requestID: request.id, answers: [["One"]] }).pipe(Effect.flip)).toEqual( - new QuestionV2.NotFoundError({ requestID: request.id }), - ) - - yield* Scope.close(firstScope, Exit.void) + yield* Scope.close(scope, Exit.void) const exit = yield* Fiber.await(fiber) expect(Exit.isFailure(exit)).toBe(true) if (Exit.isFailure(exit)) expect(exit.cause.toString()).toContain("QuestionV2.RejectedError") - yield* Scope.close(secondScope, Exit.void) }), ) }) From 289f93b87d57a74077661883e52093de8ab8d4d5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 17 Aug 2026 15:37:28 -0700 Subject: [PATCH 103/103] Rendered question prompts in the TUI, local and cross-process. The sync store only knew the v1 question event names, so the v2 engine's question.v2.* events populated nothing and the dialog never opened. The reply path also called a v1 route that 404s on the v2 daemon. In temporal mode the question tool runs inside a worker activity, so its asked event never reaches the daemon's stream. The session projector now polls the durable question list during a follow and projects the ask and answer lifecycle, so a worker-raised question opens the dialog and the reply routes back to unblock the turn. --- packages/cli/src/tui.ts | 25 +++++++++++++++++++- packages/tui/src/context/sync.tsx | 8 +++++-- packages/tui/src/routes/session/question.tsx | 17 ++++++------- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/tui.ts b/packages/cli/src/tui.ts index 17cde7afa44f..6a28e79b75c7 100644 --- a/packages/cli/src/tui.ts +++ b/packages/cli/src/tui.ts @@ -475,6 +475,10 @@ function createSessionProjector(origin: string, headers: HeadersInit | undefined // Part ids emitted per message, so a settled rewrite that coalesces parts under new ids also // removes the stale ones from the store instead of leaving duplicated text. const emittedParts = new Map>>() + // Question ids surfaced per session. A question raised inside a cross-process activity publishes + // its asked event on the worker's bus, never here, so the dialog would never open. The durable + // row is readable from this daemon, so poll it and project the asked/answered lifecycle. + const questionSeen = new Map>() // The SDK validates every frame against the event schema; a nonconforming frame kills the // stream, so synthetic events carry the required id and full property sets. let counter = 0 @@ -491,6 +495,7 @@ function createSessionProjector(origin: string, headers: HeadersInit | undefined timers.delete(sessionID) let settled = false let signature = "" + let pendingQuestions = 0 try { const [info, items] = await Promise.all([ v2(origin, `/api/session/${sessionID}`, null, headers), @@ -522,13 +527,31 @@ function createSessionProjector(origin: string, headers: HeadersInit | undefined .map((m) => `${m.info.id}:${m.parts.map((part) => `${part.id}=${part.text?.length ?? part.state?.status ?? ""}`).join(",")}`) .join(";") } catch {} + try { + const pending = (await v2(origin, `/api/session/${sessionID}/question`, null, headers)) as any[] + const seen = questionSeen.get(sessionID) ?? new Set() + const current = new Set() + for (const question of pending) { + current.add(question.id) + emit(dir, legacyEvent("question.v2.asked", { id: question.id, sessionID, questions: question.questions, tool: question.tool })) + } + // A question that left the list was answered or rejected elsewhere; clear it from the store. + for (const stale of seen) { + if (!current.has(stale)) + emit(dir, legacyEvent("question.v2.replied", { sessionID, requestID: stale, answers: [] })) + } + questionSeen.set(sessionID, current) + pendingQuestions = current.size + } catch {} const follow = follows.get(sessionID) if (follow === undefined) return if (Date.now() > follow.deadline) { follows.delete(sessionID) return } - if (settled && follow.confirmed === signature) { + // A pending question keeps the follow alive even once the assistant message looks settled: + // the turn is blocked on the answer, so the next change only lands after the user replies. + if (settled && follow.confirmed === signature && pendingQuestions === 0) { follows.delete(sessionID) return } diff --git a/packages/tui/src/context/sync.tsx b/packages/tui/src/context/sync.tsx index 71e050d11e68..1b1c0c7a22ea 100644 --- a/packages/tui/src/context/sync.tsx +++ b/packages/tui/src/context/sync.tsx @@ -224,8 +224,11 @@ export const { break } + // The v2 engine publishes the versioned names; the unversioned ones are the v1 vocabulary. case "question.replied": - case "question.rejected": { + case "question.rejected": + case "question.v2.replied": + case "question.v2.rejected": { const requests = store.question[event.properties.sessionID] if (!requests) break const match = search(requests, event.properties.requestID, (r) => r.id) @@ -240,7 +243,8 @@ export const { break } - case "question.asked": { + case "question.asked": + case "question.v2.asked": { const request = event.properties const requests = store.question[request.sessionID] if (!requests) { diff --git a/packages/tui/src/routes/session/question.tsx b/packages/tui/src/routes/session/question.tsx index 5ab467a351d3..d0919146b834 100644 --- a/packages/tui/src/routes/session/question.tsx +++ b/packages/tui/src/routes/session/question.tsx @@ -47,17 +47,18 @@ export function QuestionPrompt(props: { request: QuestionRequest; directory?: st function submit() { const answers = questions().map((_, i) => store.answers[i] ?? []) - void sdk.client.question.reply({ + // The v2 daemon only serves the session-scoped question routes. + void sdk.client.v2.session.question.reply({ + sessionID: props.request.sessionID, requestID: props.request.id, - directory: props.directory, - answers, + questionV2Reply: { answers }, }) } function reject() { - void sdk.client.question.reject({ + void sdk.client.v2.session.question.reject({ + sessionID: props.request.sessionID, requestID: props.request.id, - directory: props.directory, }) } @@ -71,10 +72,10 @@ export function QuestionPrompt(props: { request: QuestionRequest; directory?: st setStore("custom", inputs) } if (single()) { - void sdk.client.question.reply({ + void sdk.client.v2.session.question.reply({ + sessionID: props.request.sessionID, requestID: props.request.id, - directory: props.directory, - answers: [[answer]], + questionV2Reply: { answers: [[answer]] }, }) return }