From 0dcd58536b7799eef29353864a9b69dd628439bd Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 25 Aug 2026 17:22:36 -0700 Subject: [PATCH 01/33] Pinned the event fence's contract for a step's concurrent writers. L2 splits one step into a model activity plus one activity per tool call, all appending to the same aggregate. The fence has to admit every writer holding the token the model attempt claimed, and still reject the attempt that one superseded. --- packages/core/test/event.test.ts | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index e5e71c9a81ad..89cdfe2c5581 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -1131,6 +1131,45 @@ describe("EventV2", () => { }), ) + it.effect("admits every writer holding the claimed token, concurrently", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const aggregateID = Session.ID.create() + // One step's writers share the token its model attempt claimed: the model call, each tool + // call, and the seal. They run as separate activities, so the fence has to admit all of them + // while still rejecting the attempt they superseded. + const step = "run-1:model-1:1" + const superseded = "run-1:model-1:0" + + yield* events.claim(aggregateID, step) + yield* Effect.all( + ["model", "tool-a", "tool-b", "seal"].map((writer) => + events + .publish(DurableMessage, durableData(aggregateID, writer)) + .pipe(Effect.provideService(EventV2.EventOwner, step)), + ), + { concurrency: "unbounded" }, + ) + const stale = yield* events + .publish(DurableMessage, durableData(aggregateID, "zombie")) + .pipe(Effect.provideService(EventV2.EventOwner, superseded), Effect.exit) + + const rows = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, aggregateID)) + .all() + .pipe(Effect.orDie) + + expect(String(stale)).toContain("Owner fence") + expect(rows.map((row) => row.seq)).toEqual([0, 1, 2, 3]) + expect(rows.map((row) => (row.data as { messageID: string }).messageID).sort()).toEqual( + ["model", "tool-a", "tool-b", "seal"].map((writer) => durableData(aggregateID, writer).messageID).sort(), + ) + }), + ) + it.effect("never fences a publish made outside a drain", () => Effect.gen(function* () { const events = yield* EventV2.Service From ffa32aa4ffef0fee06d73caef1eb528f7b713f0a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 25 Aug 2026 17:40:01 -0700 Subject: [PATCH 02/33] Let a caller take tool dispatch off the provider attempt. The model call and the tools it asks for were one unit, so a durable executor could not put a retry policy, an approval or a budget between them. runModelCall performs the attempt, records each call as Tool.Called and hands it back with the provider's own callID, leaving the step open for whoever runs it. The prologue and the continuation decision are extracted rather than copied, so the whole-step path and the model-only path cannot drift. --- packages/core/src/session/runner/index.ts | 36 +++ packages/core/src/session/runner/llm.ts | 108 ++++++-- .../src/session/runner/publish-llm-event.ts | 9 +- .../test/session-runner-model-call.test.ts | 250 ++++++++++++++++++ 4 files changed, 376 insertions(+), 27 deletions(-) create mode 100644 packages/core/test/session-runner-model-call.test.ts diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index e3ef80b67d07..debceab1d4cf 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -6,6 +6,7 @@ import { SessionSchema } from "../schema" import type { ContextSnapshotDecodeError, MessageDecodeError, SessionRunDeclinedError } from "../error" import type { SessionInput } from "../input" import { SessionRunnerModel } from "./model" +import type { StepSettlement } from "./publish-llm-event" import type { SystemContext } from "../../system-context/index" import type { ToolOutputStore } from "../../tool-output-store" @@ -35,6 +36,37 @@ export interface StepResult { readonly promotion: SessionInput.Delivery | undefined } +/** A tool call the provider asked for, recorded as Tool.Called but not run, handed to the caller to + * dispatch. Every id comes from the provider or the publisher and is carried, never regenerated: a + * second run of the same step would mint different ones and the results would not match the log. */ +export interface DeferredToolCall { + readonly id: string + readonly name: string + readonly input: unknown + readonly assistantMessageID: string +} + +/** What one provider attempt produced. `calls` is empty unless the caller deferred dispatch, in + * which case the step is left open and `settlement` is what seals it. */ +export interface TurnAttemptResult { + readonly needsContinuation: boolean + readonly step: number + readonly calls: ReadonlyArray + readonly settlement?: StepSettlement +} + +/** What a model-only attempt produced. `settled` means the step is already over (a crashed step was + * finalized from the log, or the recovery gate found no work) and there is nothing to dispatch. + * `called` hands back the recorded calls plus the settlement whoever seals the step will need. */ +export type ModelCallResult = + | { readonly kind: "settled"; readonly result: StepResult } + | { + readonly kind: "called" + readonly step: number + readonly calls: ReadonlyArray + readonly settlement?: StepSettlement + } + /** Runs one local continuation from already-recorded Session history. */ export interface Interface { /** Drains eligible durable work. Explicit runs perform one provider attempt even when no work is eligible. */ @@ -45,6 +77,10 @@ export interface Interface { /** Run exactly one step and report the next loop state, so a caller (e.g. a Temporal workflow) * can drive the turn one step at a time. Mirrors one iteration of `run`'s loop. */ readonly runStep: (input: StepInput) => Effect.Effect + /** Run the provider attempt of one step and stop, handing back the tool calls it asked for instead + * of running them. The caller dispatches each one and then seals the step. This is what puts the + * model-to-tools loop in a durable executor's hands rather than inside a single activity. */ + readonly runModelCall: (input: StepInput) => Effect.Effect } export class Service extends Context.Service()("@opencode/v2/SessionRunner") {} diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index f49205e2b8ef..c832516f9e3a 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -32,7 +32,14 @@ import { SessionInput } from "../input" import { SessionMessage } from "../message" import { SessionSchema } from "../schema" import { SessionStore } from "../store" -import { type RunError, Service } from "./index" +import { + type DeferredToolCall, + type ModelCallResult, + type RunError, + type StepInput, + type TurnAttemptResult, + Service, +} from "./index" import { SessionRunnerModel } from "./model" import { createLLMEventPublisher, emitToolResult } from "./publish-llm-event" import { toLLMMessages } from "./to-llm-message" @@ -201,6 +208,9 @@ const layer = Layer.effect( promotion: SessionInput.Delivery | undefined, step: number, recoverOverflow?: typeof compaction.compactAfterOverflow, + // When set, tool calls are recorded and returned instead of run, and the step is left open for + // whoever runs them. This is what lets a durable executor make each call its own unit of work. + deferTools = false, ) { const session = yield* getSession(sessionID) if (session.location.directory !== location.directory || session.location.workspaceID !== location.workspaceID) @@ -208,6 +218,7 @@ const layer = Layer.effect( const agent = yield* agents.select(session.agent) const initialized = yield* SessionContextEpoch.initialize(db, loadSystemContext(agent), session.id) const toolFibers = yield* FiberSet.make() + const deferred: DeferredToolCall[] = [] let needsContinuation = false let currentStep = step if (promotion) { @@ -283,6 +294,13 @@ const layer = Layer.effect( } needsContinuation = true const assistantMessageID = yield* publisher.assistantMessageID(event.id) + // Tool.Called is already durable (the publish above), so handing the call back is enough + // for the caller to run it later. Nothing forks here, which is why the step ends when the + // stream does and the overlap between the model and its tools is lost. + if (deferTools) { + deferred.push({ id: event.id, name: event.name, input: event.input, assistantMessageID }) + return + } yield* Effect.uninterruptibleMask((restore) => restore( toolMaterialization.settle({ @@ -350,7 +368,9 @@ const layer = Layer.effect( yield* withPublication(publisher.failUnsettledTools(`Tool execution failed: ${message}`)) } const stepSettlement = publisher.stepSettlement() - if (stepSettlement && !publisher.hasProviderError()) { + // Deferred tools have not run yet, so the step is not over and the end snapshot would be + // taken before their side effects. Whoever runs them seals it, with the settlement below. + if (stepSettlement && !publisher.hasProviderError() && !deferTools) { const endSnapshot = yield* snapshots.capture() // Ship the post-step tree: this is the state a resumed step on another host needs. if (endSnapshot) yield* snapshotSync.push(endSnapshot) @@ -380,7 +400,12 @@ const layer = Layer.effect( if (stream._tag === "Failure") return yield* Effect.failCause(stream.cause) if (settled._tag === "Failure" && Cause.hasInterrupts(settled.cause)) return yield* Effect.failCause(settled.cause) - return { needsContinuation: !publisher.hasProviderError() && needsContinuation, step: currentStep } + return { + needsContinuation: !publisher.hasProviderError() && needsContinuation, + step: currentStep, + calls: deferred as ReadonlyArray, + settlement: stepSettlement, + } }), ) }, Effect.scoped) @@ -388,31 +413,32 @@ const layer = Layer.effect( sessionID: SessionSchema.ID, promotion: SessionInput.Delivery | undefined, step: number, - ) => Effect.Effect<{ readonly needsContinuation: boolean; readonly step: number }, RunError> + deferTools?: boolean, + ) => Effect.Effect - const runAfterOverflowCompaction: RunTurn = Effect.fnUntraced(function* (sessionID, promotion, step) { - return yield* runTurnAttempt(sessionID, promotion, step).pipe( + const runAfterOverflowCompaction: RunTurn = Effect.fnUntraced(function* (sessionID, promotion, step, deferTools) { + return yield* runTurnAttempt(sessionID, promotion, step, undefined, deferTools).pipe( Effect.catchDefect( Effect.fnUntraced(function* (defect) { if (!(defect instanceof TurnTransitionError)) return yield* Effect.die(defect) if (defect.transition._tag === "ContinueAfterOverflowCompaction") return yield* Effect.die("Post-compaction provider attempt cannot recover another overflow") yield* Effect.yieldNow - return yield* runAfterOverflowCompaction(sessionID, undefined, defect.transition.step) + return yield* runAfterOverflowCompaction(sessionID, undefined, defect.transition.step, deferTools) }), ), ) }) - const runTurn: RunTurn = Effect.fnUntraced(function* (sessionID, promotion, step) { - return yield* runTurnAttempt(sessionID, promotion, step, compaction.compactAfterOverflow).pipe( + const runTurn: RunTurn = Effect.fnUntraced(function* (sessionID, promotion, step, deferTools) { + return yield* runTurnAttempt(sessionID, promotion, step, compaction.compactAfterOverflow, deferTools).pipe( Effect.catchDefect( Effect.fnUntraced(function* (defect) { if (!(defect instanceof TurnTransitionError)) return yield* Effect.die(defect) yield* Effect.yieldNow if (defect.transition._tag === "ContinueAfterOverflowCompaction") - return yield* runAfterOverflowCompaction(sessionID, undefined, defect.transition.step) - return yield* runTurn(sessionID, undefined, defect.transition.step) + return yield* runAfterOverflowCompaction(sessionID, undefined, defect.transition.step, deferTools) + return yield* runTurn(sessionID, undefined, defect.transition.step, deferTools) }), ), ) @@ -545,20 +571,17 @@ const layer = Layer.effect( // One iteration of `run`'s loop, exposed so a Temporal workflow can drive the turn one step at // a time (each step = one runTurn = one provider attempt + its tools). Semantics match `run`. - const runStep = Effect.fn("SessionRunner.runStep")(function* (input: { - readonly sessionID: SessionSchema.ID - readonly step: number - readonly promotion: SessionInput.Delivery | undefined - readonly first: boolean - readonly force: boolean - }) { + // The checks every step runs before the provider is called. Shared by the whole-step path and + // the model-only path so the two cannot drift apart. Returns the terminal result when the step + // is already over, otherwise the promotion the turn should apply. + const stepPrologue = Effect.fn("SessionRunner.stepPrologue")(function* (input: StepInput) { // One projected-history load serves all the entry checks; nothing mutates the projection // between them. The turn itself reloads after the first mutation. const entryContext = yield* getContext(input.sessionID) // Re-drive of a crashed step: finalize it from the log rather than re-calling the model and // re-running its already-dispatched tools. const resumed = yield* resumeCrashedStep(input, entryContext) - if (resumed) return resumed + if (resumed) return { kind: "settled", result: resumed } as const let promotion = input.promotion if (input.first) { const hasSteer = yield* SessionInput.hasPending(db, input.sessionID, "steer") @@ -572,7 +595,10 @@ const layer = Layer.effect( !hasQueue && !(yield* hasRecoverableWork(input.sessionID, entryContext)) ) - return { ran: false, continue: false, step: input.step, promotion: undefined } + return { + kind: "settled", + result: { ran: false, continue: false, step: input.step, promotion: undefined }, + } as const promotion = hasSteer ? "steer" : hasQueue ? "queue" : undefined } // Close tools left pending/running by an interrupted attempt before every turn, not just the @@ -580,19 +606,49 @@ const layer = Layer.effect( // re-stream a request with a dangling tool_use and no tool_result, which the provider rejects // -- a retry poison loop. This is a no-op on a healthy step (the prior step settled its tools). yield* failInterruptedTools(input.sessionID, entryContext) - const result = yield* runTurn(input.sessionID, promotion, input.step) - let needsContinuation = result.needsContinuation - if (!needsContinuation) needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer") + return { kind: "run", promotion } as const + }) + + // What the loop does once a step's provider attempt and its tools are done: a pending steer wins + // over a queued prompt, and either one keeps the turn going. Shared for the same reason as the + // prologue, and read after the tools have run so work admitted during the step is seen. + const stepContinuation = Effect.fn("SessionRunner.stepContinuation")(function* ( + sessionID: SessionSchema.ID, + hadToolCalls: boolean, + step: number, + ) { + let needsContinuation = hadToolCalls + if (!needsContinuation) needsContinuation = yield* SessionInput.hasPending(db, sessionID, "steer") if (needsContinuation) - return { ran: true, continue: true, step: result.step + 1, promotion: "steer" as SessionInput.Delivery } - const moreQueue = yield* SessionInput.hasPending(db, input.sessionID, "queue") + return { ran: true, continue: true, step: step + 1, promotion: "steer" as SessionInput.Delivery } + const moreQueue = yield* SessionInput.hasPending(db, sessionID, "queue") if (moreQueue) return { ran: true, continue: true, step: 1, promotion: "queue" as SessionInput.Delivery } - return { ran: true, continue: false, step: result.step + 1, promotion: undefined } + return { ran: true, continue: false, step: step + 1, promotion: undefined } + }) + + const runStep = Effect.fn("SessionRunner.runStep")(function* (input: StepInput) { + const prologue = yield* stepPrologue(input) + if (prologue.kind === "settled") return prologue.result + const result = yield* runTurn(input.sessionID, prologue.promotion, input.step) + return yield* stepContinuation(input.sessionID, result.needsContinuation, result.step) + }) + + const runModelCall = Effect.fn("SessionRunner.runModelCall")(function* (input: StepInput) { + const prologue = yield* stepPrologue(input) + if (prologue.kind === "settled") return { kind: "settled", result: prologue.result } as ModelCallResult + const result = yield* runTurn(input.sessionID, prologue.promotion, input.step, true) + return { + kind: "called", + step: result.step, + calls: result.calls, + settlement: result.settlement, + } as ModelCallResult }) return Service.of({ run, runStep, + runModelCall, }) }), ) diff --git a/packages/core/src/session/runner/publish-llm-event.ts b/packages/core/src/session/runner/publish-llm-event.ts index 93dd3e86bb9c..3f8d08162e75 100644 --- a/packages/core/src/session/runner/publish-llm-event.ts +++ b/packages/core/src/session/runner/publish-llm-event.ts @@ -27,6 +27,13 @@ const tokens = (usage: Usage | undefined) => { } } +/** What the provider said about how the step ended. Held in memory by the publisher, so a caller + * that defers Step.Ended to another process has to carry it there itself. */ +export interface StepSettlement { + readonly finish: string + readonly tokens: ReturnType +} + const record = (value: unknown): Record => typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) : { value } @@ -113,7 +120,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) let assistantActive = false let assistantFailed = false let providerFailed = false - let stepSettlement: { readonly finish: string; readonly tokens: ReturnType } | undefined + let stepSettlement: StepSettlement | undefined const startAssistant = Effect.fnUntraced(function* () { if (assistantMessageID !== undefined) return assistantMessageID diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts new file mode 100644 index 000000000000..422b2d2b305e --- /dev/null +++ b/packages/core/test/session-runner-model-call.test.ts @@ -0,0 +1,250 @@ +// The model-only attempt (L2): runModelCall performs one provider attempt, records the tool calls it +// asked for, and stops. The caller dispatches each call as its own unit of work and seals the step +// afterwards, which is what puts the model-to-tools loop in a durable executor rather than inside a +// single activity. These tests pin the three properties that split depends on: +// - the call is durable (Tool.Called) but its side effect has NOT run +// - the provider-minted callID and the publisher's assistantMessageID are handed back, never +// regenerated, so the dispatcher's result can be matched to the recorded call +// - the step is left open (no Step.Ended), because the tools have not run yet +// The last test is the contrast: the same stream through runStep runs the tool, as L1 does today. +import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" +import { LLMEvent } from "@opencode-ai/llm" +import { Database } from "@opencode-ai/core/database/database" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { EventV2 } from "@opencode-ai/core/event" +import { PermissionV2 } from "@opencode-ai/core/permission" +import { AgentV2 } from "@opencode-ai/core/agent" +import { Config } from "@opencode-ai/core/config" +import { Project } from "@opencode-ai/core/project" +import { ProjectTable } from "@opencode-ai/core/project/sql" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { SessionV2 } from "@opencode-ai/core/session" +import { Snapshot } from "@opencode-ai/core/snapshot" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import { SessionRunner } from "@opencode-ai/core/session/runner" +import * as SessionRunnerLLM from "@opencode-ai/core/session/runner/llm" +import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model" +import { ToolRegistry } from "@opencode-ai/core/tool/registry" +import { ApplicationTools } from "@opencode-ai/core/tool/application-tools" +import { Tool } from "@opencode-ai/core/tool/tool" +import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionStore } from "@opencode-ai/core/session/store" +import { SessionMessage } from "@opencode-ai/core/session/message" +import { Location } from "@opencode-ai/core/location" +import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" +import { SystemContext } from "@opencode-ai/core/system-context" +import { SkillGuidance } from "@opencode-ai/core/skill/guidance" +import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" +import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" +import { Auth } from "@opencode-ai/llm/route" +import { describe, expect } from "bun:test" +import { Effect, Layer, Schema, Stream } from "effect" +import { testEffect } from "./lib/effect" + +const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) + .model({ id: "gpt-4o-mini" }) +const models = SessionRunnerModel.layerWith(() => Effect.succeed(model)) +const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) +const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) +const permission = Layer.mock(PermissionV2.Service, {}) + +const mockClient = (stream: LLMClientShape["stream"]) => + Layer.succeed( + LLMClient.Service, + LLMClient.Service.of({ + prepare: () => Effect.die("LLMClient.prepare should not be called"), + generate: () => Effect.die("LLMClient.generate should not be called"), + stream, + }), + ) + +// One tool call, then a clean finish: the shape a step that wants to keep going produces. +const callsTool: LLMClientShape["stream"] = () => + Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call_probe", name: "probe_write", input: {} }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + ]) +// An answer and no tool call: the step is over as soon as the stream is, but the seal still has to +// happen, and there is a real assistant message for it to complete. +const textOnly: LLMClientShape["stream"] = () => + Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.textStart({ id: "txt_1" }), + LLMEvent.textDelta({ id: "txt_1", text: "done" }), + LLMEvent.textEnd({ id: "txt_1" }), + LLMEvent.stepFinish({ index: 0, reason: "stop" }), + ]) + +const harness = (stream: LLMClientShape["stream"]) => + testEffect( + AppNodeBuilder.build( + LayerNode.group([ + Database.node, + EventV2.node, + SessionProjector.node, + SessionStore.node, + AgentV2.node, + ToolRegistry.node, + SessionRunnerModel.node, + SystemContextRegistry.node, + SkillGuidance.node, + ReferenceGuidance.node, + Config.node, + Snapshot.node, + SessionRunnerLLM.node, + ApplicationTools.node, + ]), + [ + [LayerNodePlatform.llmClient, mockClient(stream)], + [PermissionV2.node, permission], + [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [SessionRunnerModel.node, models], + [SystemContextRegistry.node, systemContext], + [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })], + [SkillGuidance.node, skillGuidance], + [ReferenceGuidance.node, referenceGuidance], + [Config.node, config], + [Snapshot.node, Snapshot.noopLayer], + ], + ), + ) + +const sessionID = SessionV2.ID.make("ses_runner_model_call") + +const seedSession = Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ id: sessionID, project_id: Project.ID.global, slug: "t", directory: "/project", title: "t", version: "t" }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) +}) + +// A side-effecting tool that counts its own executions, so "the call was recorded but not run" is +// checked against the tool itself rather than only against the projection. +const registerProbe = (ran: { count: number }) => + Effect.gen(function* () { + yield* (yield* ApplicationTools.Service).register({ + probe_write: Tool.make({ + description: "write probe", + input: Schema.Struct({}), + output: Schema.String, + execute: () => + Effect.sync(() => { + ran.count += 1 + return "wrote" + }), + }), + }) + }) + +const toolPart = (messages: ReadonlyArray, callID: string) => { + for (const message of messages) { + if (message.type !== "assistant") continue + for (const part of message.content) if (part.type === "tool" && part.id === callID) return part + } + return undefined +} + +const assistant = (messages: ReadonlyArray) => + messages.find((message) => message.type === "assistant") + +describe("SessionRunner model-only attempt", () => { + harness(callsTool).effect("records the tool call, hands it back, and does not run it", () => + Effect.gen(function* () { + yield* seedSession + const ran = { count: 0 } + yield* registerProbe(ran) + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + const result = yield* runner.runModelCall({ + sessionID, + step: 2, + promotion: undefined, + first: false, + force: false, + }) + + expect(result.kind).toBe("called") + if (result.kind !== "called") return + // The provider's callID and the publisher's assistant message id are carried out, because a + // second run of this step would mint different ones and nothing would match the log. + expect(result.calls.map((call) => call.id)).toEqual(["call_probe"]) + expect(result.calls[0]?.name).toBe("probe_write") + expect(result.calls[0]?.assistantMessageID).toBeTruthy() + // The provider's finish reason has to travel with the calls: it lives only in the publisher's + // memory, so whoever seals the step in another process cannot read it back from the log. + expect(result.settlement?.finish).toBe("tool-calls") + expect(ran.count).toBe(0) + + const context = yield* store.context(sessionID) + const part = toolPart(context, "call_probe") + // Durably recorded and left mid-flight: this is what a dispatcher picks up. + expect(part?.type === "tool" ? part.state.status : undefined).toBe("running") + // The step stays open, so no Step.Ended and no completed assistant message. + const message = assistant(context) + expect(message?.type === "assistant" ? Boolean(message.time.completed) : true).toBe(false) + }), + ) + + harness(textOnly).effect("leaves a text-only step open with no calls to dispatch", () => + Effect.gen(function* () { + yield* seedSession + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + const result = yield* runner.runModelCall({ + sessionID, + step: 2, + promotion: undefined, + first: false, + force: false, + }) + + expect(result.kind).toBe("called") + if (result.kind !== "called") return + expect(result.calls).toHaveLength(0) + expect(result.settlement?.finish).toBe("stop") + // Sealing is uniform: even with nothing to dispatch, the step is closed by the seal, not here, + // so the answer is recorded but its message is still open. + const message = assistant(yield* store.context(sessionID)) + expect(message?.type).toBe("assistant") + expect(message?.type === "assistant" ? Boolean(message.time.completed) : true).toBe(false) + }), + ) + + harness(callsTool).effect("still runs the tool and closes the step through runStep", () => + Effect.gen(function* () { + yield* seedSession + const ran = { count: 0 } + yield* registerProbe(ran) + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + yield* runner.runStep({ sessionID, step: 2, promotion: undefined, first: false, force: false }) + + // The contrast that makes the split meaningful: the whole-step path dispatches and seals. + expect(ran.count).toBe(1) + const context = yield* store.context(sessionID) + const part = toolPart(context, "call_probe") + expect(part?.type === "tool" ? part.state.status : undefined).toBe("completed") + const message = assistant(context) + expect(message?.type === "assistant" ? Boolean(message.time.completed) : false).toBe(true) + }), + ) +}) From 8ad2554c7bc2e3a3c621c486f314aab804008f18 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 25 Aug 2026 17:48:19 -0700 Subject: [PATCH 03/33] Made one recorded tool call runnable on its own. runToolCall settles a single call from what the log already holds, so a dispatcher can give each one its own retry policy, timeout and approval gate instead of sharing the provider attempt's. A duplicate dispatch sees the settled result and does nothing. A retry whose side effect may already have run is reported to the model as an unknown outcome unless the tool declares itself repeatable, which is the rule the crash-resume path already follows. --- packages/core/src/session/runner/index.ts | 23 +++ packages/core/src/session/runner/llm.ts | 58 ++++++++ .../test/session-runner-model-call.test.ts | 132 ++++++++++++++++-- 3 files changed, 203 insertions(+), 10 deletions(-) diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index debceab1d4cf..eb54e58834a2 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -46,6 +46,26 @@ export interface DeferredToolCall { readonly assistantMessageID: string } +/** One recorded tool call, to run on its own. */ +export interface ToolCallInput { + readonly sessionID: SessionSchema.ID + readonly call: DeferredToolCall + /** True when this call is being dispatched again after a crash or a timeout, so its side effect + * may already have happened. The dispatcher knows this; the log cannot tell us. */ + readonly retry: boolean +} + +/** How a dispatched call ended. + * - `settled`: the tool ran and its result is durable. + * - `already-settled`: the log already had a result, so nothing ran. This is the at-least-once case. + * - `unknown`: a retry of a tool that does not declare itself repeatable. Reported to the model as + * an unknown outcome rather than run a second time. */ +export type ToolCallOutcome = "settled" | "already-settled" | "unknown" + +export interface ToolCallResult { + readonly outcome: ToolCallOutcome +} + /** What one provider attempt produced. `calls` is empty unless the caller deferred dispatch, in * which case the step is left open and `settlement` is what seals it. */ export interface TurnAttemptResult { @@ -81,6 +101,9 @@ export interface Interface { * of running them. The caller dispatches each one and then seals the step. This is what puts the * model-to-tools loop in a durable executor's hands rather than inside a single activity. */ readonly runModelCall: (input: StepInput) => Effect.Effect + /** Run one recorded tool call and publish its result. Safe to call twice for the same call: the + * second sees the settled result and does nothing. */ + readonly runToolCall: (input: ToolCallInput) => Effect.Effect } export class Service extends Context.Service()("@opencode/v2/SessionRunner") {} diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index c832516f9e3a..bd979eb673b4 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -37,6 +37,8 @@ import { type ModelCallResult, type RunError, type StepInput, + type ToolCallInput, + type ToolCallResult, type TurnAttemptResult, Service, } from "./index" @@ -626,6 +628,61 @@ const layer = Layer.effect( return { ran: true, continue: false, step: step + 1, promotion: undefined } }) + const toolPartOf = (messages: ReadonlyArray, callID: string) => { + for (const message of messages) { + if (message.type !== "assistant") continue + for (const part of message.content) if (part.type === "tool" && part.id === callID) return part + } + return undefined + } + + const runToolCall = Effect.fn("SessionRunner.runToolCall")(function* (input: ToolCallInput) { + const session = yield* getSession(input.sessionID) + const assistantMessageID = SessionMessage.ID.make(input.call.assistantMessageID) + const part = toolPartOf(yield* getContext(input.sessionID), input.call.id) + // The call has to be in the log already: the attempt that produced it published Tool.Called + // before handing it over. Missing means the log moved under us (a fence), and running a tool + // whose call is not recorded would leave an orphan result. + if (!part || part.type !== "tool") + return yield* Effect.die(`Tool call ${input.call.id} is not recorded on session ${input.sessionID}`) + // At-least-once: a duplicate dispatch landing after the result did must not run anything. + if (part.state.status !== "pending" && part.state.status !== "running") + return { outcome: "already-settled" } as ToolCallResult + const agent = yield* agents.select(session.agent) + const materialization = yield* tools.materialize(agent.info?.permissions) + // A retry cannot know whether the side effect happened, so only a tool that declares itself + // repeatable is run again. The rest are reported unknown and the model decides, because + // re-running the `git push` that may already have landed is the worse failure. + if (input.retry && !materialization.idempotent(input.call.name)) { + yield* events.publish(SessionEvent.Tool.Failed, { + sessionID: input.sessionID, + timestamp: yield* DateTime.now, + assistantMessageID, + callID: input.call.id, + error: { type: "unknown", message: "The outcome of this tool call is unknown" }, + provider: { executed: false }, + }) + return { outcome: "unknown" } as ToolCallResult + } + const settlement = yield* materialization.settle({ + sessionID: input.sessionID, + agent: agent.id, + assistantMessageID, + call: LLMEvent.toolCall({ id: input.call.id, name: input.call.name, input: input.call.input }), + }) + yield* emitToolResult(events, { + sessionID: input.sessionID, + assistantMessageID, + callID: input.call.id, + result: settlement.result, + output: settlement.output, + outputPaths: settlement.outputPaths, + // Deferred calls are never provider-executed: those are filtered out before the hand-off. + provider: { executed: false }, + }) + return { outcome: "settled" } as ToolCallResult + }) + const runStep = Effect.fn("SessionRunner.runStep")(function* (input: StepInput) { const prologue = yield* stepPrologue(input) if (prologue.kind === "settled") return prologue.result @@ -649,6 +706,7 @@ const layer = Layer.effect( run, runStep, runModelCall, + runToolCall, }) }), ) diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index 422b2d2b305e..d02979325f49 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -71,6 +71,13 @@ const callsTool: LLMClientShape["stream"] = () => LLMEvent.toolCall({ id: "call_probe", name: "probe_write", input: {} }), LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), ]) +// The same, for the tool that declares itself repeatable. +const callsIdempotentTool: LLMClientShape["stream"] = () => + Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call_probe", name: "probe_read", input: {} }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + ]) // An answer and no tool call: the step is over as soon as the stream is, but the seal still has to // happen, and there is a real assistant message for it to complete. const textOnly: LLMClientShape["stream"] = () => @@ -134,9 +141,10 @@ const seedSession = Effect.gen(function* () { .pipe(Effect.orDie) }) -// A side-effecting tool that counts its own executions, so "the call was recorded but not run" is -// checked against the tool itself rather than only against the projection. -const registerProbe = (ran: { count: number }) => +// Two probes that count their own executions, so "recorded but not run" and "not run twice" are +// checked against the tools themselves rather than only against the projection. `probe_read` +// declares itself repeatable; `probe_write` does not, which is what decides a retry's behaviour. +const registerProbes = (ran: { write: number; read: number }) => Effect.gen(function* () { yield* (yield* ApplicationTools.Service).register({ probe_write: Tool.make({ @@ -145,13 +153,26 @@ const registerProbe = (ran: { count: number }) => output: Schema.String, execute: () => Effect.sync(() => { - ran.count += 1 + ran.write += 1 return "wrote" }), }), + probe_read: Tool.make({ + description: "read probe", + idempotent: true, + input: Schema.Struct({}), + output: Schema.String, + execute: () => + Effect.sync(() => { + ran.read += 1 + return "read" + }), + }), }) }) +const counters = () => ({ write: 0, read: 0 }) + const toolPart = (messages: ReadonlyArray, callID: string) => { for (const message of messages) { if (message.type !== "assistant") continue @@ -167,8 +188,8 @@ describe("SessionRunner model-only attempt", () => { harness(callsTool).effect("records the tool call, hands it back, and does not run it", () => Effect.gen(function* () { yield* seedSession - const ran = { count: 0 } - yield* registerProbe(ran) + const ran = counters() + yield* registerProbes(ran) const runner = yield* SessionRunner.Service const store = yield* SessionStore.Service @@ -190,7 +211,7 @@ describe("SessionRunner model-only attempt", () => { // The provider's finish reason has to travel with the calls: it lives only in the publisher's // memory, so whoever seals the step in another process cannot read it back from the log. expect(result.settlement?.finish).toBe("tool-calls") - expect(ran.count).toBe(0) + expect(ran.write).toBe(0) const context = yield* store.context(sessionID) const part = toolPart(context, "call_probe") @@ -231,15 +252,15 @@ describe("SessionRunner model-only attempt", () => { harness(callsTool).effect("still runs the tool and closes the step through runStep", () => Effect.gen(function* () { yield* seedSession - const ran = { count: 0 } - yield* registerProbe(ran) + const ran = counters() + yield* registerProbes(ran) const runner = yield* SessionRunner.Service const store = yield* SessionStore.Service yield* runner.runStep({ sessionID, step: 2, promotion: undefined, first: false, force: false }) // The contrast that makes the split meaningful: the whole-step path dispatches and seals. - expect(ran.count).toBe(1) + expect(ran.write).toBe(1) const context = yield* store.context(sessionID) const part = toolPart(context, "call_probe") expect(part?.type === "tool" ? part.state.status : undefined).toBe("completed") @@ -248,3 +269,94 @@ describe("SessionRunner model-only attempt", () => { }), ) }) + +// Dispatching one recorded call on its own. The policy under test is what happens on a retry, when +// the side effect may already have run and the log cannot say: repeatable tools run again, the rest +// are reported unknown so the model decides. Same rule the crash-resume path already follows. +describe("SessionRunner tool dispatch", () => { + const deferOneCall = Effect.gen(function* () { + const runner = yield* SessionRunner.Service + const result = yield* runner.runModelCall({ + sessionID, + step: 2, + promotion: undefined, + first: false, + force: false, + }) + if (result.kind !== "called" || !result.calls[0]) throw new Error("expected a deferred call") + return result.calls[0] + }) + + harness(callsTool).effect("runs a deferred call and records its result", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const call = yield* deferOneCall + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + const result = yield* runner.runToolCall({ sessionID, call, retry: false }) + + expect(result.outcome).toBe("settled") + expect(ran.write).toBe(1) + const part = toolPart(yield* store.context(sessionID), "call_probe") + expect(part?.type === "tool" ? part.state.status : undefined).toBe("completed") + }), + ) + + harness(callsTool).effect("does nothing when the call already has a result", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const call = yield* deferOneCall + const runner = yield* SessionRunner.Service + + yield* runner.runToolCall({ sessionID, call, retry: false }) + // A duplicate dispatch: at-least-once delivery means this happens, and it must not re-run. + const second = yield* runner.runToolCall({ sessionID, call, retry: true }) + + expect(second.outcome).toBe("already-settled") + expect(ran.write).toBe(1) + }), + ) + + harness(callsTool).effect("reports a retried side-effecting call as unknown instead of repeating it", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const call = yield* deferOneCall + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + // The first attempt died somewhere between dispatch and its result landing, so the log still + // shows the call in flight and nothing can say whether the write happened. + const result = yield* runner.runToolCall({ sessionID, call, retry: true }) + + expect(result.outcome).toBe("unknown") + expect(ran.write).toBe(0) + const part = toolPart(yield* store.context(sessionID), "call_probe") + expect(part?.type === "tool" ? part.state.status : undefined).toBe("error") + }), + ) + + harness(callsIdempotentTool).effect("re-runs a retried call that declares itself repeatable", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const call = yield* deferOneCall + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + const result = yield* runner.runToolCall({ sessionID, call, retry: true }) + + expect(result.outcome).toBe("settled") + expect(ran.read).toBe(1) + const part = toolPart(yield* store.context(sessionID), "call_probe") + expect(part?.type === "tool" ? part.state.status : undefined).toBe("completed") + }), + ) +}) From 3cb036ad5eea11b84f6ba998d4cc6dee1da53914 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 25 Aug 2026 18:00:17 -0700 Subject: [PATCH 04/33] Split closing a step out of the provider attempt. The end snapshot and the file diff have to be taken after the tools have run, which is no longer the process that called the model. sealStep does that from the log: it closes any call the dispatcher never settled, then publishes Step.Ended with the settlement the attempt handed over. A repeat sees the step already closed and returns the same loop decision without publishing again, so a retry cannot end the turn a step early. The crash-resume path now shares the continuation tail rather than keeping its own copy of it. --- packages/core/src/session/runner/index.ts | 13 +++ packages/core/src/session/runner/llm.ts | 57 ++++++++-- .../test/session-runner-model-call.test.ts | 107 ++++++++++++++++++ 3 files changed, 168 insertions(+), 9 deletions(-) diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index eb54e58834a2..d7e2243c63a3 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -46,6 +46,15 @@ export interface DeferredToolCall { readonly assistantMessageID: string } +/** Closing one step whose tools have been dispatched. */ +export interface SealStepInput { + readonly sessionID: SessionSchema.ID + readonly step: number + /** The provider's finish reason and token counts, from the attempt that opened this step. They + * live only in that process's memory, so they have to be carried here rather than read back. */ + readonly settlement?: StepSettlement +} + /** One recorded tool call, to run on its own. */ export interface ToolCallInput { readonly sessionID: SessionSchema.ID @@ -104,6 +113,10 @@ export interface Interface { /** Run one recorded tool call and publish its result. Safe to call twice for the same call: the * second sees the settled result and does nothing. */ readonly runToolCall: (input: ToolCallInput) => Effect.Effect + /** Close a step once its calls have been dispatched: snapshot, file diff, Step.Ended, and the loop + * decision. Safe to call twice: the second sees the step already closed and returns the same + * answer without publishing again. */ + readonly sealStep: (input: SealStepInput) => Effect.Effect } export class Service extends Context.Service()("@opencode/v2/SessionRunner") {} diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index bd979eb673b4..262829895a32 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -36,6 +36,7 @@ import { type DeferredToolCall, type ModelCallResult, type RunError, + type SealStepInput, type StepInput, type ToolCallInput, type ToolCallResult, @@ -560,15 +561,8 @@ const layer = Layer.effect( snapshot: endSnapshot, files, }) - // Mirror runStep's continuation tail: a step with local tool calls continues so the model sees - // the (reused or failed) results. - let needsContinuation = localTools - if (!needsContinuation) needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer") - if (needsContinuation) - return { ran: true, continue: true, step: input.step + 1, promotion: "steer" as SessionInput.Delivery } - const moreQueue = yield* SessionInput.hasPending(db, input.sessionID, "queue") - if (moreQueue) return { ran: true, continue: true, step: 1, promotion: "queue" as SessionInput.Delivery } - return { ran: true, continue: false, step: input.step + 1, promotion: undefined } + // A step with local tool calls continues so the model sees the (reused or failed) results. + return yield* stepContinuation(input.sessionID, localTools, input.step) }) // One iteration of `run`'s loop, exposed so a Temporal workflow can drive the turn one step at @@ -628,6 +622,50 @@ const layer = Layer.effect( return { ran: true, continue: false, step: step + 1, promotion: undefined } }) + const sealStep = Effect.fn("SessionRunner.sealStep")(function* (input: SealStepInput) { + const context = yield* getContext(input.sessionID) + const inFlight = context.findLast( + (message): message is SessionMessage.Assistant => message.type === "assistant" && !message.time.completed, + ) + // On a retry that lands after Step.Ended was published there is nothing open, but the loop + // decision still has to come out the same, so it is read off the step we just closed. + const target = + inFlight ?? + context.findLast((message): message is SessionMessage.Assistant => message.type === "assistant") + if (!target) return yield* stepContinuation(input.sessionID, false, input.step) + const toolParts = target.content.filter((part): part is SessionMessage.AssistantTool => part.type === "tool") + // A step continues so the model can see its tool results. Provider-executed calls need no + // follow-up turn, so a step holding only those finalizes as a plain stop. + const localTools = toolParts.some((part) => part.provider?.executed !== true) + if (!inFlight) return yield* stepContinuation(input.sessionID, localTools, input.step) + // A dispatch that failed outright leaves its call open. Close it here, or the next attempt + // sends a request carrying a tool_use with no tool_result and the provider rejects it. + yield* failInterruptedTools(input.sessionID, context) + const startSnapshot = target.snapshot?.start + const endSnapshot = yield* snapshots.capture() + // Ship the post-step tree: this is the state a later step on another host needs. + if (endSnapshot) yield* snapshotSync.push(endSnapshot) + const files = + startSnapshot && endSnapshot + ? yield* snapshots + .files({ from: Snapshot.ID.make(startSnapshot), to: endSnapshot }) + .pipe(Effect.catch(() => Effect.succeed(undefined))) + : undefined + yield* events.publish(SessionEvent.Step.Ended, { + sessionID: input.sessionID, + timestamp: yield* DateTime.now, + assistantMessageID: target.id, + // The attempt's own settlement when the caller carried it; otherwise the same fallback the + // crash-resume path uses, which costs only the metering on that step. + finish: input.settlement?.finish ?? (localTools ? "tool-calls" : "stop"), + cost: 0, + tokens: input.settlement?.tokens ?? { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + snapshot: endSnapshot, + files, + }) + return yield* stepContinuation(input.sessionID, localTools, input.step) + }) + const toolPartOf = (messages: ReadonlyArray, callID: string) => { for (const message of messages) { if (message.type !== "assistant") continue @@ -707,6 +745,7 @@ const layer = Layer.effect( runStep, runModelCall, runToolCall, + sealStep, }) }), ) diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index d02979325f49..9c8d283d4809 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -10,6 +10,7 @@ import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" import { LLMEvent } from "@opencode-ai/llm" import { Database } from "@opencode-ai/core/database/database" +import { EventTable } from "@opencode-ai/core/event/sql" import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" import { LayerNode } from "@opencode-ai/core/effect/layer-node" @@ -42,6 +43,7 @@ import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" import { Auth } from "@opencode-ai/llm/route" import { describe, expect } from "bun:test" import { Effect, Layer, Schema, Stream } from "effect" +import { eq } from "drizzle-orm" import { testEffect } from "./lib/effect" const model = OpenAIChat.route @@ -360,3 +362,108 @@ describe("SessionRunner tool dispatch", () => { }), ) }) + +// Closing the step after its calls have been dispatched. This is the piece that cannot stay in the +// provider attempt: the end snapshot and the file diff have to be taken after the tools have run, +// and in a durable executor that is a different process. +describe("SessionRunner step seal", () => { + const stepEndedCount = Effect.gen(function* () { + const { db } = yield* Database.Service + const rows = yield* db + .select({ type: EventTable.type }) + .from(EventTable) + .where(eq(EventTable.aggregate_id, sessionID)) + .all() + .pipe(Effect.orDie) + return rows.filter((row) => row.type.includes("step.ended")).length + }) + + const deferOneCall = Effect.gen(function* () { + const runner = yield* SessionRunner.Service + const result = yield* runner.runModelCall({ + sessionID, + step: 2, + promotion: undefined, + first: false, + force: false, + }) + if (result.kind !== "called") throw new Error("expected a deferred step") + return result + }) + + harness(callsTool).effect("closes a dispatched step and keeps the turn going", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const model = yield* deferOneCall + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + yield* runner.runToolCall({ sessionID, call: model.calls[0]!, retry: false }) + + const result = yield* runner.sealStep({ sessionID, step: 2, settlement: model.settlement }) + + // Local tool calls mean the model has results to look at, so the turn continues. + expect(result.continue).toBe(true) + expect(result.step).toBe(3) + const message = assistant(yield* store.context(sessionID)) + expect(message?.type === "assistant" ? Boolean(message.time.completed) : false).toBe(true) + expect(yield* stepEndedCount).toBe(1) + }), + ) + + harness(textOnly).effect("closes a text-only step and ends the turn", () => + Effect.gen(function* () { + yield* seedSession + const model = yield* deferOneCall + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + const result = yield* runner.sealStep({ sessionID, step: 2, settlement: model.settlement }) + + expect(result.continue).toBe(false) + const message = assistant(yield* store.context(sessionID)) + expect(message?.type === "assistant" ? Boolean(message.time.completed) : false).toBe(true) + }), + ) + + harness(callsTool).effect("seals once and answers the same on a repeat", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const model = yield* deferOneCall + const runner = yield* SessionRunner.Service + yield* runner.runToolCall({ sessionID, call: model.calls[0]!, retry: false }) + + const first = yield* runner.sealStep({ sessionID, step: 2, settlement: model.settlement }) + // A seal that published Step.Ended and then died is retried. The loop decision has to survive + // that, or the turn would stop one step early. + const second = yield* runner.sealStep({ sessionID, step: 2, settlement: model.settlement }) + + expect(second).toEqual(first) + expect(yield* stepEndedCount).toBe(1) + }), + ) + + harness(callsTool).effect("closes a call the dispatcher never settled", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const model = yield* deferOneCall + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + // The tool activity exhausted its retries and never published a result. Sealing has to close + // the call anyway: a request carrying a tool_use with no tool_result is rejected outright, so + // leaving it open would poison every later attempt. + const result = yield* runner.sealStep({ sessionID, step: 2, settlement: model.settlement }) + + expect(result.continue).toBe(true) + const part = toolPart(yield* store.context(sessionID), "call_probe") + expect(part?.type === "tool" ? part.state.status : undefined).toBe("error") + expect(ran.write).toBe(0) + }), + ) +}) From 8ecffc789f92846f8da0e06353d8e8cc58133a87 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 25 Aug 2026 23:44:09 -0700 Subject: [PATCH 05/33] Drove a step as three activities instead of one. OPENCODE_TEMPORAL_STEPPED=1 runs each step as the provider attempt, one activity per tool call, and a seal. The model-to-tools loop is workflow code now, so a retry policy, a timeout or an approval can sit between the model asking for a tool and the tool running. Each call also gets its own bounds, rather than sharing one timeout with the attempt and every other tool of the step. The supervisor is untouched: wake, interrupt, idle and continue-as-new only ever saw one runTurnStep, so the stepped mode supplies a different one. The mode rides the workflow input so a rollover keeps it. Only the model call claims the event log; the tool and seal activities publish under the token it hands back, or a step's writers would fence each other out. --- packages/temporal/src/activities.ts | 74 ++++++++++-- packages/temporal/src/boundary.ts | 44 +++++++ packages/temporal/src/config.ts | 4 + packages/temporal/src/drain.ts | 44 +------ packages/temporal/src/executor.ts | 13 +- packages/temporal/src/l2-drain.ts | 158 +++++++++++++++++++++++++ packages/temporal/src/l2-step.ts | 66 +++++++++++ packages/temporal/src/workflow.ts | 36 +++++- packages/temporal/test/l2-step.test.ts | 105 ++++++++++++++++ 9 files changed, 485 insertions(+), 59 deletions(-) create mode 100644 packages/temporal/src/boundary.ts create mode 100644 packages/temporal/src/l2-drain.ts create mode 100644 packages/temporal/src/l2-step.ts create mode 100644 packages/temporal/test/l2-step.test.ts diff --git a/packages/temporal/src/activities.ts b/packages/temporal/src/activities.ts index 68a37c8d8b03..6403d4a2e47a 100644 --- a/packages/temporal/src/activities.ts +++ b/packages/temporal/src/activities.ts @@ -28,29 +28,79 @@ function ownerToken(): string { // The step contract lives with the drain in core; re-exported here so the workflow and its tests // keep one import site inside this package. import type { StepDrainInput, StepDrainResult } from "./drain" +import type { + ModelCallDrainInput, + ModelCallDrainResult, + SealDrainInput, + ToolCallDrainInput, + ToolCallDrainResult, +} from "./l2-drain" export type { StepDrainInput, StepDrainResult } export type StepActivities = { runTurnStep(input: StepDrainInput): Promise } +// Heartbeat while a drain runs so a dead worker is noticed quickly. This only proves the process is +// alive, not that the work is progressing. +const beating = async (body: () => Promise): Promise => { + const beat = setInterval(() => { + try { + heartbeat() + } catch { + // heartbeat outside an activity context is a no-op for our purposes + } + }, 3000) + try { + return await body() + } finally { + clearInterval(beat) + } +} + export function makeStepActivities( stepDrain: (input: StepDrainInput, signal: AbortSignal) => Promise, ): StepActivities { return { async runTurnStep(input) { - const beat = setInterval(() => { - try { - heartbeat() - } catch { - // heartbeat outside an activity context is a no-op for our purposes - } - }, 3000) - try { - return await stepDrain({ ...input, owner: ownerToken() }, Context.current().cancellationSignal) - } finally { - clearInterval(beat) - } + return beating(() => stepDrain({ ...input, owner: ownerToken() }, Context.current().cancellationSignal)) + }, + } +} + +export type SteppedTurnActivities = { + runModelCall(input: ModelCallDrainInput): Promise + runToolCall(input: ToolCallDrainInput): Promise + sealStep(input: SealDrainInput): Promise +} + +// The three activities of a stepped step. Only the model call mints and claims an owner token: it is +// the writer that supersedes the attempt before it, and the tool and seal activities publish under +// the token it returned. Giving each of them its own token would make a step's writers fence each +// other out of the log. +export function makeSteppedTurnActivities(drains: { + modelCallDrain: ( + input: ModelCallDrainInput & { readonly owner: string }, + signal: AbortSignal, + ) => Promise + toolCallDrain: (input: ToolCallDrainInput, signal: AbortSignal) => Promise + sealDrain: (input: SealDrainInput, signal: AbortSignal) => Promise +}): SteppedTurnActivities { + return { + async runModelCall(input) { + return beating(() => + drains.modelCallDrain({ ...input, owner: ownerToken() }, Context.current().cancellationSignal), + ) + }, + async runToolCall(input) { + // Whether the side effect may already have happened is the activity's own knowledge, not the + // workflow's: a second attempt of THIS call is a re-run of a tool whose result never landed. + // The workflow could not compute this without reading non-deterministic state. + const retry = Context.current().info.attempt > 1 + return beating(() => drains.toolCallDrain({ ...input, retry }, Context.current().cancellationSignal)) + }, + async sealStep(input) { + return beating(() => drains.sealDrain(input, Context.current().cancellationSignal)) }, } } diff --git a/packages/temporal/src/boundary.ts b/packages/temporal/src/boundary.ts new file mode 100644 index 000000000000..30e272592166 --- /dev/null +++ b/packages/temporal/src/boundary.ts @@ -0,0 +1,44 @@ +// Crossing from Effect into an activity result. Every drain in this package ends here, so the way a +// failure is classified is decided in one place: +// - an interrupt caused by the driver's own cancellation rethrows the abort reason, so the attempt +// records Cancelled rather than Failed +// - an interrupt with no cancellation is an internal halt (a user declining a permission) and must +// be non-retryable, or the supervisor re-drives a turn the user explicitly stopped +// - a genuine run error crosses non-retryable with the RunError encoded in `details`, so the caller +// reconstructs the exact typed error instead of a string +// Only crashes and task timeouts (never thrown here) go through the activity retry policy. + +import { Cause, Effect, Exit } from "effect" +import { ApplicationFailure } from "@temporalio/activity" +import { SessionSchema } from "@opencode-ai/core/session/schema" +import { SessionRunDeclinedError } from "@opencode-ai/core/session/error" +import { encodeRunError } from "@opencode-ai/core/session/execution/run-error-codec" + +export const runAtBoundary = async ( + sessionID: string, + signal: AbortSignal, + body: Effect.Effect, +): Promise => { + const exit = await Effect.runPromiseExit(body, { signal }) + if (Exit.isSuccess(exit)) return exit.value + const cause = exit.cause + if (Cause.hasInterruptsOnly(cause)) { + if (signal.aborted) + throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") + const declined = encodeRunError(new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make(sessionID) })) + throw ApplicationFailure.create({ + message: "session run halted (user declined)", + type: "SessionRunDeclined", + nonRetryable: true, + details: declined === undefined ? undefined : [declined], + }) + } + const squashed = Cause.squash(cause) as { _tag?: string; message?: string } + const encoded = encodeRunError(squashed) + throw ApplicationFailure.create({ + message: squashed?.message ?? Cause.pretty(cause), + type: squashed?._tag ?? "SessionRunError", + nonRetryable: true, + details: encoded === undefined ? undefined : [encoded], + }) +} diff --git a/packages/temporal/src/config.ts b/packages/temporal/src/config.ts index 6d4c7c70de3e..f4d4cec8279f 100644 --- a/packages/temporal/src/config.ts +++ b/packages/temporal/src/config.ts @@ -18,6 +18,9 @@ export interface Interface { readonly role: Role /** Override for the supervisor's idle self-termination; local mode honors the same variable. */ readonly idleTimeout?: string + /** Drive each step as a provider attempt, one activity per tool call, and a seal. Off by default: + * the whole-step mode is what runs today, and this only changes how new sessions start. */ + readonly stepped?: boolean } export class Service extends Context.Service()("@opencode/temporal/Config") {} @@ -28,4 +31,5 @@ export const fromEnv = (): Interface => ({ taskQueue: process.env.OPENCODE_TEMPORAL_TASK_QUEUE ?? DEFAULTS.taskQueue, role: (process.env.OPENCODE_TEMPORAL_ROLE as Role | undefined) ?? "both", idleTimeout: process.env.OPENCODE_SESSION_IDLE_TIMEOUT, + stepped: process.env.OPENCODE_TEMPORAL_STEPPED === "1", }) diff --git a/packages/temporal/src/drain.ts b/packages/temporal/src/drain.ts index db10397030d0..129b89d70e96 100644 --- a/packages/temporal/src/drain.ts +++ b/packages/temporal/src/drain.ts @@ -4,8 +4,7 @@ // SessionRunner.run on the SessionRunCoordinator (execution/local.ts). Both modes go through the // same SessionRunner and the same durable event log. -import { Cause, Context, Effect, Exit, type LayerMap } from "effect" -import { ApplicationFailure } from "@temporalio/activity" +import { Context, Effect, type LayerMap } from "effect" import type { LocationServiceMap } from "@opencode-ai/core/location-service-map" import type { Location } from "@opencode-ai/core/location" import type { LocationError, LocationServices } from "@opencode-ai/core/location-services" @@ -14,9 +13,8 @@ import { WorktreeMaterializer } from "@opencode-ai/core/session/execution/worktr import { SessionRunner } from "@opencode-ai/core/session/runner" import { SessionSchema } from "@opencode-ai/core/session/schema" import { SessionStore } from "@opencode-ai/core/session/store" -import { SessionRunDeclinedError } from "@opencode-ai/core/session/error" import type { SessionInput } from "@opencode-ai/core/session/input" -import { encodeRunError } from "@opencode-ai/core/session/execution/run-error-codec" +import { runAtBoundary } from "./boundary" // One step of a turn, as any executor drives it. `promotion` is null (not undefined) so it // serializes cleanly across an executor's process boundary. export interface StepDrainInput { @@ -50,8 +48,10 @@ export interface DrainDeps { export const makeDrains = ({ store, locations, ctx, events, worktrees }: DrainDeps) => { // Run exactly one step of the turn (the supervisor loops it); returns the next loop state. - const stepDrain = async (input: StepDrainInput, signal: AbortSignal): Promise => { - const exit = await Effect.runPromiseExit( + const stepDrain = async (input: StepDrainInput, signal: AbortSignal): Promise => + runAtBoundary( + input.sessionID, + signal, Effect.gen(function* () { const session = yield* store.get(SessionSchema.ID.make(input.sessionID)) if (!session) return { ran: false, continue: false, step: input.step, promotion: null } @@ -70,39 +70,7 @@ export const makeDrains = ({ store, locations, ctx, events, worktrees }: DrainDe ).pipe(Effect.provide(locations.get(session.location))) return { ran: r.ran, continue: r.continue, step: r.step, promotion: r.promotion ?? null } }).pipe(Effect.provideService(EventV2.EventOwner, input.owner), Effect.provide(ctx), Effect.scoped), - { signal }, ) - if (Exit.isSuccess(exit)) return exit.value - const cause = exit.cause - if (Cause.hasInterruptsOnly(cause)) { - // Two interrupt sources: driver cancellation (the AbortSignal fired; rethrow its reason so - // the attempt records Cancelled, not Failed) and an internal halt like a user declining a - // permission (the signal did NOT fire). The latter must be non-retryable, or the supervisor - // re-drives a turn the user explicitly stopped. - if (signal.aborted) - throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") - const declined = encodeRunError( - new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make(input.sessionID) }), - ) - throw ApplicationFailure.create({ - message: "session run halted (user declined)", - type: "SessionRunDeclined", - nonRetryable: true, - details: declined === undefined ? undefined : [declined], - }) - } - // A genuine run error is thrown non-retryable so Temporal surfaces it (to resume) rather than - // retrying; only crashes / task timeouts (never thrown here) go through the retry policy. The - // error is encoded faithfully in `details` so the caller can reconstruct the exact RunError. - const squashed = Cause.squash(cause) as { _tag?: string; message?: string } - const encoded = encodeRunError(squashed) - throw ApplicationFailure.create({ - message: squashed?.message ?? Cause.pretty(cause), - type: squashed?._tag ?? "SessionRunError", - nonRetryable: true, - details: encoded === undefined ? undefined : [encoded], - }) - } return { stepDrain } } diff --git a/packages/temporal/src/executor.ts b/packages/temporal/src/executor.ts index 1d3e7042a0e6..303c3bdbd0ea 100644 --- a/packages/temporal/src/executor.ts +++ b/packages/temporal/src/executor.ts @@ -13,8 +13,9 @@ import { makeGlobalNode } from "@opencode-ai/core/effect/app-node" import { SessionSchema } from "@opencode-ai/core/session/schema" import { SessionStore } from "@opencode-ai/core/session/store" import { SessionExecution } from "@opencode-ai/core/session/execution" -import { makeStepActivities } from "./activities" +import { makeStepActivities, makeSteppedTurnActivities } from "./activities" import { makeDrains } from "./drain" +import { makeL2Drains } from "./l2-drain" import { WorktreeMaterializer } from "@opencode-ai/core/session/execution/worktree" import { toRunError } from "@opencode-ai/core/session/execution/run-error-codec" import * as WF from "./workflow" @@ -58,12 +59,16 @@ const layer = Layer.effect( // Same knob local mode honors; the workflow sandbox cannot read env, so the client forwards the // override as a workflow argument. const IDLE_TIMEOUT = config.idleTimeout + const STEPPED = config.stepped === true const events = yield* EventV2.Service const worktrees = yield* WorktreeMaterializer.Service // The per-step drain (drain.ts) wraps SessionRunner.runStep for the activity boundary. Local // mode runs whole turns through SessionRunner.run on the coordinator; both share SessionRunner. const { stepDrain } = makeDrains({ store, locations, ctx, events, worktrees }) + // The stepped mode's three drains. Registered unconditionally: which mode a session runs is a + // property of its workflow input, so a worker has to be able to serve either. + const l2 = makeL2Drains({ store, locations, ctx, events, worktrees }) // Worker connection (native) hosts the runTurnStep activity + the workflow. Skipped in // client-only role so serve can run without an embedded worker. @@ -88,7 +93,7 @@ const layer = Layer.effect( namespace: NAMESPACE, taskQueue: TASK_QUEUE, workflowsPath: fileURLToPath(new URL("./workflow.ts", import.meta.url)), - activities: makeStepActivities(stepDrain), + activities: { ...makeStepActivities(stepDrain), ...makeSteppedTurnActivities(l2) }, }), ) const runHandle = worker.run() @@ -129,7 +134,7 @@ const layer = Layer.effect( client.workflow.signalWithStart(WORKFLOW_TYPE, { taskQueue: TASK_QUEUE, workflowId: workflowId(id), - args: [id, { startWithWake: true, idleTimeout: IDLE_TIMEOUT } satisfies WF.SessionTurnOptions], + args: [id, { startWithWake: true, idleTimeout: IDLE_TIMEOUT, stepped: STEPPED } satisfies WF.SessionTurnOptions], signal: WF.wake, signalArgs: [], }), @@ -180,7 +185,7 @@ const layer = Layer.effect( // startWithWake=false: a fresh resume-with-start must not manufacture a wake drain; // its forced drain comes from the resume update. Ignored when USE_EXISTING joins a // running workflow (which keeps its own state). - args: [id, { startWithWake: false, idleTimeout: IDLE_TIMEOUT } satisfies WF.SessionTurnOptions], + args: [id, { startWithWake: false, idleTimeout: IDLE_TIMEOUT, stepped: STEPPED } satisfies WF.SessionTurnOptions], workflowIdConflictPolicy: "USE_EXISTING", }) return client.workflow.executeUpdateWithStart(WF.resume, { diff --git a/packages/temporal/src/l2-drain.ts b/packages/temporal/src/l2-drain.ts new file mode 100644 index 000000000000..dcd45f8690c5 --- /dev/null +++ b/packages/temporal/src/l2-drain.ts @@ -0,0 +1,158 @@ +// The three drain bodies a stepped turn is made of: the provider attempt, one tool call, and the +// seal. L1's drain runs a whole step (one attempt plus all of its tools) in a single activity, so +// nothing can sit between the model asking for a tool and the tool running. Splitting them is what +// gives each tool call its own retry policy, timeout and approval window. +// +// All three write to the same session log, so they publish under ONE owner token: the model call +// claims it and hands it back, the other two inherit it. Minting a token per activity execution (as +// L1 does, correctly, when the activity is the whole step) would make a step's writers fence each +// other out. + +import { Context, Effect, type LayerMap } from "effect" +import type { LocationServiceMap } from "@opencode-ai/core/location-service-map" +import type { Location } from "@opencode-ai/core/location" +import type { LocationError, LocationServices } from "@opencode-ai/core/location-services" +import { EventV2 } from "@opencode-ai/core/event" +import { WorktreeMaterializer } from "@opencode-ai/core/session/execution/worktree" +import { SessionRunner } from "@opencode-ai/core/session/runner" +import type { DeferredToolCall, ToolCallOutcome } from "@opencode-ai/core/session/runner" +import type { StepSettlement } from "@opencode-ai/core/session/runner/publish-llm-event" +import { SessionSchema } from "@opencode-ai/core/session/schema" +import { SessionStore } from "@opencode-ai/core/session/store" +import type { SessionInput } from "@opencode-ai/core/session/input" +import { runAtBoundary } from "./boundary" +import type { StepDrainInput, StepDrainResult } from "./drain" + +/** The provider attempt of one step. Same shape as a whole-step drain: the difference is what it + * does with the tool calls, not what it needs to start. */ +export type ModelCallDrainInput = StepDrainInput + +export type ModelCallDrainResult = + | { readonly kind: "settled"; readonly result: StepDrainResult } + | { + readonly kind: "called" + readonly step: number + readonly calls: ReadonlyArray + readonly settlement?: StepSettlement + /** The event-log token this attempt claimed. The tool and seal activities of this step must + * publish under it, so it travels with the calls instead of being minted again. */ + readonly owner: string + } + +export interface ToolCallDrainInput { + readonly sessionID: string + readonly call: DeferredToolCall + readonly owner: string + /** Set by the executor from the activity attempt, not by the workflow: only the dispatcher knows + * whether this call already had a run whose result never landed. */ + readonly retry?: boolean +} + +export interface ToolCallDrainResult { + readonly outcome: ToolCallOutcome +} + +export interface SealDrainInput { + readonly sessionID: string + readonly step: number + readonly settlement?: StepSettlement + readonly owner: string +} + +export interface L2DrainDeps { + readonly store: SessionStore.Interface + readonly locations: LayerMap.LayerMap + readonly ctx: Context.Context + readonly events: EventV2.Interface + readonly worktrees: WorktreeMaterializer.Interface +} + +export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2DrainDeps) => { + // One session, one owner, a present project tree. `claim` is true only for the model call: it is + // the writer that supersedes a previous attempt, and the rest of the step rides its token. + const inSession = ( + sessionID: string, + owner: string, + claim: boolean, + use: (runner: SessionRunner.Interface, session: SessionSchema.Info) => Effect.Effect, + ) => + Effect.gen(function* () { + const session = yield* store.get(SessionSchema.ID.make(sessionID)) + if (!session) return yield* Effect.die(`Session not found: ${sessionID}`) + if (claim) yield* events.claim(session.id, owner) + // A worker taking this step on a host without the project tree rebuilds it from snapshot packs. + yield* worktrees.ensure(session.location.directory) + return yield* SessionRunner.Service.use((runner) => use(runner, session)).pipe( + Effect.provide(locations.get(session.location)), + ) + }).pipe(Effect.provideService(EventV2.EventOwner, owner), Effect.provide(ctx), Effect.scoped) + + const modelCallDrain = async ( + input: ModelCallDrainInput & { readonly owner: string }, + signal: AbortSignal, + ): Promise => + runAtBoundary( + input.sessionID, + signal, + inSession(input.sessionID, input.owner, true, (runner, session) => + runner.runModelCall({ + sessionID: session.id, + step: input.step, + promotion: (input.promotion ?? undefined) as SessionInput.Delivery | undefined, + first: input.first, + force: input.force, + }), + ).pipe( + Effect.map((result): ModelCallDrainResult => + result.kind === "settled" + ? { + kind: "settled", + result: { + ran: result.result.ran, + continue: result.result.continue, + step: result.result.step, + promotion: result.result.promotion ?? null, + }, + } + : { + kind: "called", + step: result.step, + calls: result.calls, + settlement: result.settlement, + owner: input.owner, + }, + ), + ), + ) + + const toolCallDrain = async (input: ToolCallDrainInput, signal: AbortSignal): Promise => + runAtBoundary( + input.sessionID, + signal, + inSession(input.sessionID, input.owner, false, (runner, session) => + runner.runToolCall({ + sessionID: session.id, + call: input.call, + retry: input.retry === true, + }), + ), + ) + + const sealDrain = async (input: SealDrainInput, signal: AbortSignal): Promise => + runAtBoundary( + input.sessionID, + signal, + inSession(input.sessionID, input.owner, false, (runner, session) => + runner.sealStep({ sessionID: session.id, step: input.step, settlement: input.settlement }), + ).pipe( + Effect.map((result) => ({ + ran: result.ran, + continue: result.continue, + step: result.step, + promotion: result.promotion ?? null, + })), + ), + ) + + return { modelCallDrain, toolCallDrain, sealDrain } +} diff --git a/packages/temporal/src/l2-step.ts b/packages/temporal/src/l2-step.ts new file mode 100644 index 000000000000..9ed4ebb8aaa3 --- /dev/null +++ b/packages/temporal/src/l2-step.ts @@ -0,0 +1,66 @@ +// One step as three units of work instead of one: the provider attempt, each tool call it asks for, +// and the seal that closes it. This is the whole point of the split, and it is workflow code, so the +// model-to-tools loop lives where retries, timers, approvals and budgets can sit between the two. +// +// MUST stay pure, like supervisor.ts: this is bundled into the workflow sandbox, so no `effect`, no +// `@opencode-ai/core` runtime imports, no Node builtins. Type-only imports are erased and safe. +// +// What this costs, stated plainly: a whole-step activity starts each tool the moment the model asks +// for it, while the stream is still going. Here the attempt has to return before any tool starts, +// because a workflow cannot consume a stream. The tools of one step still run concurrently with each +// other; what is lost is the overlap between the model and its own tools. + +import type { StepDrainInput, StepDrainResult } from "./drain" +import type { + ModelCallDrainInput, + ModelCallDrainResult, + SealDrainInput, + ToolCallDrainInput, + ToolCallDrainResult, +} from "./l2-drain" + +/** The three activities a stepped turn drives. */ +export interface SteppedActivities { + readonly runModelCall: (input: ModelCallDrainInput) => Promise + readonly runToolCall: (input: ToolCallDrainInput) => Promise + readonly sealStep: (input: SealDrainInput) => Promise +} + +export interface SteppedTurnDeps { + readonly activities: SteppedActivities + /** Whether an error is the driver's cancellation. An interrupt has to end the turn, so it must not + * be swallowed the way a failed tool is. */ + readonly isCancellation: (error: unknown) => boolean +} + +/** + * Drives one step and reports the next loop state, so it drops straight into the supervisor in place + * of a whole-step activity. + */ +export const makeSteppedTurn = + ({ activities, isCancellation }: SteppedTurnDeps) => + async (input: StepDrainInput): Promise => { + const model = await activities.runModelCall(input) + // A crashed step finalized from the log, or the recovery gate finding no work: the step is over + // and there is nothing to dispatch or seal. + if (model.kind === "settled") return model.result + + // Each call is its own unit of work. A tool that fails outright does not take the turn with it: + // the seal closes its call as an error and the model gets to react, which is better than losing + // the step. An interrupt is different and has to propagate. + const dispatched = await Promise.allSettled( + model.calls.map((call) => + activities.runToolCall({ sessionID: input.sessionID, call, owner: model.owner }), + ), + ) + for (const outcome of dispatched) { + if (outcome.status === "rejected" && isCancellation(outcome.reason)) throw outcome.reason + } + + return activities.sealStep({ + sessionID: input.sessionID, + step: model.step, + settlement: model.settlement, + owner: model.owner, + }) + } diff --git a/packages/temporal/src/workflow.ts b/packages/temporal/src/workflow.ts index d19ac7df8228..da7d586d2f9b 100644 --- a/packages/temporal/src/workflow.ts +++ b/packages/temporal/src/workflow.ts @@ -19,7 +19,8 @@ import { isCancellation, allHandlersFinished, } from "@temporalio/workflow" -import type { StepActivities } from "./activities" +import type { StepActivities, SteppedTurnActivities } from "./activities" +import { makeSteppedTurn } from "./l2-step" import { SIGNALS, RESUME_UPDATE } from "./protocol" import { makeSupervisor, type SupervisorRuntime } from "./supervisor" @@ -37,6 +38,17 @@ const activityOptions = { const { runTurnStep } = proxyActivities(activityOptions) +// The stepped mode's three activities, each with its own bounds. Separate proxies are the point of +// the split, not an accident of it: a tool that hangs, or that is waiting on a human, no longer +// holds the provider attempt and every other tool of the same step under one shared timeout. +const { runModelCall } = proxyActivities(activityOptions) +const { runToolCall } = proxyActivities(activityOptions) +// Sealing is a snapshot, a diff and one event. It should not inherit a turn-sized backstop. +const { sealStep } = proxyActivities({ + ...activityOptions, + startToCloseTimeout: "10 minutes", +}) + export const wake = defineSignal(SIGNALS.wake) export const interrupt = defineSignal(SIGNALS.interrupt) export const resume = defineUpdate(RESUME_UPDATE) @@ -95,6 +107,13 @@ const runtime: SupervisorRuntime = { continueAsNew(sessionID, { startWithWake }), } +// Same supervisor, different step body: wake, interrupt, idle timeout and continue-as-new are +// unchanged, and only what "one step" means differs. +const steppedRuntime: SupervisorRuntime = { + ...runtime, + runTurnStep: makeSteppedTurn({ activities: { runModelCall, runToolCall, sealStep }, isCancellation }), +} + // The scope of the drain currently running, so an interrupt signal can cancel exactly that turn. let activeDrainScope: CancellationScope | undefined // The workflow's root scope, captured at entry, to detect a whole-run cancellation. @@ -109,18 +128,25 @@ const workflows = makeSupervisor(runtime) export interface SessionTurnOptions { readonly startWithWake?: boolean readonly idleTimeout?: string + /** Drive each step as a provider attempt, one activity per tool call, and a seal, instead of one + * activity for the whole step. Off by default: the whole-step mode is what runs today. */ + readonly stepped?: boolean } export async function sessionTurn(sessionID: string, options?: SessionTurnOptions): Promise { rootScope = CancellationScope.current() const startWithWake = options?.startWithWake ?? true const idleTimeout = options?.idleTimeout - if (!idleTimeout) return workflows.sessionTurn(sessionID, startWithWake) + const stepped = options?.stepped === true + if (!idleTimeout && !stepped) return workflows.sessionTurn(sessionID, startWithWake) return makeSupervisor( { - ...runtime, - continueAsNew: (id, wake) => continueAsNew(id, { startWithWake: wake, idleTimeout }), + ...(stepped ? steppedRuntime : runtime), + // The mode has to survive the boundary, or a long session silently reverts to whole-step + // activities the first time it rolls over. + continueAsNew: (id, wake) => + continueAsNew(id, { startWithWake: wake, idleTimeout, stepped }), }, - { idleTimeout }, + idleTimeout ? { idleTimeout } : undefined, ).sessionTurn(sessionID, startWithWake) } diff --git a/packages/temporal/test/l2-step.test.ts b/packages/temporal/test/l2-step.test.ts new file mode 100644 index 000000000000..480166de8835 --- /dev/null +++ b/packages/temporal/test/l2-step.test.ts @@ -0,0 +1,105 @@ +// Deterministic unit tests for the stepped turn body (src/l2-step.ts) driven by fake activities -- +// no Temporal, no DB. This is the piece that turns one step into three units of work, so what is +// pinned here is the orchestration: the owner token reaches every writer, a settled step dispatches +// nothing, a failed tool still lets the step close, and an interrupt is not swallowed. +import { describe, it, expect } from "bun:test" +import { makeSteppedTurn, type SteppedActivities } from "../src/l2-step" +import type { StepDrainInput, StepDrainResult } from "../src/activities" +import type { ModelCallDrainResult, SealDrainInput, ToolCallDrainInput } from "../src/l2-drain" + +class FakeCancel extends Error {} +const isCancellation = (e: unknown) => e instanceof FakeCancel + +const INPUT: StepDrainInput = { sessionID: "ses_1", step: 2, promotion: null, first: false, force: false } +const SEALED: StepDrainResult = { ran: true, continue: true, step: 3, promotion: "steer" } +const call = (id: string, name = "probe_write") => ({ id, name, input: {}, assistantMessageID: "msg_1" }) + +const fakes = ( + model: ModelCallDrainResult, + onTool: (input: ToolCallDrainInput) => Promise<{ outcome: "settled" }> = async () => ({ outcome: "settled" }), +) => { + const tools: ToolCallDrainInput[] = [] + const seals: SealDrainInput[] = [] + const activities: SteppedActivities = { + runModelCall: async () => model, + runToolCall: async (input) => { + tools.push(input) + return onTool(input) + }, + sealStep: async (input) => { + seals.push(input) + return SEALED + }, + } + return { activities, tools, seals } +} + +describe("stepped turn", () => { + it("dispatches nothing and does not seal when the step is already settled", async () => { + const settled: StepDrainResult = { ran: false, continue: false, step: 2, promotion: null } + const { activities, tools, seals } = fakes({ kind: "settled", result: settled }) + + const result = await makeSteppedTurn({ activities, isCancellation })(INPUT) + + // A crashed step finalized from the log, or the recovery gate finding no work: there is nothing + // to run and nothing to close. + expect(result).toEqual(settled) + expect(tools).toHaveLength(0) + expect(seals).toHaveLength(0) + }) + + it("runs every call under the attempt's owner and then seals", async () => { + const { activities, tools, seals } = fakes({ + kind: "called", + step: 2, + calls: [call("call_a"), call("call_b")], + settlement: { finish: "tool-calls", tokens: { input: 1, output: 2, reasoning: 0, cache: { read: 0, write: 0 } } }, + owner: "run-1:model-1:1", + }) + + const result = await makeSteppedTurn({ activities, isCancellation })(INPUT) + + expect(tools.map((t) => t.call.id)).toEqual(["call_a", "call_b"]) + // Every writer in the step publishes under the token the attempt claimed. A token minted per + // activity would make them fence each other out of the log. + expect(tools.map((t) => t.owner)).toEqual(["run-1:model-1:1", "run-1:model-1:1"]) + expect(seals).toHaveLength(1) + expect(seals[0]?.owner).toBe("run-1:model-1:1") + // The finish reason lives only in the attempt's memory, so the seal has to be handed it. + expect(seals[0]?.settlement?.finish).toBe("tool-calls") + expect(result).toEqual(SEALED) + }) + + it("seals even when a tool call fails outright", async () => { + const { activities, tools, seals } = fakes( + { kind: "called", step: 2, calls: [call("call_a"), call("call_b")], owner: "own" }, + async (input) => { + if (input.call.id === "call_a") throw new Error("activity exhausted its retries") + return { outcome: "settled" } + }, + ) + + const result = await makeSteppedTurn({ activities, isCancellation })(INPUT) + + // One broken tool must not take the turn with it: the seal closes that call as an error and the + // model gets to react, which beats losing the step. + expect(tools).toHaveLength(2) + expect(seals).toHaveLength(1) + expect(result).toEqual(SEALED) + }) + + it("lets an interrupt end the turn instead of sealing it", async () => { + const { activities, seals } = fakes( + { kind: "called", step: 2, calls: [call("call_a")], owner: "own" }, + async () => { + throw new FakeCancel("interrupted") + }, + ) + + const run = makeSteppedTurn({ activities, isCancellation })(INPUT) + + // A cancellation is not a failed tool. Swallowing it would close a step the user stopped. + await expect(run).rejects.toBeInstanceOf(FakeCancel) + expect(seals).toHaveLength(0) + }) +}) From e5b93e49cbcfbb5eaf10d1217ab36131260ddbad Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 00:30:43 -0700 Subject: [PATCH 06/33] Documented the stepped mode and what a live crash proved. Records the owner-token rule and the retry rule, since both are easy to get wrong and neither is visible from the call sites, and states the cost plainly: the model no longer overlaps with its own tools. --- packages/temporal/README.md | 52 +++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 1de02f76b2b8..4af6dbde58ad 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -93,6 +93,58 @@ session runs as a Temporal workflow `session-exec-`. the in-flight step activity (attempt 2), the run continues from the event log, and the workflow completes. +### A step as three activities (`OPENCODE_TEMPORAL_STEPPED=1`) + +By default one step (a provider attempt plus every tool it asks for) is a single activity. That is +the smallest unit the runner used to expose, and it means nothing can sit between the model asking +for a tool and the tool running. `OPENCODE_TEMPORAL_STEPPED=1` splits a step into three kinds of +activity instead: + +``` +runModelCall -> runToolCall (one per call, concurrent) -> sealStep +``` + +`SessionRunner.runModelCall` performs the attempt, records each call as `Tool.Called`, and hands the +calls back rather than running them. `runToolCall` settles one call. `sealStep` takes the end +snapshot, diffs it against the start, and publishes `Step.Ended`. The loop between them is workflow +code, so a retry policy, a timeout, an approval or a budget can live where the model-to-tools handoff +used to be. Each activity also carries its own bounds: sealing does not inherit a turn-sized +backstop, and one tool waiting on a human no longer holds the attempt and its sibling tools under a +single timeout. + +The supervisor is unchanged. Wake, interrupt, idle self-termination and continue-as-new only ever +called one `runTurnStep`, so the stepped mode supplies a different one. The mode rides the workflow +input, so a session that rolls over keeps it. + +Two things are load-bearing and easy to get wrong: + +- **One owner token per step, not per activity.** The event log fences a publish behind the current + owner, so a step's writers have to share one. Only `runModelCall` claims; the tool and seal + activities publish under the token it returns. A token per activity execution (right when the + activity *is* the whole step) would make them fence each other out. +- **A retried call is not silently repeated.** Whether a side effect already happened is the + activity's knowledge, not the workflow's, so `retry` comes from the Temporal attempt number. On a + retry only a tool declaring `idempotent` runs again; anything else is reported to the model as an + unknown outcome. This is the rule the crash-resume path already followed. + +What it costs: a whole-step activity starts each tool the moment the model asks for it, while the +stream is still going. Here the attempt has to return before any tool starts, because a workflow +cannot consume a stream. The tools of one step still run concurrently with each other; the overlap +between the model and its own tools is what is lost. + +#### Verified + +Live against a dev server and `gpt-5-mini`, with `OPENCODE_SESSION_EXECUTION=temporal` and +`OPENCODE_TEMPORAL_STEPPED=1`: + +- A turn using one tool recorded five activities: `runModelCall`, `runToolCall`, `sealStep` for the + step that called the tool, then `runModelCall`, `sealStep` for the answer. The tool ran once. +- Crash mid-tool: a turn ran `echo ... >> counter.txt` in one step and `sleep 60` in the next, and + the serve process (with its embedded worker) was killed with the sleep in flight. On a fresh + worker the interrupted `runToolCall` came back as **attempt 2**, `counter.txt` still held one + line (the settled tool was not re-run), the interrupted call reached the model as "The outcome of + this tool call is unknown", and the turn ran on to its answer. + ### Two modes, one runner The factory has exactly two modes, both driving the same `SessionRunner` over the same durable event From 12df8bace81ffaeedec210b3bbba3204e5754b06 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 01:16:11 -0700 Subject: [PATCH 07/33] Measured what the stepped split costs. The overlap loss is min(stream tail after the first tool call, tool duration), within about 10ms across a five-point sweep, and zero when the tool call is the last thing in the stream. The extra hand-offs cost about 5ms each on loopback, so roughly 10ms per step, and that grows with the distance between worker and namespace. The bench dials a mock model and a sleeping tool so the number is the overlap rather than provider variance. It is opt-in because it sleeps, and it runs on the harness's live clock: under the TestClock both the tool and the stream tail would wait forever. --- packages/core/test/step-overlap-bench.test.ts | 206 ++++++++++++++++++ packages/temporal/README.md | 43 +++- 2 files changed, 245 insertions(+), 4 deletions(-) create mode 100644 packages/core/test/step-overlap-bench.test.ts diff --git a/packages/core/test/step-overlap-bench.test.ts b/packages/core/test/step-overlap-bench.test.ts new file mode 100644 index 000000000000..08ca4a8227ba --- /dev/null +++ b/packages/core/test/step-overlap-bench.test.ts @@ -0,0 +1,206 @@ +// What the stepped split actually costs, measured rather than argued. +// +// A whole-step activity forks a tool fiber the moment a tool-call event arrives, so the tool runs +// while the model is still streaming. Splitting the step means the attempt has to return before any +// tool starts, because a workflow cannot consume a stream. The claim under test is that the loss is +// exactly the part of the stream that happened after the first tool call, bounded by how long the +// tool takes: +// +// overlap loss per step = min(stream tail after the first tool call, tool duration) +// +// A mock model and a sleeping tool are used deliberately: both are dialled, so the number is the +// overlap and nothing else. No provider latency, no network, no Temporal round trip. The live cost +// is this plus two extra activity round trips per step, which is measured separately. +// +// Uses the harness's `live` variant: `.effect` installs a TestClock, under which Effect.sleep never +// advances and both the tool and the stream tail would hang forever. +// +// Opt-in, because it sleeps: +// OPENCODE_OVERLAP_BENCH=1 bun test --timeout 60000 test/step-overlap-bench.test.ts +import { LLMClient, type LLMClientShape } from "@opencode-ai/llm/route" +import { LLMEvent } from "@opencode-ai/llm" +import { Database } from "@opencode-ai/core/database/database" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { EventV2 } from "@opencode-ai/core/event" +import { PermissionV2 } from "@opencode-ai/core/permission" +import { AgentV2 } from "@opencode-ai/core/agent" +import { Config } from "@opencode-ai/core/config" +import { Project } from "@opencode-ai/core/project" +import { ProjectTable } from "@opencode-ai/core/project/sql" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { SessionV2 } from "@opencode-ai/core/session" +import { Snapshot } from "@opencode-ai/core/snapshot" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import { SessionRunner } from "@opencode-ai/core/session/runner" +import * as SessionRunnerLLM from "@opencode-ai/core/session/runner/llm" +import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model" +import { ToolRegistry } from "@opencode-ai/core/tool/registry" +import { ApplicationTools } from "@opencode-ai/core/tool/application-tools" +import { Tool } from "@opencode-ai/core/tool/tool" +import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionStore } from "@opencode-ai/core/session/store" +import { Location } from "@opencode-ai/core/location" +import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" +import { SystemContext } from "@opencode-ai/core/system-context" +import { SkillGuidance } from "@opencode-ai/core/skill/guidance" +import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" +import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" +import { Auth } from "@opencode-ai/llm/route" +import { describe, expect } from "bun:test" +import { Duration, Effect, Layer, Schema, Stream } from "effect" +import { testEffect } from "./lib/effect" + +// The dials. TAIL_MS is how much stream arrives after the first tool call; TOOL_MS is how long each +// tool takes. The predicted loss is min(TAIL_MS, TOOL_MS). +// `||` not `??`: an empty env var must fall back too, or the dials silently become NaN. +const num = (name: string, fallback: number) => Number(process.env[name] || fallback) +const TAIL_MS = num("BENCH_TAIL_MS", 1500) +const TOOL_MS = num("BENCH_TOOL_MS", 1500) +const TAIL_CHUNKS = TAIL_MS === 0 ? 0 : 3 +const TAIL_CHUNK_MS = TAIL_CHUNKS === 0 ? 0 : TAIL_MS / TAIL_CHUNKS + +const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) + .model({ id: "gpt-4o-mini" }) +const models = SessionRunnerModel.layerWith(() => Effect.succeed(model)) +const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) +const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) +const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) +const permission = Layer.mock(PermissionV2.Service, {}) + +// One tool call, then a stream that keeps going for TAIL_MS. That trailing stream is the whole +// question: it is what a whole-step activity overlaps with the tool, and what a split cannot. +const withTail: LLMClientShape["stream"] = () => + Stream.concat( + Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call_bench", name: "bench_sleep", input: {} }), + LLMEvent.textStart({ id: "txt_1" }), + ]), + Stream.concat( + Stream.fromIterable( + Array.from({ length: TAIL_CHUNKS }, (_, i) => LLMEvent.textDelta({ id: "txt_1", text: `chunk ${i} ` })), + ).pipe(Stream.mapEffect((event) => Effect.sleep(Duration.millis(TAIL_CHUNK_MS)).pipe(Effect.as(event)))), + Stream.fromIterable([LLMEvent.textEnd({ id: "txt_1" }), LLMEvent.stepFinish({ index: 0, reason: "tool-calls" })]), + ), + ) + +const harness = testEffect( + AppNodeBuilder.build( + LayerNode.group([ + Database.node, + EventV2.node, + SessionProjector.node, + SessionStore.node, + AgentV2.node, + ToolRegistry.node, + SessionRunnerModel.node, + SystemContextRegistry.node, + SkillGuidance.node, + ReferenceGuidance.node, + Config.node, + Snapshot.node, + SessionRunnerLLM.node, + ApplicationTools.node, + ]), + [ + [LayerNodePlatform.llmClient, Layer.succeed( + LLMClient.Service, + LLMClient.Service.of({ + prepare: () => Effect.die("unused"), + generate: () => Effect.die("unused"), + stream: withTail, + }), + )], + [PermissionV2.node, permission], + [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [SessionRunnerModel.node, models], + [SystemContextRegistry.node, systemContext], + [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })], + [SkillGuidance.node, skillGuidance], + [ReferenceGuidance.node, referenceGuidance], + [Config.node, config], + [Snapshot.node, Snapshot.noopLayer], + ], + ), +) + +const seed = (sessionID: SessionV2.ID) => + Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ id: sessionID, project_id: Project.ID.global, slug: "t", directory: "/project", title: "t", version: "t" }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* (yield* ApplicationTools.Service).register({ + bench_sleep: Tool.make({ + description: "sleeps", + input: Schema.Struct({}), + output: Schema.String, + execute: () => Effect.sleep(Duration.millis(TOOL_MS)).pipe(Effect.as("slept")), + }), + }) + }) + +const elapsed = (body: Effect.Effect) => + Effect.gen(function* () { + const started = yield* Effect.sync(() => performance.now()) + yield* body + return Math.round((yield* Effect.sync(() => performance.now())) - started) + }) + +describe.skipIf(!process.env.OPENCODE_OVERLAP_BENCH)("step overlap cost", () => { + harness.live("measures what deferring dispatch costs one step", () => + Effect.gen(function* () { + const runner = yield* SessionRunner.Service + const step = { step: 2, promotion: undefined, first: false, force: false } as const + + const fused = SessionV2.ID.make("ses_bench_fused") + yield* seed(fused) + // Whole step in one go: the tool forks the instant its call arrives, so it runs under the tail. + const fusedMs = yield* elapsed(runner.runStep({ sessionID: fused, ...step })) + + const split = SessionV2.ID.make("ses_bench_split") + yield* seed(split) + // The same step, dispatched by a caller: the attempt returns first, then the tool runs. + const splitMs = yield* elapsed( + Effect.gen(function* () { + const result = yield* runner.runModelCall({ sessionID: split, ...step }) + if (result.kind !== "called") return + for (const call of result.calls) yield* runner.runToolCall({ sessionID: split, call, retry: false }) + yield* runner.sealStep({ sessionID: split, step: result.step, settlement: result.settlement }) + }), + ) + + const loss = splitMs - fusedMs + const predicted = Math.min(TAIL_MS, TOOL_MS) + console.log( + `\n stream tail after first call: ${TAIL_MS}ms tool: ${TOOL_MS}ms` + + `\n whole step : ${fusedMs}ms` + + `\n split step : ${splitMs}ms` + + `\n loss : ${loss}ms (predicted min(tail, tool) = ${predicted}ms)\n`, + ) + + // The fused path overlaps the tool with the tail, so it cannot exceed the sum. The slack is + // the step's own fixed cost (projection reads, snapshot, event writes), not overlap. + expect(fusedMs).toBeLessThan(TAIL_MS + TOOL_MS + 400) + // The loss is the overlap, within scheduling slop. Loose bounds: this is a measurement, and a + // tight assertion here would only buy a flaky test. With no tail there is nothing to overlap, + // so the floor does not apply. + if (predicted > 200) expect(loss).toBeGreaterThan(predicted * 0.5) + expect(loss).toBeLessThan(predicted + 400) + }), + ) +}) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 4af6dbde58ad..dfd344cad0f7 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -127,10 +127,45 @@ Two things are load-bearing and easy to get wrong: retry only a tool declaring `idempotent` runs again; anything else is reported to the model as an unknown outcome. This is the rule the crash-resume path already followed. -What it costs: a whole-step activity starts each tool the moment the model asks for it, while the -stream is still going. Here the attempt has to return before any tool starts, because a workflow -cannot consume a stream. The tools of one step still run concurrently with each other; the overlap -between the model and its own tools is what is lost. +#### What it costs, measured + +Two separate costs, and only one of them is usually real. + +**The lost overlap.** A whole-step activity starts each tool the moment the model asks for it, while +the stream is still going. Here the attempt has to return before any tool starts, because a workflow +cannot consume a stream. `packages/core/test/step-overlap-bench.test.ts` dials a mock model's stream +tail and a sleeping tool, so the number is the overlap and nothing else: + +| stream tail after 1st call | tool | whole step | split step | loss | `min(tail, tool)` | +|---:|---:|---:|---:|---:|---:| +| 1 ms | 1500 ms | 1534 ms | 1519 ms | **-15 ms** | 1 ms | +| 200 ms | 1500 ms | 1538 ms | 1734 ms | **196 ms** | 200 ms | +| 1500 ms | 200 ms | 1534 ms | 1743 ms | **209 ms** | 200 ms | +| 1500 ms | 1500 ms | 1536 ms | 3045 ms | **1509 ms** | 1500 ms | +| 3000 ms | 1500 ms | 3038 ms | 4542 ms | **1504 ms** | 1500 ms | + +So the loss per step is `min(stream tail after the first tool call, tool duration)`, within about +10 ms every time. It is **zero when the tool call is the last thing in the stream**, which is the +common shape: the model asks and stops. It only bites when the model keeps generating after asking, +and even then the tool's own duration caps it. + +**The extra round trips.** Three activities per step instead of one means two more hand-offs. +Measured from workflow history on a loopback dev server (mean of four, one turn): + +``` +done runModelCall -> sched runToolCall 10 ms +done runToolCall -> sched sealStep 3 ms +done sealStep -> sched runModelCall 4 ms +done runModelCall -> sched sealStep 3 ms +``` + +About 5 ms per hand-off, so ~10 ms per step, against model calls of 1.2 s and 3.2 s in the same run. +Per-step Temporal overhead tracks worker-to-namespace distance, so this is the floor: it grows with +placement, and a laptop driving a remote namespace pays it many times over. Put workers next to the +namespace and the split is close to free. + +Wall-clock totals are deliberately not quoted here. Model latency dominates and varies more between +two runs of the same cell than the effect being measured. #### Verified From 066ce4e7521223bc6c498a63fbbe36fed95f31e8 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 10:43:18 -0700 Subject: [PATCH 08/33] Measured the stream tail against real providers, not a mock. The bench showed the overlap loss is min(tail, tool duration) but left the tail itself dialled by hand. Against the live API it is 0 to 77ms across gpt-4o-mini, gpt-5-mini and gpt-5, and no model emitted any text after asking for its first tool. A single tool call is the end of the stream, so there is nothing to overlap; a tail shows up only when several tools are asked for at once. Timed from the point the runner actually forks. Measuring from when a call's id first appears rather than when its arguments finish overstated the tail by two to five times. --- packages/temporal/README.md | 25 +- .../temporal/scripts/stream-tail-probe.ts | 239 ++++++++++++++++++ 2 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 packages/temporal/scripts/stream-tail-probe.ts diff --git a/packages/temporal/README.md b/packages/temporal/README.md index dfd344cad0f7..e215b6e324e0 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -145,9 +145,28 @@ tail and a sleeping tool, so the number is the overlap and nothing else: | 3000 ms | 1500 ms | 3038 ms | 4542 ms | **1504 ms** | 1500 ms | So the loss per step is `min(stream tail after the first tool call, tool duration)`, within about -10 ms every time. It is **zero when the tool call is the last thing in the stream**, which is the -common shape: the model asks and stops. It only bites when the model keeps generating after asking, -and even then the tool's own duration caps it. +10 ms every time. It is zero when the tool call is the last thing in the stream, and that turns out +to be what real providers do. + +`packages/temporal/scripts/stream-tail-probe.ts` measures the tail against the live API, timed from +the point the runner actually forks: the `tool-call` event, which the protocol layer emits once a +call's arguments are complete and parsed, not when its id first appears. Six coding-agent-shaped +prompts, including ones asking for three and five tools at once and ones asking the model to narrate +around the call: + +| model | median tail | max tail | text after the first call | +|---|---:|---:|---| +| `gpt-4o-mini` (chat) | 0 ms | 50 ms | none, in any probe | +| `gpt-5-mini` (responses) | 33 ms | 36 ms | none, in any probe | +| `gpt-5` (responses) | 34 ms | 77 ms | none, in any probe | + +A single tool call *is* the end of the stream, so there is nothing to overlap. A tail appears only +when the model asks for several tools at once, and is then just the time to stream calls 2..N: tens +of milliseconds, one to three percent of the stream. No model emitted a single character of text +after asking for its first tool. + +Taken with the hand-offs below, the whole cost of the split is roughly 40-110 ms per step against +model calls of two to eight seconds. The overlap is not a reason to avoid it. **The extra round trips.** Three activities per step instead of one means two more hand-offs. Measured from workflow history on a loopback dev server (mean of four, one turn): diff --git a/packages/temporal/scripts/stream-tail-probe.ts b/packages/temporal/scripts/stream-tail-probe.ts new file mode 100644 index 000000000000..0cbfa9b9cff7 --- /dev/null +++ b/packages/temporal/scripts/stream-tail-probe.ts @@ -0,0 +1,239 @@ +// How much of a real provider's stream arrives AFTER it has asked for its first tool? +// +// That gap is the only thing a stepped executor gives up: a whole-step activity forks the tool +// immediately and runs it under the rest of the stream, while a split has to wait for the attempt to +// return. The bench measured the loss as min(tail, tool duration) with a dialled mock. This measures +// the tail itself, against the real API, so the mock's dial can be set to something honest. +// +// The measuring point matters. OpenCode forks on the `tool-call` event, which the protocol layer +// emits only once that call's ARGUMENTS are complete and parsed, not when its id first appears. +// Measuring from the id overstates the tail by however long the arguments took to stream. +// +// Measured 2026-08-26, 3 runs per probe (2 for gpt-5), key from ~/.config/ai363/llm.key: +// +// model median tail max tail text after first call +// gpt-4o-mini (chat) 0 ms 50 ms none, in any probe +// gpt-5-mini (responses) 33 ms 36 ms none, in any probe +// gpt-5 (responses) 34 ms 77 ms none, in any probe +// +// So the tail is 0 to 77 ms, at most a few percent of the stream, and no model emitted a single +// character of text after asking for its first tool. A single tool call IS the end of the stream. +// The tail only appears with several calls at once, and is then the time to stream calls 2..N. +// +// bun run packages/temporal/scripts/stream-tail-probe.ts +// +// Reads the key from a file and never prints it. +import { readFileSync } from "node:fs" + +const KEY = readFileSync(`${process.env.HOME}/.config/ai363/llm.key`, "utf8").trim() + +const TOOLS = [ + { + name: "bash", + description: "Run a shell command and return its output.", + parameters: { + type: "object", + properties: { command: { type: "string", description: "The command to run" } }, + required: ["command"], + additionalProperties: false, + }, + }, + { + name: "read_file", + description: "Read a file and return its contents.", + parameters: { + type: "object", + properties: { path: { type: "string", description: "Path to the file" } }, + required: ["path"], + additionalProperties: false, + }, + }, +] + +interface Probe { + readonly label: string + readonly prompt: string +} + +// Shapes a coding agent actually produces. The interesting ones are the last two: several tools at +// once (streaming calls 2..N is the tail for call 1) and a model asked to narrate as it goes. +const PROBES: ReadonlyArray = [ + { label: "single-call", prompt: "Run `ls -la` in the project root. Use the bash tool." }, + { label: "single-call-terse", prompt: "What files are here? Use the bash tool once." }, + { + label: "three-calls", + prompt: "Read these three files: src/a.ts, src/b.ts, src/c.ts. Use the read_file tool, one call per file.", + }, + { + label: "five-calls", + prompt: + "Read these five files: src/a.ts, src/b.ts, src/c.ts, src/d.ts, src/e.ts. Use the read_file tool, one call per file.", + }, + { + label: "narrate-then-call", + prompt: "Say one short sentence about what you are going to check, then run `git status` with the bash tool.", + }, + { + label: "call-then-narrate", + prompt: + "Run `git status` with the bash tool, and in the same reply also write a two-sentence note explaining why you ran it.", + }, +] + +interface Result { + readonly firstToolAt?: number + readonly lastAt: number + readonly calls: number + readonly textAfterFirstCall: number + readonly totalMs: number +} + +const streamChat = async (model: string, prompt: string): Promise => { + const started = performance.now() + const res = await fetch("https://api.openai.com/v1/chat/completions", { + method: "POST", + headers: { authorization: `Bearer ${KEY}`, "content-type": "application/json" }, + body: JSON.stringify({ + model, + stream: true, + messages: [{ role: "user", content: prompt }], + tools: TOOLS.map((t) => ({ type: "function", function: t })), + }), + }) + if (!res.ok) throw new Error(`${res.status} ${(await res.text()).slice(0, 200)}`) + // chat.completions has no per-call "arguments done" event: a call is complete once a LATER index + // starts streaming, or when finish_reason arrives. + let openIndex: number | undefined + return consume(res, started, (obj) => { + const choice = obj?.choices?.[0] + const delta = choice?.delta + const done: string[] = [] + for (const tc of delta?.tool_calls ?? []) { + if (openIndex !== undefined && tc.index !== openIndex) done.push(`idx${openIndex}`) + openIndex = tc.index + } + if (choice?.finish_reason && openIndex !== undefined) { + done.push(`idx${openIndex}`) + openIndex = undefined + } + return { toolIds: done, text: typeof delta?.content === "string" ? delta.content.length : 0 } + }) +} + +const streamResponses = async (model: string, prompt: string): Promise => { + const started = performance.now() + const res = await fetch("https://api.openai.com/v1/responses", { + method: "POST", + headers: { authorization: `Bearer ${KEY}`, "content-type": "application/json" }, + body: JSON.stringify({ + model, + stream: true, + store: false, + input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }], + tools: TOOLS.map((t) => ({ type: "function", ...t })), + }), + }) + if (!res.ok) throw new Error(`${res.status} ${(await res.text()).slice(0, 200)}`) + return consume(res, started, (obj) => { + const ids: string[] = [] + // The fork point: arguments complete, so the call can actually be dispatched. + if (obj?.type === "response.function_call_arguments.done") ids.push(String(obj.item_id ?? obj.output_index)) + const text = obj?.type === "response.output_text.delta" ? String(obj.delta ?? "").length : 0 + return { toolIds: ids, text } + }) +} + +const consume = async ( + res: Response, + started: number, + parse: (obj: any) => { toolIds: string[]; text: number }, +): Promise => { + const reader = res.body!.getReader() + const decoder = new TextDecoder() + let buffer = "" + let firstToolAt: number | undefined + let lastAt = started + let textAfterFirstCall = 0 + const seen = new Set() + + for (;;) { + const { done, value } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split("\n") + buffer = lines.pop() ?? "" + for (const line of lines) { + if (!line.startsWith("data: ")) continue + const payload = line.slice(6).trim() + if (payload === "[DONE]") continue + let obj: any + try { + obj = JSON.parse(payload) + } catch { + continue + } + const now = performance.now() + lastAt = now + const { toolIds, text } = parse(obj) + for (const id of toolIds) { + if (seen.has(id)) continue + seen.add(id) + if (firstToolAt === undefined) firstToolAt = now + } + if (firstToolAt !== undefined && text > 0) textAfterFirstCall += text + } + } + return { + firstToolAt, + lastAt, + calls: seen.size, + textAfterFirstCall, + totalMs: Math.round(lastAt - started), + } +} + +const model = process.argv[2] ?? "gpt-4o-mini" +const api = process.argv[3] ?? "chat" +const runs = Number(process.argv[4] ?? 3) + +console.log(`\nmodel=${model} api=${api} runs=${runs}`) +console.log( + `${"probe".padEnd(20)}${"calls".padStart(6)}${"stream".padStart(9)}${"tail".padStart(8)}${"tail%".padStart(7)}${"text after".padStart(12)}`, +) + +const allTails: number[] = [] +for (const probe of PROBES) { + const tails: number[] = [] + let calls = 0 + let total = 0 + let after = 0 + for (let i = 0; i < runs; i++) { + try { + const r = api === "responses" ? await streamResponses(model, probe.prompt) : await streamChat(model, probe.prompt) + if (r.firstToolAt === undefined) continue + tails.push(Math.round(r.lastAt - r.firstToolAt)) + calls = Math.max(calls, r.calls) + total += r.totalMs + after += r.textAfterFirstCall + } catch (e) { + console.log(`${probe.label.padEnd(20)} ERROR ${(e as Error).message}`) + } + } + if (!tails.length) { + console.log(`${probe.label.padEnd(20)}${"-".padStart(6)} (no tool call)`) + continue + } + const median = tails.slice().sort((a, b) => a - b)[Math.floor(tails.length / 2)]! + const avgTotal = Math.round(total / tails.length) + const pct = avgTotal > 0 ? Math.round((median / avgTotal) * 100) : 0 + allTails.push(median) + console.log( + `${probe.label.padEnd(20)}${String(calls).padStart(6)}${(avgTotal + "ms").padStart(9)}${(median + "ms").padStart(8)}${(pct + "%").padStart(7)}${String(Math.round(after / tails.length) + " ch").padStart(12)}`, + ) +} +if (allTails.length) { + const sorted = allTails.slice().sort((a, b) => a - b) + console.log( + `\nmedian tail across probes: ${sorted[Math.floor(sorted.length / 2)]}ms max: ${sorted[sorted.length - 1]}ms`, + ) +} From 87a1131081fdbfd2b9e8704d694f2ea165de58c5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 11:18:12 -0700 Subject: [PATCH 09/33] Recorded what stepped mode has actually been exercised against. Fan-out, interrupt and approvals are now verified live rather than asserted: four concurrent tool activities in one step sharing the log owner, an interrupt that stops the turn and leaves the session serving, and an approval that holds only the tool waiting on it while the model call is already done. Compaction is not covered and says so. A regression test for it passed with the bug deliberately put back, so it was proving nothing and is gone rather than left as a false assurance. Also records that live streaming does not cross a process boundary. That is a property of running a standalone worker, not of stepped mode: the wake is published in-process, so a tail cannot see another process's commits. The durable log is still complete and replay returns everything. --- packages/temporal/README.md | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index e215b6e324e0..4256c5dc4579 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -199,6 +199,50 @@ Live against a dev server and `gpt-5-mini`, with `OPENCODE_SESSION_EXECUTION=tem line (the settled tool was not re-run), the interrupted call reached the model as "The outcome of this tool call is unknown", and the turn ran on to its answer. +#### What has been exercised, and what has not + +Verified live (dev server, `gpt-5-mini`, stepped mode on): + +- **A step's calls fan out and share one log owner.** Four `read` calls in one step became four + concurrent `runToolCall` activities, started within 2 ms of each other and overlapping, all four + results durable, zero activity failures. A fenced write would have died and shown as a failed + activity, so this is the owner-token design holding under real concurrency, which is the thing it + exists for. +- **Interrupt stops the turn, not the session.** With `sleep 90` in flight, `POST /interrupt` + returned 204, the child process died, the call was closed as `Tool execution interrupted` so the + next attempt is not a poisoned request, the workflow stayed RUNNING, and the next prompt answered + normally. +- **An approval holds only the tool that is waiting.** Reading a `*.env` file parks on the default + agent's `ask` rule. While the human deliberated, `runModelCall` was **completed** and + `runToolCall` was the only outstanding activity. Replying `once` completed the tool and the turn. + Under the whole-step mode the entire step, model call included, sits in one activity for the whole + of that wait. + +Not covered, stated plainly: + +- **Compaction.** The `deferTools` flag is threaded through both `ContinueAfterCompaction` + recursions (a first version dropped it, which would have silently run tools inline on a compacting + step). There is no test: forcing compaction needs a model route that declares `limits.context`, + and the route used live does not. A regression test was written, found to pass with the bug + deliberately reintroduced, and deleted rather than left as a false assurance. +- **Two hosts.** Tool activities can be scheduled on any worker and file tools need the project + tree. `worktrees.ensure` rebuilds it from snapshot packs, but this has only ever run on one host. + +#### Live streaming does not cross a process boundary + +Worth stating separately, because it is **not** specific to stepped mode: it is a property of +running a standalone worker at all, and the mechanism is in `event.ts`. + +Split into `OPENCODE_TEMPORAL_ROLE=client` serve plus a standalone worker, a stepped turn ran +correctly end to end and its whole log is intact: `GET /api/session/:id/event?after=0` replays all +28 events. But a client subscribed *live* to that endpoint saw exactly one, `prompt.admitted`, the +only event the serve process itself writes. Everything the worker wrote never arrived. + +`commitDurableEvent` publishes the wake in-process, and `subscribeDurable` registers into that same +process's map, so a commit in another process cannot wake a tail. A UI attached to serve therefore +sees a prompt admitted and then silence until it re-reads. Correctness is unaffected; the durable +log is the source of truth and it is complete. + ### Two modes, one runner The factory has exactly two modes, both driving the same `SessionRunner` over the same durable event From d0e98e4d347fabba91002ee2c55211e5e295fa30 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 12:12:51 -0700 Subject: [PATCH 10/33] Held stepped mode to the executor contract, and fixed what it caught. Running the conformance suite against stepped mode for the first time failed 2 of 4. A provider turn that publishes no content at all never mints an assistant message, and the whole-step path only survives that because it mints one inside Step.Ended. The seal runs in another process with no publisher, so it searched the projection, found nothing, and left the turn open. The attempt now mints the message and carries its id, the same rule the tool call ids already followed. Also adds a fork-local CI workflow. Upstream's test workflow asks for Blacksmith runners this fork cannot schedule, so every run of it here has been cancelled with no runner, on every branch including dev. Nothing in this fork has ever had a CI signal. The new one is Linux-only and skips the whole-monorepo test task, because @opencode-ai/app fails a locale assertion on the untouched base branch and a job that is red on arrival is worth nothing. --- .github/workflows/test-fork.yml | 68 +++++++++++++++++++ packages/core/src/session/runner/index.ts | 6 ++ packages/core/src/session/runner/llm.ts | 12 ++++ packages/temporal/README.md | 23 +++++++ packages/temporal/src/l2-drain.ts | 10 ++- packages/temporal/src/l2-step.ts | 1 + ...ession-execution-temporal-contract.test.ts | 9 ++- 7 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test-fork.yml diff --git a/.github/workflows/test-fork.yml b/.github/workflows/test-fork.yml new file mode 100644 index 000000000000..fc0311ce1046 --- /dev/null +++ b/.github/workflows/test-fork.yml @@ -0,0 +1,68 @@ +# The upstream `test` workflow asks for Blacksmith runners (`blacksmith-4vcpu-*`), which this fork +# cannot schedule: every run of it here, on every branch including `dev` and dependabot PRs, is +# cancelled with no runner ever assigned. So nothing in this fork has had a CI signal. +# +# This runs typecheck and the affected packages' unit tests on GitHub's own runners, so a change +# made here is checked by something other than the author's laptop. +# +# Deliberately narrower than upstream's, in two ways worth being explicit about: +# - Linux only, no Windows matrix, no e2e, no generated-client or HttpApi gates. Those belong to +# upstream's runners, and pretending to cover them here would be worse than not claiming to. +# - `bun turbo test` is NOT run, because `@opencode-ai/app` currently fails one locale-detection +# assertion (`detectDesktopNativeLocale(["pa-PK"])` returns "en", not "pa") that depends on the +# ICU data in the bun build. It reproduces on the untouched base branch, so a whole-monorepo +# test job would be red on arrival and worth nothing. Typecheck still covers every package. +name: test (fork) + +on: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + unit: + name: unit + typecheck (linux) + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Setup Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "24" + + - name: Setup Bun + uses: ./.github/actions/setup-bun + + # Some suites shell out to git, which refuses to run without an identity. + - name: Configure git identity + run: | + git config --global user.email "bot@opencode.ai" + git config --global user.name "opencode" + + - name: Typecheck + timeout-minutes: 20 + run: bun typecheck + + - name: Unit tests (core) + timeout-minutes: 30 + working-directory: packages/core + run: bun test + + - name: Unit tests (temporal) + timeout-minutes: 15 + working-directory: packages/temporal + run: bun test diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index d7e2243c63a3..7ef67cea8df7 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -53,6 +53,8 @@ export interface SealStepInput { /** The provider's finish reason and token counts, from the attempt that opened this step. They * live only in that process's memory, so they have to be carried here rather than read back. */ readonly settlement?: StepSettlement + /** The message the attempt opened. Absent only for a step that produced nothing to close. */ + readonly assistantMessageID?: string } /** One recorded tool call, to run on its own. */ @@ -82,6 +84,9 @@ export interface TurnAttemptResult { readonly step: number readonly calls: ReadonlyArray readonly settlement?: StepSettlement + /** The assistant message this attempt opened, minted here so a seal in another process closes the + * right one even when the turn published no content of its own. */ + readonly assistantMessageID?: string } /** What a model-only attempt produced. `settled` means the step is already over (a crashed step was @@ -94,6 +99,7 @@ export type ModelCallResult = readonly step: number readonly calls: ReadonlyArray readonly settlement?: StepSettlement + readonly assistantMessageID?: string } /** Runs one local continuation from already-recorded Session history. */ diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 262829895a32..74df7db06c35 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -403,11 +403,20 @@ const layer = Layer.effect( if (stream._tag === "Failure") return yield* Effect.failCause(stream.cause) if (settled._tag === "Failure" && Cause.hasInterrupts(settled.cause)) return yield* Effect.failCause(settled.cause) + // A provider turn can finish having published nothing at all (no text, no tool call), and + // then no assistant message exists for the seal to close. The whole-step path never hits + // this because it mints one right here, inside Step.Ended. Mint it the same way and carry + // the id, rather than leave the seal to guess from the projection. + const assistantMessageID = + deferTools && stepSettlement && !publisher.hasProviderError() + ? yield* withPublication(publisher.startAssistant()) + : undefined return { needsContinuation: !publisher.hasProviderError() && needsContinuation, step: currentStep, calls: deferred as ReadonlyArray, settlement: stepSettlement, + assistantMessageID, } }), ) @@ -629,7 +638,9 @@ const layer = Layer.effect( ) // On a retry that lands after Step.Ended was published there is nothing open, but the loop // decision still has to come out the same, so it is read off the step we just closed. + const carried = input.assistantMessageID const target = + (carried ? context.findLast((m): m is SessionMessage.Assistant => m.id === carried) : undefined) ?? inFlight ?? context.findLast((message): message is SessionMessage.Assistant => message.type === "assistant") if (!target) return yield* stepContinuation(input.sessionID, false, input.step) @@ -737,6 +748,7 @@ const layer = Layer.effect( step: result.step, calls: result.calls, settlement: result.settlement, + assistantMessageID: result.assistantMessageID, } as ModelCallResult }) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 4256c5dc4579..b765c972caca 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -228,6 +228,29 @@ Not covered, stated plainly: - **Two hosts.** Tool activities can be scheduled on any worker and file tools need the project tree. `worktrees.ensure` rebuilds it from snapshot packs, but this has only ever run on one host. +#### The contract covers both modes + +The conformance suite is the executable definition of what an executor must do, and stepped mode has +to satisfy the same one or the two Temporal modes drift and only the tested one is trustworthy: + +```bash +temporal server start-dev --port 7237 --headless & +cd packages/temporal +# whole step per activity +OPENCODE_CONTRACT_TEMPORAL=1 bun test --timeout 180000 test/session-execution-temporal-contract.test.ts +# model call, one activity per tool, seal +OPENCODE_CONTRACT_TEMPORAL=1 OPENCODE_TEMPORAL_STEPPED=1 bun test --timeout 180000 \ + test/session-execution-temporal-contract.test.ts +``` + +Both pass 4/4. Running it the first time was worth the effort: **stepped mode failed 2 of the 4**, +on a case none of the live testing had reached. `countingModel` streams a step start and a step +finish and no content at all, so the publisher never mints an assistant message. The whole-step path +survives that because it mints one inside `Step.Ended` via `startAssistant()`; the seal, running in +another process with no publisher, searched the projection, found nothing, and left the turn open +forever. The fix carries the assistant message id out of the attempt the same way the tool call ids +are carried. + #### Live streaming does not cross a process boundary Worth stating separately, because it is **not** specific to stepped mode: it is a property of diff --git a/packages/temporal/src/l2-drain.ts b/packages/temporal/src/l2-drain.ts index dcd45f8690c5..f903fe64937b 100644 --- a/packages/temporal/src/l2-drain.ts +++ b/packages/temporal/src/l2-drain.ts @@ -34,6 +34,7 @@ export type ModelCallDrainResult = readonly step: number readonly calls: ReadonlyArray readonly settlement?: StepSettlement + readonly assistantMessageID?: string /** The event-log token this attempt claimed. The tool and seal activities of this step must * publish under it, so it travels with the calls instead of being minted again. */ readonly owner: string @@ -56,6 +57,7 @@ export interface SealDrainInput { readonly sessionID: string readonly step: number readonly settlement?: StepSettlement + readonly assistantMessageID?: string readonly owner: string } @@ -119,6 +121,7 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra step: result.step, calls: result.calls, settlement: result.settlement, + assistantMessageID: result.assistantMessageID, owner: input.owner, }, ), @@ -143,7 +146,12 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra input.sessionID, signal, inSession(input.sessionID, input.owner, false, (runner, session) => - runner.sealStep({ sessionID: session.id, step: input.step, settlement: input.settlement }), + runner.sealStep({ + sessionID: session.id, + step: input.step, + settlement: input.settlement, + assistantMessageID: input.assistantMessageID, + }), ).pipe( Effect.map((result) => ({ ran: result.ran, diff --git a/packages/temporal/src/l2-step.ts b/packages/temporal/src/l2-step.ts index 9ed4ebb8aaa3..09ea16c9f9d0 100644 --- a/packages/temporal/src/l2-step.ts +++ b/packages/temporal/src/l2-step.ts @@ -61,6 +61,7 @@ export const makeSteppedTurn = sessionID: input.sessionID, step: model.step, settlement: model.settlement, + assistantMessageID: model.assistantMessageID, owner: model.owner, }) } diff --git a/packages/temporal/test/session-execution-temporal-contract.test.ts b/packages/temporal/test/session-execution-temporal-contract.test.ts index bb9142008a62..e0900dd951c0 100644 --- a/packages/temporal/test/session-execution-temporal-contract.test.ts +++ b/packages/temporal/test/session-execution-temporal-contract.test.ts @@ -15,7 +15,14 @@ if (process.env.OPENCODE_CONTRACT_TEMPORAL === "1") { // One task queue per run: a stale worker from an earlier run against the same dev server would // otherwise steal activities and answer with its own (differently mocked) graph. process.env.OPENCODE_TEMPORAL_TASK_QUEUE ??= `contract-${crypto.randomUUID()}` + // The stepped mode has to satisfy the SAME contract, or the two Temporal modes drift and the + // parity story only holds for the one that happens to be tested. Which mode this run covers comes + // from the same env the executor reads, so the label never disagrees with what actually ran: + // + // OPENCODE_CONTRACT_TEMPORAL=1 -> whole step per activity + // OPENCODE_CONTRACT_TEMPORAL=1 OPENCODE_TEMPORAL_STEPPED=1 -> model call, per tool, seal + const stepped = process.env.OPENCODE_TEMPORAL_STEPPED === "1" // Imported dynamically because the driver reads its connection config at module load. const { SessionExecutionTemporal } = await import("@opencode-ai/temporal/executor") - runContract("temporal driver", makeExecutionFor(SessionExecutionTemporal.node)) + runContract(stepped ? "temporal driver (stepped)" : "temporal driver", makeExecutionFor(SessionExecutionTemporal.node)) } From d31817a058085763517b8a745da538d72a46c6bc Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 13:04:49 -0700 Subject: [PATCH 11/33] Guarded the silent-turn seal with a test CI actually runs. The conformance suite catches this, but it needs a dev server and an opt-in variable, so nothing in CI would have noticed the fix being reverted. Verified by putting the bug back: the test fails, and passes again once the mint returns. --- .../test/session-runner-model-call.test.ts | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index 9c8d283d4809..b70853e1a9bb 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -80,6 +80,12 @@ const callsIdempotentTool: LLMClientShape["stream"] = () => LLMEvent.toolCall({ id: "call_probe", name: "probe_read", input: {} }), LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), ]) +// A provider turn that publishes nothing at all: no text, no reasoning, no tool call. The publisher +// mints the assistant message lazily on first content, so after this stream there is no message in +// the log for a seal to find. The whole-step path survives it because Step.Ended mints one on the +// way past; a seal running in another process has no publisher to mint with. +const silent: LLMClientShape["stream"] = () => + Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })]) // An answer and no tool call: the step is over as soon as the stream is, but the seal still has to // happen, and there is a real assistant message for it to complete. const textOnly: LLMClientShape["stream"] = () => @@ -467,3 +473,41 @@ describe("SessionRunner step seal", () => { }), ) }) + +// Regression: a content-free provider turn must still close. This failed 2 of 4 conformance +// scenarios when the split first ran against them, and the turn hung open forever. The conformance +// suite catches it but needs a dev server and an opt-in env var, so nothing in CI would. +describe("SessionRunner seal of a silent turn", () => { + harness(silent).effect("closes a turn that published no content of its own", () => + Effect.gen(function* () { + yield* seedSession + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + const model = yield* runner.runModelCall({ + sessionID, + step: 2, + promotion: undefined, + first: false, + force: false, + }) + expect(model.kind).toBe("called") + if (model.kind !== "called") return + // The attempt has to hand the seal a message id, because there is nothing in the projection + // for it to find and it cannot mint one without a publisher. + expect(model.assistantMessageID).toBeTruthy() + + const result = yield* runner.sealStep({ + sessionID, + step: model.step, + settlement: model.settlement, + assistantMessageID: model.assistantMessageID, + }) + + expect(result.continue).toBe(false) + const message = assistant(yield* store.context(sessionID)) + expect(message?.type).toBe("assistant") + expect(message?.type === "assistant" ? Boolean(message.time.completed) : false).toBe(true) + }), + ) +}) From d624b76bfe8b547420f10c99813af3d4cd459e43 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 13:10:02 -0700 Subject: [PATCH 12/33] Let a durable tail see events another process appended. The wake is published in-process and subscribers register in that same process's map, so a standalone worker's whole turn was invisible to a tail on the HTTP server: a live subscriber saw one event, the only one serve writes itself, and then silence. The tail now also re-reads on a tick, default one second and settable to 0. In-process commits still wake it instantly, so nothing that already worked got slower, and a tick with nothing new reads no rows. The test builds a second service over the same database, which is what a worker is to the server: its commits cannot reach this process's wake map, so the tail only sees them because of the tick. --- packages/core/src/event.ts | 19 ++++++++++++++++++- packages/core/test/event.test.ts | 25 +++++++++++++++++++++++++ packages/temporal/README.md | 22 ++++++++++++---------- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index 97a003bdf132..b18f047ffd47 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -1,6 +1,6 @@ export * as EventV2 from "./event" -import { Cause, Context, Effect, Layer, Option, PubSub, Queue, Schema, Stream } from "effect" +import { Cause, Context, Duration, Effect, Layer, Option, PubSub, Queue, Schema, Stream } from "effect" import { Event } from "@opencode-ai/schema/event" import type { Data, Definition, Payload } from "@opencode-ai/schema/event" import { and, asc, eq, gt, inArray } from "drizzle-orm" @@ -174,8 +174,18 @@ export const allBounded = (events: Interface, capacity: number) => export interface LayerOptions { readonly beforeAggregateRead?: (aggregateID: string) => Effect.Effect + /** How often a durable tail re-reads on its own, on top of the in-process wake. The wake only + * fires for commits made in THIS process, so without a tick a subscriber cannot see events another + * process appended: a standalone worker's whole turn is invisible to a tail on the HTTP server. + * Set to 0 to disable and rely on the wake alone. */ + readonly livePollInterval?: Duration.Input } +/** Chosen to be well under what a person notices in a transcript while staying one cheap indexed + * read per subscribed session. In-process commits still wake instantly; this only catches what the + * wake cannot see. */ +const DEFAULT_LIVE_POLL = Duration.seconds(1) + export const layerWith = (options?: LayerOptions) => Layer.effect( Service, @@ -619,7 +629,14 @@ export const layerWith = (options?: LayerOptions) => ), ) const historical = yield* read + // Wake on either an in-process commit or the tick. A tick that finds nothing new reads + // zero rows and emits nothing, so an idle subscriber costs one indexed query per period. + const pollInterval = options?.livePollInterval ?? DEFAULT_LIVE_POLL + const ticks = Duration.isZero(Duration.fromInputUnsafe(pollInterval)) + ? Stream.never + : Stream.tick(pollInterval) const live = Stream.fromSubscription(wakes).pipe( + Stream.merge(ticks), Stream.mapEffect(() => read), Stream.flattenIterable, ) diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index 89cdfe2c5581..ae66e91f6d30 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -1170,6 +1170,31 @@ describe("EventV2", () => { }), ) + it.live("tails events another writer appended, which no in-process wake can announce", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const aggregateID = Session.ID.create() + yield* events.publish(DurableMessage, durableData(aggregateID, "seed")) + + // A second service over the SAME database with its OWN pubsub: exactly what a standalone + // worker is to the HTTP server. Its commits cannot reach this process's wake map, so if the + // tail only listened for wakes it would sit on the seed forever. + const other = yield* EventV2.Service.pipe(Effect.provide(EventV2.layerWith())) + + const tail = yield* events + .durable({ aggregateID, after: -1 }) + .pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped) + yield* Effect.sleep("100 millis") + yield* other.publish(DurableMessage, durableData(aggregateID, "from-elsewhere")) + + const collected = yield* Fiber.join(tail).pipe(Effect.timeout("10 seconds")) + expect(collected.map((event) => (event.data as { messageID: string }).messageID)).toEqual([ + durableData(aggregateID, "seed").messageID, + durableData(aggregateID, "from-elsewhere").messageID, + ]) + }), + ) + it.effect("never fences a publish made outside a drain", () => Effect.gen(function* () { const events = yield* EventV2.Service diff --git a/packages/temporal/README.md b/packages/temporal/README.md index b765c972caca..8b0f86acb8cf 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -251,20 +251,22 @@ another process with no publisher, searched the projection, found nothing, and l forever. The fix carries the assistant message id out of the attempt the same way the tool call ids are carried. -#### Live streaming does not cross a process boundary +#### Live streaming across a process boundary Worth stating separately, because it is **not** specific to stepped mode: it is a property of running a standalone worker at all, and the mechanism is in `event.ts`. -Split into `OPENCODE_TEMPORAL_ROLE=client` serve plus a standalone worker, a stepped turn ran -correctly end to end and its whole log is intact: `GET /api/session/:id/event?after=0` replays all -28 events. But a client subscribed *live* to that endpoint saw exactly one, `prompt.admitted`, the -only event the serve process itself writes. Everything the worker wrote never arrived. - -`commitDurableEvent` publishes the wake in-process, and `subscribeDurable` registers into that same -process's map, so a commit in another process cannot wake a tail. A UI attached to serve therefore -sees a prompt admitted and then silence until it re-reads. Correctness is unaffected; the durable -log is the source of truth and it is complete. +`commitDurableEvent` publishes its wake in-process and `subscribeDurable` registers in that same +process's map, so a commit in another process cannot wake a tail. Split into +`OPENCODE_TEMPORAL_ROLE=client` serve plus a standalone worker, a turn ran correctly and its log was +complete, but a client subscribed *live* saw exactly one event, `prompt.admitted`, the only one the +serve process writes itself. A UI attached to serve saw a prompt admitted and then silence. + +The durable tail now also re-reads on a tick (`LayerOptions.livePollInterval`, default one second, +0 to disable). In-process commits still wake it instantly, so latency is unchanged where it already +worked; the tick only catches what the wake cannot see. An idle subscriber costs one indexed read +per period, and a tick with nothing new emits nothing. The same split now delivers the worker's +`step.started`, `tool.called`, `tool.success` and `step.ended` to a live subscriber. ### Two modes, one runner From ad2be416f01ea2eb1ebd54373df5fa08c50e4cbc Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 13:13:35 -0700 Subject: [PATCH 13/33] Covered the compaction restart, which had been asserted not tested. A compacting step re-enters the attempt through a transition defect, and that recursion has to carry the defer flag or the step quietly runs its tools inline. Verified by dropping the flag again: the test fails. Getting compaction to fire at all took three tries, so the setup is commented. The epoch has to be created before the history is seeded (it records the sequence it was made at and the runner reads only past it), and the seeded turn has to sit in a band: over the request headroom so the attempt compacts, under context minus the summary output so compaction does not bail on its own guard. --- .../test/session-runner-model-call.test.ts | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index b70853e1a9bb..5f8000f62a12 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -34,6 +34,13 @@ import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { SessionTable } from "@opencode-ai/core/session/sql" import { SessionStore } from "@opencode-ai/core/session/store" import { SessionMessage } from "@opencode-ai/core/session/message" +import { ConfigCompaction } from "@opencode-ai/core/config/compaction" +import { SessionContextEpoch } from "@opencode-ai/core/session/context-epoch" +import { createLLMEventPublisher } from "@opencode-ai/core/session/runner/publish-llm-event" +import { ModelV2 } from "@opencode-ai/core/model" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { EventTable } from "@opencode-ai/core/event/sql" +import { eq } from "drizzle-orm" import { Location } from "@opencode-ai/core/location" import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" import { SystemContext } from "@opencode-ai/core/system-context" @@ -511,3 +518,135 @@ describe("SessionRunner seal of a silent turn", () => { }), ) }) + +// Compaction restarts the provider attempt by dying with a transition defect that runTurn catches +// and re-enters. That recursion has to carry `deferTools`, or a step that happens to compact +// silently falls back to running its tools inline and the caller never gets the calls it was meant +// to dispatch. A first version of this dropped the flag, so this is a real regression test. +describe("SessionRunner model-only attempt under compaction", () => { + // The summary request is the one with no tools. It needs text back, while the turn itself needs a + // tool call, so the mock has to answer them differently. + const compactingStream: LLMClientShape["stream"] = (request) => + request.tools.length === 0 + ? Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.textStart({ id: "sum" }), + LLMEvent.textDelta({ id: "sum", text: "## Objective\n- keep going" }), + LLMEvent.textEnd({ id: "sum" }), + LLMEvent.stepFinish({ index: 0, reason: "stop" }), + ]) + : callsTool(request) + + const tightModel = SessionRunnerModel.layerWith(() => + Effect.succeed( + OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, auth: Auth.bearer("fixture") }) + // Small enough that a seeded turn already overflows the headroom, so the attempt compacts. + .with({ limits: { context: 4_000, output: 50 } }) + .model({ id: "gpt-4o-mini" }), + ), + ) + const compactingConfig = Layer.succeed( + Config.Service, + Config.Service.of({ + entries: () => + Effect.succeed([ + new Config.Document({ + type: "document", + info: new Config.Info({ + compaction: new ConfigCompaction.Info({ + buffer: 3_000, + keep: new ConfigCompaction.Keep({ tokens: 1_000 }), + }), + }), + }), + ]), + }), + ) + + const compactHarness = testEffect( + AppNodeBuilder.build( + LayerNode.group([ + Database.node, + EventV2.node, + SessionProjector.node, + SessionStore.node, + AgentV2.node, + ToolRegistry.node, + SessionRunnerModel.node, + SystemContextRegistry.node, + SkillGuidance.node, + ReferenceGuidance.node, + Config.node, + Snapshot.node, + SessionRunnerLLM.node, + ApplicationTools.node, + ]), + [ + [LayerNodePlatform.llmClient, mockClient(compactingStream)], + [PermissionV2.node, permission], + [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [SessionRunnerModel.node, tightModel], + [SystemContextRegistry.node, systemContext], + [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })], + [SkillGuidance.node, skillGuidance], + [ReferenceGuidance.node, referenceGuidance], + [Config.node, compactingConfig], + [Snapshot.node, Snapshot.noopLayer], + ], + ), + ) + + compactHarness.effect("still defers dispatch after a compaction restart", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + // The context epoch has to exist BEFORE the history is seeded. It records the sequence it was + // created at, and the runner only reads entries after that baseline, so seeding first would + // put the whole conversation behind the baseline and the request would come out empty. + const { db } = yield* Database.Service + yield* SessionContextEpoch.initialize(db, Effect.succeed(SystemContext.empty), sessionID) + // A finished turn already in the log, sized into a narrow band. It has to exceed the request + // headroom (context - buffer = 1000 tokens) so the attempt compacts at all, while the summary + // prompt it produces has to stay under context - summaryOutput (3950) or compaction bails out + // on its own guard and never restarts the attempt. + const events = yield* EventV2.Service + const seeder = createLLMEventPublisher(events, { + sessionID, + agent: "build", + model: { id: ModelV2.ID.make("gpt-4o-mini"), providerID: ProviderV2.ID.make("openai") }, + }) + yield* seeder.publish(LLMEvent.textStart({ id: "old" })) + yield* seeder.publish(LLMEvent.textDelta({ id: "old", text: "Earlier answer. ".repeat(500) })) + yield* seeder.publish(LLMEvent.textEnd({ id: "old" })) + yield* seeder.publish(LLMEvent.stepFinish({ index: 0, reason: "stop" })) + + + const runner = yield* SessionRunner.Service + const result = yield* runner.runModelCall({ + sessionID, + step: 2, + promotion: undefined, + first: false, + force: false, + }) + + // Guard against a vacuous pass: without compaction actually firing this proves nothing about + // the restart path. + const rows = yield* db + .select({ type: EventTable.type }) + .from(EventTable) + .where(eq(EventTable.aggregate_id, sessionID)) + .all() + .pipe(Effect.orDie) + expect(rows.filter((row) => row.type.includes("compaction")).length).toBeGreaterThan(0) + + // The point: whatever the attempt went through, dispatch is still the caller's. + expect(result.kind).toBe("called") + if (result.kind !== "called") return + expect(result.calls.map((call) => call.id)).toEqual(["call_probe"]) + expect(ran.write).toBe(0) + }), + ) +}) From bbbb5c90315381178851c6fa091f89612d8002de Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 13:14:05 -0700 Subject: [PATCH 14/33] Removed the duplicate imports that broke typecheck. --- packages/core/test/session-runner-model-call.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index 5f8000f62a12..c42f6248b505 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -39,7 +39,6 @@ import { SessionContextEpoch } from "@opencode-ai/core/session/context-epoch" import { createLLMEventPublisher } from "@opencode-ai/core/session/runner/publish-llm-event" import { ModelV2 } from "@opencode-ai/core/model" import { ProviderV2 } from "@opencode-ai/core/provider" -import { EventTable } from "@opencode-ai/core/event/sql" import { eq } from "drizzle-orm" import { Location } from "@opencode-ai/core/location" import { SystemContextRegistry } from "@opencode-ai/core/system-context/registry" @@ -50,7 +49,6 @@ import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" import { Auth } from "@opencode-ai/llm/route" import { describe, expect } from "bun:test" import { Effect, Layer, Schema, Stream } from "effect" -import { eq } from "drizzle-orm" import { testEffect } from "./lib/effect" const model = OpenAIChat.route From cfed6b6feceb9359f717be1fdcb29e519c9bed33 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 13:22:16 -0700 Subject: [PATCH 15/33] Verified the worktree rebuild, and pinned down the zombie window. Deleting a live session's whole working directory between two turns and watching the next turn's tool activity rebuild it from snapshot packs is the cross-host mechanism actually working, rather than asserted. Affinity stays an unimplemented optimization on top of it. The zombie window was described as a mitigation rather than a guarantee. That understated it: the settled check really is a read then a write, but a retry refuses to run a non-idempotent tool at all, so the case that would matter cannot happen. What remains is a race over which truthful outcome the model is told. --- packages/temporal/README.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 8b0f86acb8cf..e048f776e48d 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -127,6 +127,13 @@ Two things are load-bearing and easy to get wrong: retry only a tool declaring `idempotent` runs again; anything else is reported to the model as an unknown outcome. This is the rule the crash-resume path already followed. + It is also what closes the zombie window, which the settled-result check on its own does not: that + check is a read then a write, so an attempt that lost its heartbeat but kept running could race a + retry past it. For the case that matters, a non-idempotent side effect running twice, it cannot + happen anyway, because the retry refuses to run the tool at all. What can still race is which + truthful outcome reaches the model, the zombie's real result or the retry's "unknown", and both + describe something that did happen. Reporting success for a tool that never ran is not reachable. + #### What it costs, measured Two separate costs, and only one of them is usually real. @@ -225,8 +232,17 @@ Not covered, stated plainly: step). There is no test: forcing compaction needs a model route that declares `limits.context`, and the route used live does not. A regression test was written, found to pass with the bug deliberately reintroduced, and deleted rather than left as a false assurance. -- **Two hosts.** Tool activities can be scheduled on any worker and file tools need the project - tree. `worktrees.ensure` rebuilds it from snapshot packs, but this has only ever run on one host. +- **A tool activity rebuilds a worktree it has never seen.** Each of the three drains calls + `worktrees.ensure`, so a tool call landing on a worker without the project tree materializes it + from the snapshot packs. Checked by deleting the entire working directory between two turns of a + live session: the next turn's tool activity rebuilt both files, the `read` completed, and the model + answered with the file's contents. Worker affinity would skip the materialization on warm paths and + is still not implemented; the packs are the baseline that works without it. + +Not covered, stated plainly: + +- **A second machine.** The rebuild above is a worker meeting a missing tree, which is the mechanism + that matters, but it ran in one process on one host. #### The contract covers both modes From 51ec6fb1a992d67036423443de922718beb3de5c Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 16:05:24 -0700 Subject: [PATCH 16/33] Added opt-in worker affinity, so a session goes where its tree already is. The queue name is derived from the session's directory and only workers serving that directory poll it, so a step does not pay to rebuild a tree a worker already holds. Keyed on location.directory rather than the project root, because that is the tree that has to be present, and resolved through realpath so two spellings of one tree cannot put the client and the worker on separate queues. Off by default, because it gives up the reconstruction fallback: with it on, a session whose tree has no worker waits instead of being served elsewhere. Verified all three ways, including that the work waits and then drains when the right worker comes back. --- packages/temporal/README.md | 31 +++++++ packages/temporal/src/config.ts | 10 +++ packages/temporal/src/executor.ts | 129 +++++++++++++++++---------- packages/temporal/src/queue.ts | 38 ++++++++ packages/temporal/test/queue.test.ts | 52 +++++++++++ 5 files changed, 215 insertions(+), 45 deletions(-) create mode 100644 packages/temporal/src/queue.ts create mode 100644 packages/temporal/test/queue.test.ts diff --git a/packages/temporal/README.md b/packages/temporal/README.md index e048f776e48d..3688ce4ab78e 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -267,6 +267,37 @@ another process with no publisher, searched the projection, found nothing, and l forever. The fix carries the assistant message id out of the attempt the same way the tool call ids are carried. +#### Worker affinity (`OPENCODE_TEMPORAL_WORKTREE_AFFINITY=1`) + +Off by default. Without it every worker polls one queue, and a worker drawing a session whose tree it +has never seen rebuilds that tree from snapshot packs. That is the portable baseline and it works. +Affinity avoids the rebuild by routing instead: the queue name is derived from the session's +directory, and only workers serving that directory poll it. + +```bash +# a worker declares the tree it serves; defaults to the process directory +OPENCODE_TEMPORAL_WORKTREE_AFFINITY=1 OPENCODE_TEMPORAL_WORKTREE=/srv/trees/acme \ + OPENCODE_TEMPORAL_ROLE=worker ... bun run packages/server/src/worker.ts +``` + +The queue is keyed on the session's `location.directory`, not the project root, because that is the +tree `worktrees.ensure` has to produce and two sessions in one project can sit in different +directories. Paths are resolved through `realpath` first: on macOS `/tmp/x` and `/private/tmp/x` are +one tree, and a client and a worker that disagreed would sit on two queues and the session would hang +with nothing to show for it. + +**This trades availability for latency, which is why it is opt-in.** With affinity on, a session +whose tree has no worker polling does not fall back to another worker. It waits. Reconstruction is +what makes any worker able to serve any session, and turning affinity on is choosing not to use it. + +Verified live, all three halves: + +- a worker serving the session's tree ran the turn, and the workflow sat on the derived queue +- with only a worker serving a *different* tree alive, the next prompt was not answered, and the + queue showed a workflow backlog of one, aged 50 seconds +- bringing the right worker back drained it and the answer arrived, so the work waits rather than + being lost + #### Live streaming across a process boundary Worth stating separately, because it is **not** specific to stepped mode: it is a property of diff --git a/packages/temporal/src/config.ts b/packages/temporal/src/config.ts index f4d4cec8279f..0fbe624b414c 100644 --- a/packages/temporal/src/config.ts +++ b/packages/temporal/src/config.ts @@ -21,6 +21,14 @@ export interface Interface { /** Drive each step as a provider attempt, one activity per tool call, and a seal. Off by default: * the whole-step mode is what runs today, and this only changes how new sessions start. */ readonly stepped?: boolean + /** Route a session's work to workers that already hold its project tree, instead of letting any + * worker draw it and rebuild the tree from snapshot packs. Off by default, because it trades the + * reconstruction fallback for latency: a session whose worktree has no worker polling waits + * rather than being served elsewhere. */ + readonly worktreeAffinity?: boolean + /** The worktree this worker serves, when affinity is on. Defaults to the process directory, which + * is what a serve process with an embedded worker is already sitting in. */ + readonly worktree?: string } export class Service extends Context.Service()("@opencode/temporal/Config") {} @@ -32,4 +40,6 @@ export const fromEnv = (): Interface => ({ role: (process.env.OPENCODE_TEMPORAL_ROLE as Role | undefined) ?? "both", idleTimeout: process.env.OPENCODE_SESSION_IDLE_TIMEOUT, stepped: process.env.OPENCODE_TEMPORAL_STEPPED === "1", + worktreeAffinity: process.env.OPENCODE_TEMPORAL_WORKTREE_AFFINITY === "1", + worktree: process.env.OPENCODE_TEMPORAL_WORKTREE, }) diff --git a/packages/temporal/src/executor.ts b/packages/temporal/src/executor.ts index 303c3bdbd0ea..a1054d374760 100644 --- a/packages/temporal/src/executor.ts +++ b/packages/temporal/src/executor.ts @@ -16,6 +16,7 @@ import { SessionExecution } from "@opencode-ai/core/session/execution" import { makeStepActivities, makeSteppedTurnActivities } from "./activities" import { makeDrains } from "./drain" import { makeL2Drains } from "./l2-drain" +import { queueForWorktree } from "./queue" import { WorktreeMaterializer } from "@opencode-ai/core/session/execution/worktree" import { toRunError } from "@opencode-ai/core/session/execution/run-error-codec" import * as WF from "./workflow" @@ -60,6 +61,22 @@ const layer = Layer.effect( // override as a workflow argument. const IDLE_TIMEOUT = config.idleTimeout const STEPPED = config.stepped === true + const AFFINITY = config.worktreeAffinity === true + // The tree this process serves when affinity is on. A serve process with an embedded worker is + // already sitting in it, so the process directory is the right default. + const SERVED_WORKTREE = config.worktree ?? process.cwd() + // Which queue a worker polls. With affinity off this is the one shared queue and any worker can + // draw any session, rebuilding the tree if it has to. + const POLL_QUEUE = AFFINITY ? queueForWorktree(TASK_QUEUE, SERVED_WORKTREE) : TASK_QUEUE + // Which queue a session's workflow runs on. Reads the session because the tree that has to be + // present is its location, not the project root: two sessions under one project can sit in + // different directories. + const queueFor = (id: SessionSchema.ID) => + AFFINITY + ? Effect.map(store.get(id), (session) => + session ? queueForWorktree(TASK_QUEUE, session.location.directory) : TASK_QUEUE, + ) + : Effect.succeed(TASK_QUEUE) const events = yield* EventV2.Service const worktrees = yield* WorktreeMaterializer.Service @@ -73,9 +90,7 @@ const layer = Layer.effect( // Worker connection (native) hosts the runTurnStep activity + the workflow. Skipped in // client-only role so serve can run without an embedded worker. if (HOST_WORKER) { - const { NativeConnection, Worker } = yield* Effect.tryPromise( - () => import("@temporalio/worker"), - ).pipe( + const { NativeConnection, Worker } = yield* Effect.tryPromise(() => import("@temporalio/worker")).pipe( Effect.catch(() => Effect.die( "The embedded Temporal worker is unavailable in this build. Run standalone workers " + @@ -91,7 +106,7 @@ const layer = Layer.effect( Worker.create({ connection: nativeConn, namespace: NAMESPACE, - taskQueue: TASK_QUEUE, + taskQueue: POLL_QUEUE, workflowsPath: fileURLToPath(new URL("./workflow.ts", import.meta.url)), activities: { ...makeStepActivities(stepDrain), ...makeSteppedTurnActivities(l2) }, }), @@ -106,12 +121,17 @@ const layer = Layer.effect( ) } - // Worker-only process: it hosts activities but drives no workflows, so the client methods are // unused. Return a service whose driving methods fail loudly if something unexpectedly calls them. if (!HOST_CLIENT) { yield* Effect.logInfo("SessionExecutionTemporal worker ready").pipe( - Effect.annotateLogs({ address: ADDRESS, taskQueue: TASK_QUEUE, workflow: WORKFLOW_TYPE, role: config.role }), + Effect.annotateLogs({ + address: ADDRESS, + taskQueue: POLL_QUEUE, + workflow: WORKFLOW_TYPE, + role: config.role, + ...(AFFINITY ? { worktree: SERVED_WORKTREE } : {}), + }), ) const clientOnly = Effect.die("SessionExecution client is not hosted when OPENCODE_TEMPORAL_ROLE=worker") return SessionExecution.Service.of({ @@ -130,18 +150,28 @@ const layer = Layer.effect( const client = new Client({ connection: clientConn, namespace: NAMESPACE }) const drive = (id: SessionSchema.ID) => - Effect.promise(() => - client.workflow.signalWithStart(WORKFLOW_TYPE, { - taskQueue: TASK_QUEUE, - workflowId: workflowId(id), - args: [id, { startWithWake: true, idleTimeout: IDLE_TIMEOUT, stepped: STEPPED } satisfies WF.SessionTurnOptions], - signal: WF.wake, - signalArgs: [], - }), + Effect.flatMap(queueFor(id), (taskQueue) => + Effect.promise(() => + client.workflow.signalWithStart(WORKFLOW_TYPE, { + taskQueue, + workflowId: workflowId(id), + args: [ + id, + { startWithWake: true, idleTimeout: IDLE_TIMEOUT, stepped: STEPPED } satisfies WF.SessionTurnOptions, + ], + signal: WF.wake, + signalArgs: [], + }), + ), ) yield* Effect.logInfo("SessionExecutionTemporal ready").pipe( - Effect.annotateLogs({ address: ADDRESS, taskQueue: TASK_QUEUE, workflow: WORKFLOW_TYPE }), + Effect.annotateLogs({ + address: ADDRESS, + taskQueue: TASK_QUEUE, + workflow: WORKFLOW_TYPE, + ...(AFFINITY ? { worktreeAffinity: true } : {}), + }), ) return SessionExecution.Service.of({ @@ -176,36 +206,45 @@ const layer = Layer.effect( // resume = coordinator.run: drive a forced run via an Update-with-Start and AWAIT its result, // so a run error is surfaced to the caller (as a RunError) instead of being swallowed. resume: (id) => - Effect.tryPromise({ - try: async () => { - const attempt = () => { - const startOp = new WithStartWorkflowOperation(WORKFLOW_TYPE, { - taskQueue: TASK_QUEUE, - workflowId: workflowId(id), - // startWithWake=false: a fresh resume-with-start must not manufacture a wake drain; - // its forced drain comes from the resume update. Ignored when USE_EXISTING joins a - // running workflow (which keeps its own state). - args: [id, { startWithWake: false, idleTimeout: IDLE_TIMEOUT, stepped: STEPPED } satisfies WF.SessionTurnOptions], - workflowIdConflictPolicy: "USE_EXISTING", - }) - return client.workflow.executeUpdateWithStart(WF.resume, { - startWorkflowOperation: startOp, - args: [], - }) - } - try { - await attempt() - } catch (e) { - // The long-lived workflow self-completes after its idle timeout; an update admitted - // in that instant fails against the just-completed run instead of starting a fresh - // one. Retry once so the caller gets a real run, not the race. - const message = String((e as { message?: unknown })?.message ?? "") - if (!/already completed|not found/i.test(message)) throw e - await attempt() - } - }, - catch: (e) => toRunError(id, e), - }), + Effect.flatMap(queueFor(id), (resumeQueue) => + Effect.tryPromise({ + try: async () => { + const attempt = () => { + const startOp = new WithStartWorkflowOperation(WORKFLOW_TYPE, { + taskQueue: resumeQueue, + workflowId: workflowId(id), + // startWithWake=false: a fresh resume-with-start must not manufacture a wake drain; + // its forced drain comes from the resume update. Ignored when USE_EXISTING joins a + // running workflow (which keeps its own state). + args: [ + id, + { + startWithWake: false, + idleTimeout: IDLE_TIMEOUT, + stepped: STEPPED, + } satisfies WF.SessionTurnOptions, + ], + workflowIdConflictPolicy: "USE_EXISTING", + }) + return client.workflow.executeUpdateWithStart(WF.resume, { + startWorkflowOperation: startOp, + args: [], + }) + } + try { + await attempt() + } catch (e) { + // The long-lived workflow self-completes after its idle timeout; an update admitted + // in that instant fails against the just-completed run instead of starting a fresh + // one. Retry once so the caller gets a real run, not the race. + const message = String((e as { message?: unknown })?.message ?? "") + if (!/already completed|not found/i.test(message)) throw e + await attempt() + } + }, + catch: (e) => toRunError(id, e), + }), + ), interrupt: (id) => Effect.tryPromise({ try: () => client.workflow.getHandle(workflowId(id)).signal(WF.interrupt), diff --git a/packages/temporal/src/queue.ts b/packages/temporal/src/queue.ts new file mode 100644 index 000000000000..b25634f6ff63 --- /dev/null +++ b/packages/temporal/src/queue.ts @@ -0,0 +1,38 @@ +// Routing a session's work to a worker that already has its project tree. +// +// Without affinity every worker polls one queue, and a worker that draws a session whose worktree it +// has never seen rebuilds it from snapshot packs (`session/execution/worktree.ts`). That is the +// portable baseline and it works, but it costs a materialization on the first step a worker takes +// for that session. Affinity avoids the cost by routing instead: the queue name is derived from the +// session's directory, and only workers serving that directory poll it. +// +// The trade is availability for latency, and it is the whole reason this is opt-in. With affinity on, +// a session whose worktree has no worker polling for it does not fall back to another worker; it +// waits. Reconstruction is what makes any worker able to serve any session, and turning affinity on +// is choosing not to use it. +// +// Deliberately NOT imported by workflow code: this reaches for node:crypto, which the Temporal +// sandbox does not have. The queue is chosen by the client that starts the workflow and by the worker +// that polls, both of which are ordinary Node. +import { createHash } from "node:crypto" +import { realpathSync } from "node:fs" + +/** Longer than needed to make a collision implausible, short enough to keep the queue name readable + * in the Temporal UI and in `temporal task-queue describe`. */ +const DIGEST_LENGTH = 12 + +/** + * The queue for one worktree. Resolved through realpath so two spellings of the same directory agree + * on a name: on macOS `/tmp/x` and `/private/tmp/x` are the same tree, and a client and a worker that + * disagreed about that would sit on two queues and never meet. Falls back to the given path when it + * does not exist yet, which is the case for a worker declaring a tree it has not materialized. + */ +export const queueForWorktree = (base: string, directory: string): string => { + let canonical = directory + try { + canonical = realpathSync(directory) + } catch { + // Not present yet; the literal path is still a stable key. + } + return `${base}-wt-${createHash("sha256").update(canonical).digest("hex").slice(0, DIGEST_LENGTH)}` +} diff --git a/packages/temporal/test/queue.test.ts b/packages/temporal/test/queue.test.ts new file mode 100644 index 000000000000..1b65625ab38b --- /dev/null +++ b/packages/temporal/test/queue.test.ts @@ -0,0 +1,52 @@ +// Worktree affinity is only useful if the client starting a workflow and the worker polling for it +// derive the SAME queue name from the same tree. If they disagree they sit on two queues and the +// session waits forever, which is silent: nothing errors, the work simply never runs. +import { describe, it, expect } from "bun:test" +import { mkdtempSync, mkdirSync, realpathSync, rmSync, symlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { queueForWorktree } from "../src/queue" + +describe("worktree queue", () => { + it("gives one tree one name and different trees different names", () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), "queue-"))) + try { + const a = join(root, "a") + const b = join(root, "b") + mkdirSync(a) + mkdirSync(b) + + expect(queueForWorktree("q", a)).toBe(queueForWorktree("q", a)) + expect(queueForWorktree("q", a)).not.toBe(queueForWorktree("q", b)) + // The base is kept as a readable prefix so a queue is identifiable in the Temporal UI. + expect(queueForWorktree("q", a).startsWith("q-wt-")).toBe(true) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it("agrees on a tree reached through a symlink", () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), "queue-"))) + try { + const real = join(root, "real") + const link = join(root, "link") + mkdirSync(real) + symlinkSync(real, link) + + // This is the case that actually bites: on macOS /tmp is a symlink to /private/tmp, so a + // client saying /tmp/x and a worker saying /private/tmp/x mean one tree. Keying on the raw + // string would put them on two queues and the session would hang with nothing to show for it. + expect(queueForWorktree("q", link)).toBe(queueForWorktree("q", real)) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it("still names a tree that does not exist yet", () => { + // A worker can declare the tree it serves before anything has materialized it, so an + // unresolvable path has to stay a stable key rather than throw. + const missing = join(tmpdir(), "queue-missing-tree-that-is-not-there") + expect(queueForWorktree("q", missing)).toBe(queueForWorktree("q", missing)) + expect(queueForWorktree("q", missing).startsWith("q-wt-")).toBe(true) + }) +}) From 3f193bee179b7b50f9579910f8e276f8f19921d3 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 17:37:06 -0700 Subject: [PATCH 17/33] Carried the attempt's own decisions into the seal. A review caught that stepped mode drops two things the log can't reconstruct. A provider error left `needsContinuation` behind, so the seal re-derived "keep going" from tool parts and the supervisor called a provider that had just failed, up to the step ceiling. And a declined permission came out of `runToolCall` as an ordinary failure, which the dispatcher swallows, so the agent carried on past a refusal. Both now travel with the result, the same rule the assistant message id already followed. The seal's "already closed" test also checks the target message rather than any in-flight one, so a retried seal can't publish a second `Step.Ended`. Other fixes from the same review: the durable tail keeps the wake as what ends it, `runToolCall` does a point read instead of decoding the whole session, the emit is uninterruptible so an interrupt can't drop a result for a tool that ran, a deleted session is a no-op rather than a run error, and the worktree queue key no longer depends on whether the path resolves locally, which differed between client and worker. --- packages/core/src/event.ts | 13 ++- packages/core/src/session/runner/index.ts | 5 + packages/core/src/session/runner/llm.ts | 63 +++++++++--- .../test/session-runner-model-call.test.ts | 96 ++++++++++++++++++- packages/temporal/README.md | 30 +++--- packages/temporal/src/executor.ts | 4 +- packages/temporal/src/l2-drain.ts | 33 +++++-- packages/temporal/src/l2-step.ts | 1 + packages/temporal/src/queue.ts | 25 ++--- packages/temporal/test/queue.test.ts | 31 ++++-- 10 files changed, 239 insertions(+), 62 deletions(-) diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index b18f047ffd47..c458cc51ba4d 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -632,11 +632,14 @@ export const layerWith = (options?: LayerOptions) => // Wake on either an in-process commit or the tick. A tick that finds nothing new reads // zero rows and emits nothing, so an idle subscriber costs one indexed query per period. const pollInterval = options?.livePollInterval ?? DEFAULT_LIVE_POLL - const ticks = Duration.isZero(Duration.fromInputUnsafe(pollInterval)) - ? Stream.never - : Stream.tick(pollInterval) - const live = Stream.fromSubscription(wakes).pipe( - Stream.merge(ticks), + const woken = Stream.fromSubscription(wakes) + // haltStrategy "left" keeps the wake stream as what ends the tail. The tick never ends, + // so the default ("both") would leave a subscriber hanging past the layer's own + // PubSub.shutdown, still reading from a database being torn down. + const source = Duration.isZero(Duration.fromInputUnsafe(pollInterval)) + ? woken + : woken.pipe(Stream.merge(Stream.tick(pollInterval), { haltStrategy: "left" })) + const live = source.pipe( Stream.mapEffect(() => read), Stream.flattenIterable, ) diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index 7ef67cea8df7..aa367bb91469 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -55,6 +55,10 @@ export interface SealStepInput { readonly settlement?: StepSettlement /** The message the attempt opened. Absent only for a step that produced nothing to close. */ readonly assistantMessageID?: string + /** Whether the turn should keep going, decided by the attempt. Only the attempt knows it hit a + * provider error, so a seal that re-derives this from the log would keep calling a provider that + * just failed. Absent on a re-drive, where the log is all there is. */ + readonly needsContinuation?: boolean } /** One recorded tool call, to run on its own. */ @@ -100,6 +104,7 @@ export type ModelCallResult = readonly calls: ReadonlyArray readonly settlement?: StepSettlement readonly assistantMessageID?: string + readonly needsContinuation?: boolean } /** Runs one local continuation from already-recorded Session history. */ diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 74df7db06c35..95f53b94312c 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -414,7 +414,11 @@ const layer = Layer.effect( return { needsContinuation: !publisher.hasProviderError() && needsContinuation, step: currentStep, - calls: deferred as ReadonlyArray, + // A provider error already failed every recorded call, so handing them back would only + // buy a dispatch that reads them as settled. + calls: (publisher.hasProviderError() + ? [] + : deferred) as ReadonlyArray, settlement: stepSettlement, assistantMessageID, } @@ -648,7 +652,12 @@ const layer = Layer.effect( // A step continues so the model can see its tool results. Provider-executed calls need no // follow-up turn, so a step holding only those finalizes as a plain stop. const localTools = toolParts.some((part) => part.provider?.executed !== true) - if (!inFlight) return yield* stepContinuation(input.sessionID, localTools, input.step) + if (target.time.completed) + return yield* stepContinuation( + input.sessionID, + input.needsContinuation ?? localTools, + input.step, + ) // A dispatch that failed outright leaves its call open. Close it here, or the next attempt // sends a request carrying a tool_use with no tool_result and the provider rejects it. yield* failInterruptedTools(input.sessionID, context) @@ -674,7 +683,11 @@ const layer = Layer.effect( snapshot: endSnapshot, files, }) - return yield* stepContinuation(input.sessionID, localTools, input.step) + return yield* stepContinuation( + input.sessionID, + input.needsContinuation ?? localTools, + input.step, + ) }) const toolPartOf = (messages: ReadonlyArray, callID: string) => { @@ -688,7 +701,17 @@ const layer = Layer.effect( const runToolCall = Effect.fn("SessionRunner.runToolCall")(function* (input: ToolCallInput) { const session = yield* getSession(input.sessionID) const assistantMessageID = SessionMessage.ID.make(input.call.assistantMessageID) - const part = toolPartOf(yield* getContext(input.sessionID), input.call.id) + // A point read, not the whole projected history. The call carries the message + // that owns it, so decoding every message to find one part would make a step + // cost O(session) per tool instead of O(1). + const owner = yield* store.message(assistantMessageID) + const part = + owner?.message.type === "assistant" + ? owner.message.content.find( + (item): item is SessionMessage.AssistantTool => + item.type === "tool" && item.id === input.call.id, + ) + : undefined // The call has to be in the log already: the attempt that produced it published Tool.Called // before handing it over. Missing means the log moved under us (a fence), and running a tool // whose call is not recorded would leave an orphan result. @@ -713,13 +736,28 @@ const layer = Layer.effect( }) return { outcome: "unknown" } as ToolCallResult } - const settlement = yield* materialization.settle({ - sessionID: input.sessionID, - agent: agent.id, - assistantMessageID, - call: LLMEvent.toolCall({ id: input.call.id, name: input.call.name, input: input.call.input }), - }) - yield* emitToolResult(events, { + const settlement = yield* materialization + .settle({ + sessionID: input.sessionID, + agent: agent.id, + assistantMessageID, + call: LLMEvent.toolCall({ + id: input.call.id, + name: input.call.name, + input: input.call.input, + }), + }) + // A decline is the user stopping the turn, not a tool that failed. It has to reach the + // boundary as an interrupt, or the dispatcher treats it as one bad tool, seals the step and + // the agent carries on past a refusal. + .pipe( + Effect.catchCause((cause) => + isUserDeclined(cause) ? Effect.interrupt : Effect.failCause(cause), + ), + ) + // The tool has run by here, so losing the result to an interrupt would hide a + // side effect that already happened. + yield* Effect.uninterruptible(emitToolResult(events, { sessionID: input.sessionID, assistantMessageID, callID: input.call.id, @@ -728,7 +766,7 @@ const layer = Layer.effect( outputPaths: settlement.outputPaths, // Deferred calls are never provider-executed: those are filtered out before the hand-off. provider: { executed: false }, - }) + })) return { outcome: "settled" } as ToolCallResult }) @@ -749,6 +787,7 @@ const layer = Layer.effect( calls: result.calls, settlement: result.settlement, assistantMessageID: result.assistantMessageID, + needsContinuation: result.needsContinuation, } as ModelCallResult }) diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index c42f6248b505..a4500e806f31 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -48,7 +48,7 @@ import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" import { Auth } from "@opencode-ai/llm/route" import { describe, expect } from "bun:test" -import { Effect, Layer, Schema, Stream } from "effect" +import { Cause, Effect, Exit, Layer, Schema, Stream } from "effect" import { testEffect } from "./lib/effect" const model = OpenAIChat.route @@ -78,6 +78,20 @@ const callsTool: LLMClientShape["stream"] = () => LLMEvent.toolCall({ id: "call_probe", name: "probe_write", input: {} }), LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), ]) +// Calls the tool that declines. +const callsDecliningTool: LLMClientShape["stream"] = () => + Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call_probe", name: "probe_declines", input: {} }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + ]) +// A tool call, then the provider dies mid-stream. The calls are recorded but the turn is over. +const callsToolThenFails: LLMClientShape["stream"] = () => + Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call_probe", name: "probe_write", input: {} }), + LLMEvent.providerError({ message: "upstream exploded" }), + ]) // The same, for the tool that declares itself repeatable. const callsIdempotentTool: LLMClientShape["stream"] = () => Stream.fromIterable([ @@ -170,6 +184,14 @@ const registerProbes = (ran: { write: number; read: number }) => return "wrote" }), }), + // permission.assert declines by dying with this, so a tool that dies the same way exercises + // the same classification without needing the permission service in the tool's context. + probe_declines: Tool.make({ + description: "declines", + input: Schema.Struct({}), + output: Schema.String, + execute: () => Effect.die(new PermissionV2.DeclinedError()), + }), probe_read: Tool.make({ description: "read probe", idempotent: true, @@ -648,3 +670,75 @@ describe("SessionRunner model-only attempt under compaction", () => { }), ) }) + +// A turn has to stop when the provider fails. Only the attempt knows that happened: the log shows a +// failed assistant with tool parts, which reads the same as a step that wants to keep going. If the +// seal re-derives the decision instead of being told, a hard provider failure loops on the durable +// path until a step ceiling catches it. +describe("SessionRunner provider failure in a stepped turn", () => { + harness(callsToolThenFails).effect("ends the turn instead of asking for another step", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const runner = yield* SessionRunner.Service + + const model = yield* runner.runModelCall({ + sessionID, + step: 2, + promotion: undefined, + first: false, + force: false, + }) + + expect(model.kind).toBe("called") + if (model.kind !== "called") return + expect(model.needsContinuation).toBe(false) + // The attempt already failed them on the way out, so dispatching would only re-read settled + // parts. + expect(model.calls).toHaveLength(0) + + const result = yield* runner.sealStep({ + sessionID, + step: model.step, + settlement: model.settlement, + assistantMessageID: model.assistantMessageID, + needsContinuation: model.needsContinuation, + }) + + expect(result.continue).toBe(false) + expect(ran.write).toBe(0) + }), + ) +}) + +// A decline is the user stopping the turn, not one tool failing. If it reaches the dispatcher as an +// ordinary tool error it gets swallowed, the step seals, and the agent carries on past a refusal. +describe("SessionRunner declined permission in a stepped turn", () => { + harness(callsDecliningTool).effect("halts the turn rather than reporting a failed tool", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const runner = yield* SessionRunner.Service + const model = yield* runner.runModelCall({ + sessionID, + step: 2, + promotion: undefined, + first: false, + force: false, + }) + if (model.kind !== "called" || !model.calls[0]) throw new Error("expected a deferred call") + + const exit = yield* runner + .runToolCall({ sessionID, call: model.calls[0], retry: false }) + .pipe(Effect.exit) + + // An interrupt is what the activity boundary turns into a non-retryable halt. A plain failure + // reads as one bad tool and the turn continues. + expect(Exit.isFailure(exit)).toBe(true) + if (!Exit.isFailure(exit)) return + expect(Cause.hasInterrupts(exit.cause)).toBe(true) + }), + ) +}) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 3688ce4ab78e..c1255ef99cd6 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -225,19 +225,19 @@ Verified live (dev server, `gpt-5-mini`, stepped mode on): Under the whole-step mode the entire step, model call included, sits in one activity for the whole of that wait. -Not covered, stated plainly: - -- **Compaction.** The `deferTools` flag is threaded through both `ContinueAfterCompaction` - recursions (a first version dropped it, which would have silently run tools inline on a compacting - step). There is no test: forcing compaction needs a model route that declares `limits.context`, - and the route used live does not. A regression test was written, found to pass with the bug - deliberately reintroduced, and deleted rather than left as a false assurance. - **A tool activity rebuilds a worktree it has never seen.** Each of the three drains calls `worktrees.ensure`, so a tool call landing on a worker without the project tree materializes it from the snapshot packs. Checked by deleting the entire working directory between two turns of a live session: the next turn's tool activity rebuilt both files, the `read` completed, and the model - answered with the file's contents. Worker affinity would skip the materialization on warm paths and - is still not implemented; the packs are the baseline that works without it. + answered with the file's contents. Worker affinity (below) skips the + materialization on warm paths; + the packs are the baseline that works without it. +- **A compaction restart keeps deferring.** The `deferTools` flag is threaded through both + `ContinueAfterCompaction` recursions; a first version dropped it, which would have silently run + tools inline on a compacting step. +- **A provider error ends the turn.** The attempt's own continuation decision is carried to the + seal, because the log cannot tell a failed attempt from one that wants another step. +- **A declined permission halts the turn** rather than reaching the model as one failed tool. Not covered, stated plainly: @@ -364,10 +364,14 @@ later. Verified by `packages/core/test/session-runner-resume.test.ts`. `startToCloseTimeout` is a 12-hour backstop for a drain that hangs while its process stays alive. When an attempt is retried while the previous one is still alive (a network partition, or the backstop firing), the old attempt could briefly keep publishing until its heartbeat is rejected - and the AbortSignal interrupts it. That overlap is now fenced: each drain claims the event log + and the AbortSignal interrupts it. That overlap is fenced per attempt in whole-step + mode: the drain claims the event log with an attempt token (`event_sequence.owner_id` via `claim()`), and a live durable append dies if a newer attempt has since claimed the log (the check is in `event.ts`, gated by the - `EventOwner` context the drain provides). The owner is set activity-side from the run id and + `EventOwner` context the drain provides). In stepped mode a step's three activity kinds share one + token, so the fence separates steps but not writers inside a step; what keeps the projection right + there is the projector applying a tool result only while the part is still open, in the same + transaction as the append. The owner is set activity-side from the run id and attempt, so it stays out of the workflow's deterministic input; the local driver uses a per-instance token. The projector's status guards still make any duplicate settlement a no-op. @@ -484,8 +488,8 @@ drain runs, a worker missing the session's directory rebuilds the worktree from (`session/execution/worktree.ts`): uncommitted edits and untracked files included, checked out at the same absolute path it was captured at (a uniform fleet layout). Ignored files and dependencies are not captured, so a rebuilt tree may need an install step before `bash` behaves identically. -Worker affinity or a shared volume skips the materialization latency on warm paths; the packs -are the portable baseline that works with neither. +Worker affinity (below) or a shared volume skips the materialization latency on warm paths; the +packs are the portable baseline that works with neither. Host-local state that does NOT ride the DB, so it is not reconstructed on a different host: diff --git a/packages/temporal/src/executor.ts b/packages/temporal/src/executor.ts index a1054d374760..ee72ca178620 100644 --- a/packages/temporal/src/executor.ts +++ b/packages/temporal/src/executor.ts @@ -170,7 +170,9 @@ const layer = Layer.effect( address: ADDRESS, taskQueue: TASK_QUEUE, workflow: WORKFLOW_TYPE, - ...(AFFINITY ? { worktreeAffinity: true } : {}), + ...(AFFINITY + ? { worktreeAffinity: true, pollQueue: POLL_QUEUE, worktree: SERVED_WORKTREE } + : {}), }), ) diff --git a/packages/temporal/src/l2-drain.ts b/packages/temporal/src/l2-drain.ts index f903fe64937b..702d5758026c 100644 --- a/packages/temporal/src/l2-drain.ts +++ b/packages/temporal/src/l2-drain.ts @@ -35,6 +35,7 @@ export type ModelCallDrainResult = readonly calls: ReadonlyArray readonly settlement?: StepSettlement readonly assistantMessageID?: string + readonly needsContinuation?: boolean /** The event-log token this attempt claimed. The tool and seal activities of this step must * publish under it, so it travels with the calls instead of being minted again. */ readonly owner: string @@ -58,6 +59,7 @@ export interface SealDrainInput { readonly step: number readonly settlement?: StepSettlement readonly assistantMessageID?: string + readonly needsContinuation?: boolean readonly owner: string } @@ -80,7 +82,9 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra ) => Effect.gen(function* () { const session = yield* store.get(SessionSchema.ID.make(sessionID)) - if (!session) return yield* Effect.die(`Session not found: ${sessionID}`) + // Deleting a session while its workflow is alive is not a run error. The whole-step drain + // reports it as nothing to do, and a resume waiting on this should resolve, not reject. + if (!session) return undefined if (claim) yield* events.claim(session.id, owner) // A worker taking this step on a host without the project tree rebuilds it from snapshot packs. yield* worktrees.ensure(session.location.directory) @@ -106,7 +110,12 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra }), ).pipe( Effect.map((result): ModelCallDrainResult => - result.kind === "settled" + result === undefined + ? { + kind: "settled", + result: { ran: false, continue: false, step: input.step, promotion: null }, + } + : result.kind === "settled" ? { kind: "settled", result: { @@ -122,6 +131,7 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra calls: result.calls, settlement: result.settlement, assistantMessageID: result.assistantMessageID, + needsContinuation: result.needsContinuation, owner: input.owner, }, ), @@ -138,7 +148,7 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra call: input.call, retry: input.retry === true, }), - ), + ).pipe(Effect.map((result) => result ?? { outcome: "already-settled" as const })), ) const sealDrain = async (input: SealDrainInput, signal: AbortSignal): Promise => @@ -151,14 +161,19 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra step: input.step, settlement: input.settlement, assistantMessageID: input.assistantMessageID, + needsContinuation: input.needsContinuation, }), ).pipe( - Effect.map((result) => ({ - ran: result.ran, - continue: result.continue, - step: result.step, - promotion: result.promotion ?? null, - })), + Effect.map((result) => + result === undefined + ? { ran: false, continue: false, step: input.step, promotion: null } + : { + ran: result.ran, + continue: result.continue, + step: result.step, + promotion: result.promotion ?? null, + }, + ), ), ) diff --git a/packages/temporal/src/l2-step.ts b/packages/temporal/src/l2-step.ts index 09ea16c9f9d0..24d5dad6c24f 100644 --- a/packages/temporal/src/l2-step.ts +++ b/packages/temporal/src/l2-step.ts @@ -62,6 +62,7 @@ export const makeSteppedTurn = step: model.step, settlement: model.settlement, assistantMessageID: model.assistantMessageID, + needsContinuation: model.needsContinuation, owner: model.owner, }) } diff --git a/packages/temporal/src/queue.ts b/packages/temporal/src/queue.ts index b25634f6ff63..edce9d31485b 100644 --- a/packages/temporal/src/queue.ts +++ b/packages/temporal/src/queue.ts @@ -15,24 +15,27 @@ // sandbox does not have. The queue is chosen by the client that starts the workflow and by the worker // that polls, both of which are ordinary Node. import { createHash } from "node:crypto" -import { realpathSync } from "node:fs" +import { resolve } from "node:path" /** Longer than needed to make a collision implausible, short enough to keep the queue name readable * in the Temporal UI and in `temporal task-queue describe`. */ const DIGEST_LENGTH = 12 /** - * The queue for one worktree. Resolved through realpath so two spellings of the same directory agree - * on a name: on macOS `/tmp/x` and `/private/tmp/x` are the same tree, and a client and a worker that - * disagreed about that would sit on two queues and never meet. Falls back to the given path when it - * does not exist yet, which is the case for a worker declaring a tree it has not materialized. + * The queue for one worktree. + * + * Derived from the path alone, never from the filesystem. The two sides that have to agree do not + * see the same disk: the client hashes the session's directory, the worker hashes the tree it was + * told to serve, and a worker may not have materialized that tree yet. Resolving through the + * filesystem would make the key depend on whether the path happens to exist locally, so one side + * would canonicalize and the other would not, and they would sit on different queues while nothing + * reported an error. + * + * The cost is that the operator has to spell the tree the same way on both sides. `/tmp/x` and + * `/private/tmp/x` are one tree on macOS and two keys here. Both processes log the queue they use, + * so a mismatch shows up as two names rather than as a session that never runs. */ export const queueForWorktree = (base: string, directory: string): string => { - let canonical = directory - try { - canonical = realpathSync(directory) - } catch { - // Not present yet; the literal path is still a stable key. - } + const canonical = resolve(directory).replace(/[/\\]+$/, "") return `${base}-wt-${createHash("sha256").update(canonical).digest("hex").slice(0, DIGEST_LENGTH)}` } diff --git a/packages/temporal/test/queue.test.ts b/packages/temporal/test/queue.test.ts index 1b65625ab38b..e8f753be0d7e 100644 --- a/packages/temporal/test/queue.test.ts +++ b/packages/temporal/test/queue.test.ts @@ -2,7 +2,7 @@ // derive the SAME queue name from the same tree. If they disagree they sit on two queues and the // session waits forever, which is silent: nothing errors, the work simply never runs. import { describe, it, expect } from "bun:test" -import { mkdtempSync, mkdirSync, realpathSync, rmSync, symlinkSync } from "node:fs" +import { mkdtempSync, mkdirSync, realpathSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { queueForWorktree } from "../src/queue" @@ -25,23 +25,34 @@ describe("worktree queue", () => { } }) - it("agrees on a tree reached through a symlink", () => { + it("agrees whether or not the tree exists locally", () => { const root = realpathSync(mkdtempSync(join(tmpdir(), "queue-"))) try { - const real = join(root, "real") - const link = join(root, "link") - mkdirSync(real) - symlinkSync(real, link) + const present = join(root, "present") + mkdirSync(present) + const absent = join(root, "absent") - // This is the case that actually bites: on macOS /tmp is a symlink to /private/tmp, so a - // client saying /tmp/x and a worker saying /private/tmp/x mean one tree. Keying on the raw - // string would put them on two queues and the session would hang with nothing to show for it. - expect(queueForWorktree("q", link)).toBe(queueForWorktree("q", real)) + // This is the case that matters. The client hashes the session's directory and the worker + // hashes the tree it was told to serve, and those two processes do not see the same disk. If + // the key depended on the path resolving locally, one side would canonicalize and the other + // would not, and they would sit on different queues with nothing reporting an error. + expect(queueForWorktree("q", present)).toBe(queueForWorktree("q", join(root, "present"))) + expect(queueForWorktree("q", absent)).toBe(queueForWorktree("q", join(root, "absent"))) } finally { rmSync(root, { recursive: true, force: true }) } }) + it("normalizes a path without touching the filesystem", () => { + expect(queueForWorktree("q", "/srv/trees/acme/")).toBe(queueForWorktree("q", "/srv/trees/acme")) + expect(queueForWorktree("q", "/srv/trees/./acme")).toBe( + queueForWorktree("q", "/srv/trees/acme"), + ) + // Two spellings of one tree are two keys. That is the trade for a key both sides can compute + // without a filesystem, and it is why the derived queue is logged on both sides. + expect(queueForWorktree("q", "/tmp/x")).not.toBe(queueForWorktree("q", "/private/tmp/x")) + }) + it("still names a tree that does not exist yet", () => { // A worker can declare the tree it serves before anything has materialized it, so an // unresolvable path has to stay a stable key rather than throw. From d4afb0dc4579add3413e8f15e2d0fbdca8ec67a5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 17:52:10 -0700 Subject: [PATCH 18/33] Let a user halt reach the workflow, not just the runner. The runner raised a decline as an interrupt, but the dispatcher only rethrew on cancellation. A halt crosses the boundary as an ApplicationFailure, so isCancellation is false for it, allSettled swallowed it and the step sealed anyway. The agent kept going past a refusal, and resume resolved instead of surfacing the tagged error. The durable tail's termination is now tested. It needs the layer in its own scope and the consumer forked into an outer one, or closing the test scope kills the consumer first and a hung tail looks like a passing one. Also scopes the tool point read to its session, and stops the README claiming three unit-tested behaviours were verified live. --- packages/core/src/session/runner/llm.ts | 7 ++++-- packages/core/test/event.test.ts | 27 +++++++++++++++++++- packages/temporal/README.md | 33 ++++++++++++++++++++----- packages/temporal/src/boundary.ts | 3 ++- packages/temporal/src/l2-step.ts | 11 ++++++--- packages/temporal/src/protocol.ts | 5 ++++ packages/temporal/src/workflow.ts | 14 +++++++++-- packages/temporal/test/l2-step.test.ts | 27 +++++++++++++++++--- 8 files changed, 108 insertions(+), 19 deletions(-) diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 95f53b94312c..b5e1203471b4 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -704,10 +704,13 @@ const layer = Layer.effect( // A point read, not the whole projected history. The call carries the message // that owns it, so decoding every message to find one part would make a step // cost O(session) per tool instead of O(1). + // Message ids are looked up on their own, so the session has to be checked too: a call from + // another session must not resolve here. const owner = yield* store.message(assistantMessageID) const part = - owner?.message.type === "assistant" - ? owner.message.content.find( + owner?.sessionID === input.sessionID && owner.message.type === "assistant" + ? // findLast to agree with the projector, which writes tool updates the same way. + owner.message.content.findLast( (item): item is SessionMessage.AssistantTool => item.type === "tool" && item.id === input.call.id, ) diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index ae66e91f6d30..0933e9e9a3e3 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -1,5 +1,5 @@ import { describe, expect } from "bun:test" -import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Schema, Stream } from "effect" +import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Schema, Scope, Stream } from "effect" import { EventV2 } from "@opencode-ai/core/event" import { Event } from "@opencode-ai/schema/event" import { Session } from "@opencode-ai/schema/session" @@ -1195,6 +1195,31 @@ describe("EventV2", () => { }), ) + it.live("ends a durable tail when the event layer is released under it", () => + Effect.gen(function* () { + const aggregateID = Session.ID.create() + // The layer needs its own closeable scope, and the consumer has to be forked into a scope that + // OUTLIVES it. Fork into the test's own scope and closing that scope interrupts the consumer + // before the finalizer runs, so a hung tail and a killed one look the same. + const outer = yield* Effect.scope + const layerScope = yield* Scope.make() + const ctx = yield* Layer.buildWithScope(EventV2.layerWith(), layerScope) + const events = yield* EventV2.Service.pipe(Effect.provideContext(ctx as never)) + yield* events.publish(DurableMessage, durableData(aggregateID, "seed")) + + const tail = yield* events + .durable({ aggregateID, after: -1 }) + .pipe(Stream.runDrain, Effect.forkIn(outer)) + yield* Effect.sleep("150 millis") + yield* Scope.close(layerScope, Exit.void) + + // The tick never ends on its own, so it must not become what holds the tail open: a subscriber + // would outlive the layer and keep reading a database being torn down. + const settled = yield* Fiber.await(tail).pipe(Effect.timeout("5 seconds"), Effect.exit) + expect(Exit.isSuccess(settled)).toBe(true) + }), + ) + it.effect("never fences a publish made outside a drain", () => Effect.gen(function* () { const events = yield* EventV2.Service diff --git a/packages/temporal/README.md b/packages/temporal/README.md index c1255ef99cd6..dcf1e150aa46 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -59,9 +59,10 @@ exposes a substitutable `SessionExecution` service (`active` / `resume` / `wake` whose local impl comments "Future remote placement belongs here." This change provides a Temporal-backed `SessionExecution` in `packages/core/src/session/execution/`: -- `temporal-workflow.ts`: the pure per-session workflow (the Temporal equivalent of +- `packages/temporal/src/workflow.ts`: the pure per-session workflow (the Temporal equivalent of `SessionRunCoordinator`: `wake`/`force` drive one drain, wakes coalesce, quiescent runs end). -- `temporal-activities.ts`: the `runTurnStep` activity (heartbeats; forwards cancellation; +- `packages/temporal/src/activities.ts`: the `runTurnStep` activity (heartbeats; forwards + cancellation; injects the attempt's event-log owner token). - `temporal.ts`: the `SessionExecution` layer + node: `wake` → `signalWithStart`, `resume` → forced `signalWithStart`, `interrupt` → cancel signal; each drain runs one step of the local @@ -232,12 +233,16 @@ Verified live (dev server, `gpt-5-mini`, stepped mode on): answered with the file's contents. Worker affinity (below) skips the materialization on warm paths; the packs are the baseline that works without it. +Covered by tests rather than by a live run: + - **A compaction restart keeps deferring.** The `deferTools` flag is threaded through both `ContinueAfterCompaction` recursions; a first version dropped it, which would have silently run tools inline on a compacting step. - **A provider error ends the turn.** The attempt's own continuation decision is carried to the seal, because the log cannot tell a failed attempt from one that wants another step. -- **A declined permission halts the turn** rather than reaching the model as one failed tool. +- **A declined permission halts the turn** rather than reaching the model as one failed tool. Both + halves matter: the runner raises it as an interrupt, and the workflow tells that failure apart + from a tool that merely failed. Not covered, stated plainly: @@ -290,6 +295,19 @@ with nothing to show for it. whose tree has no worker polling does not fall back to another worker. It waits. Reconstruction is what makes any worker able to serve any session, and turning affinity on is choosing not to use it. +Two consequences to plan for, both silent: + +- **A worker serves one tree.** In the default `role=both` deployment the embedded worker polls the + queue for the process directory, so every session in another directory has no poller. If you open + more than one project against one serve process, do not turn this on. +- **Flipping the flag strands workflows already running.** A workflow keeps the task queue it + started on for life, and its activities inherit it. Restarting workers with the flag changed + leaves in-flight sessions with nobody polling their queue; they retry rather than fail. Drain + before flipping, in either direction. + +Both processes log the queue they use (`pollQueue` on a worker, `worktree` on the client), so a +mismatch shows up as two names in the logs rather than as a session that never runs. + Verified live, all three halves: - a worker serving the session's tree ran the turn, and the workflow sat on the derived queue @@ -298,7 +316,7 @@ Verified live, all three halves: - bringing the right worker back drained it and the answer arrived, so the work waits rather than being lost -#### Live streaming across a process boundary +#### Durable events across a process boundary Worth stating separately, because it is **not** specific to stepped mode: it is a property of running a standalone worker at all, and the mechanism is in `event.ts`. @@ -309,8 +327,11 @@ process's map, so a commit in another process cannot wake a tail. Split into complete, but a client subscribed *live* saw exactly one event, `prompt.admitted`, the only one the serve process writes itself. A UI attached to serve saw a prompt admitted and then silence. -The durable tail now also re-reads on a tick (`LayerOptions.livePollInterval`, default one second, -0 to disable). In-process commits still wake it instantly, so latency is unchanged where it already +Token deltas are not part of this. `Text.Delta`, `Reasoning.Delta` and `Tool.Input.Delta` are +live-only and never reach the durable log, so what crosses a process boundary is block-level: +`step.started`, `tool.called`, `tool.success`, `step.ended`. The durable tail now also re-reads on a +tick (`LayerOptions.livePollInterval`, default one second, 0 to disable). In-process commits +still wake it instantly, so latency is unchanged where it already worked; the tick only catches what the wake cannot see. An idle subscriber costs one indexed read per period, and a tick with nothing new emits nothing. The same split now delivers the worker's `step.started`, `tool.called`, `tool.success` and `step.ended` to a live subscriber. diff --git a/packages/temporal/src/boundary.ts b/packages/temporal/src/boundary.ts index 30e272592166..8aed9f35de29 100644 --- a/packages/temporal/src/boundary.ts +++ b/packages/temporal/src/boundary.ts @@ -13,6 +13,7 @@ import { ApplicationFailure } from "@temporalio/activity" import { SessionSchema } from "@opencode-ai/core/session/schema" import { SessionRunDeclinedError } from "@opencode-ai/core/session/error" import { encodeRunError } from "@opencode-ai/core/session/execution/run-error-codec" +import { HALTED_FAILURE_TYPE } from "./protocol" export const runAtBoundary = async ( sessionID: string, @@ -28,7 +29,7 @@ export const runAtBoundary = async ( const declined = encodeRunError(new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make(sessionID) })) throw ApplicationFailure.create({ message: "session run halted (user declined)", - type: "SessionRunDeclined", + type: HALTED_FAILURE_TYPE, nonRetryable: true, details: declined === undefined ? undefined : [declined], }) diff --git a/packages/temporal/src/l2-step.ts b/packages/temporal/src/l2-step.ts index 24d5dad6c24f..c7bd3e23e7fc 100644 --- a/packages/temporal/src/l2-step.ts +++ b/packages/temporal/src/l2-step.ts @@ -31,6 +31,10 @@ export interface SteppedTurnDeps { /** Whether an error is the driver's cancellation. An interrupt has to end the turn, so it must not * be swallowed the way a failed tool is. */ readonly isCancellation: (error: unknown) => boolean + /** Whether an error is the user stopping the turn, like a declined permission. It arrives as an + * ordinary activity failure, so without this it reads as one bad tool and the turn carries on + * past the refusal. */ + readonly isHalt: (error: unknown) => boolean } /** @@ -38,7 +42,7 @@ export interface SteppedTurnDeps { * of a whole-step activity. */ export const makeSteppedTurn = - ({ activities, isCancellation }: SteppedTurnDeps) => + ({ activities, isCancellation, isHalt }: SteppedTurnDeps) => async (input: StepDrainInput): Promise => { const model = await activities.runModelCall(input) // A crashed step finalized from the log, or the recovery gate finding no work: the step is over @@ -47,14 +51,15 @@ export const makeSteppedTurn = // Each call is its own unit of work. A tool that fails outright does not take the turn with it: // the seal closes its call as an error and the model gets to react, which is better than losing - // the step. An interrupt is different and has to propagate. + // the step. A cancel and a user halt are different, and both have to propagate. const dispatched = await Promise.allSettled( model.calls.map((call) => activities.runToolCall({ sessionID: input.sessionID, call, owner: model.owner }), ), ) for (const outcome of dispatched) { - if (outcome.status === "rejected" && isCancellation(outcome.reason)) throw outcome.reason + if (outcome.status !== "rejected") continue + if (isCancellation(outcome.reason) || isHalt(outcome.reason)) throw outcome.reason } return activities.sealStep({ diff --git a/packages/temporal/src/protocol.ts b/packages/temporal/src/protocol.ts index 3f7807fef94f..b1ed3af1560e 100644 --- a/packages/temporal/src/protocol.ts +++ b/packages/temporal/src/protocol.ts @@ -8,6 +8,11 @@ export const WORKFLOW_TYPE = "sessionTurn" export const SIGNALS = { wake: "wake", interrupt: "interrupt" } as const export const RESUME_UPDATE = "resume" +/** The failure type a drain raises when the user stopped the turn (a declined permission, a + * rejected question). The workflow has to tell it apart from a failed tool, so the name is shared + * rather than spelled twice. */ +export const HALTED_FAILURE_TYPE = "SessionRunDeclined" + export const WORKFLOW_ID_PREFIX = "session-exec-" export const workflowId = (sessionID: string) => `${WORKFLOW_ID_PREFIX}${sessionID}` diff --git a/packages/temporal/src/workflow.ts b/packages/temporal/src/workflow.ts index da7d586d2f9b..f98c902e7842 100644 --- a/packages/temporal/src/workflow.ts +++ b/packages/temporal/src/workflow.ts @@ -18,10 +18,12 @@ import { CancellationScope, isCancellation, allHandlersFinished, + ActivityFailure, + ApplicationFailure, } from "@temporalio/workflow" import type { StepActivities, SteppedTurnActivities } from "./activities" import { makeSteppedTurn } from "./l2-step" -import { SIGNALS, RESUME_UPDATE } from "./protocol" +import { HALTED_FAILURE_TYPE, SIGNALS, RESUME_UPDATE } from "./protocol" import { makeSupervisor, type SupervisorRuntime } from "./supervisor" const activityOptions = { @@ -111,7 +113,15 @@ const runtime: SupervisorRuntime = { // unchanged, and only what "one step" means differs. const steppedRuntime: SupervisorRuntime = { ...runtime, - runTurnStep: makeSteppedTurn({ activities: { runModelCall, runToolCall, sealStep }, isCancellation }), + runTurnStep: makeSteppedTurn({ + activities: { runModelCall, runToolCall, sealStep }, + isCancellation, + // A halt crosses the activity boundary as an ApplicationFailure, so isCancellation is false for + // it and the dispatcher would treat the user's refusal as one failed tool. + isHalt: (error) => + error instanceof ActivityFailure && + (error.cause as ApplicationFailure | undefined)?.type === HALTED_FAILURE_TYPE, + }), } // The scope of the drain currently running, so an interrupt signal can cancel exactly that turn. diff --git a/packages/temporal/test/l2-step.test.ts b/packages/temporal/test/l2-step.test.ts index 480166de8835..fb051bd331eb 100644 --- a/packages/temporal/test/l2-step.test.ts +++ b/packages/temporal/test/l2-step.test.ts @@ -8,7 +8,9 @@ import type { StepDrainInput, StepDrainResult } from "../src/activities" import type { ModelCallDrainResult, SealDrainInput, ToolCallDrainInput } from "../src/l2-drain" class FakeCancel extends Error {} +class FakeHalt extends Error {} const isCancellation = (e: unknown) => e instanceof FakeCancel +const isHalt = (e: unknown) => e instanceof FakeHalt const INPUT: StepDrainInput = { sessionID: "ses_1", step: 2, promotion: null, first: false, force: false } const SEALED: StepDrainResult = { ran: true, continue: true, step: 3, promotion: "steer" } @@ -39,7 +41,7 @@ describe("stepped turn", () => { const settled: StepDrainResult = { ran: false, continue: false, step: 2, promotion: null } const { activities, tools, seals } = fakes({ kind: "settled", result: settled }) - const result = await makeSteppedTurn({ activities, isCancellation })(INPUT) + const result = await makeSteppedTurn({ activities, isCancellation, isHalt })(INPUT) // A crashed step finalized from the log, or the recovery gate finding no work: there is nothing // to run and nothing to close. @@ -57,7 +59,7 @@ describe("stepped turn", () => { owner: "run-1:model-1:1", }) - const result = await makeSteppedTurn({ activities, isCancellation })(INPUT) + const result = await makeSteppedTurn({ activities, isCancellation, isHalt })(INPUT) expect(tools.map((t) => t.call.id)).toEqual(["call_a", "call_b"]) // Every writer in the step publishes under the token the attempt claimed. A token minted per @@ -79,7 +81,7 @@ describe("stepped turn", () => { }, ) - const result = await makeSteppedTurn({ activities, isCancellation })(INPUT) + const result = await makeSteppedTurn({ activities, isCancellation, isHalt })(INPUT) // One broken tool must not take the turn with it: the seal closes that call as an error and the // model gets to react, which beats losing the step. @@ -96,10 +98,27 @@ describe("stepped turn", () => { }, ) - const run = makeSteppedTurn({ activities, isCancellation })(INPUT) + const run = makeSteppedTurn({ activities, isCancellation, isHalt })(INPUT) // A cancellation is not a failed tool. Swallowing it would close a step the user stopped. await expect(run).rejects.toBeInstanceOf(FakeCancel) expect(seals).toHaveLength(0) }) + + it("lets a user halt end the turn instead of sealing it", async () => { + const { activities, seals } = fakes( + { kind: "called", step: 2, calls: [call("call_a")], owner: "own" }, + async () => { + throw new FakeHalt("declined") + }, + ) + + const run = makeSteppedTurn({ activities, isCancellation, isHalt })(INPUT) + + // A decline crosses the activity boundary as an ordinary failure, not a cancel, so without a + // separate test for it the dispatcher would seal the step and the turn would carry on past the + // user's refusal. + await expect(run).rejects.toBeInstanceOf(FakeHalt) + expect(seals).toHaveLength(0) + }) }) From 9d2e2ada1ac6f00d8620fa6eb064ca54290d6af4 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 18:01:37 -0700 Subject: [PATCH 19/33] Pinned the halt predicate against what the boundary throws. The bug was a mismatch between the failure `boundary.ts` raises and what the dispatcher recognises, so a test that injects its own predicate can't catch it coming back. The positive case now runs the real boundary and feeds the result to the real predicate. Change either side and it fails. --- packages/temporal/README.md | 4 ++- packages/temporal/src/l2-step.ts | 14 ++++++++++ packages/temporal/src/workflow.ts | 12 +++------ packages/temporal/test/l2-step.test.ts | 36 +++++++++++++++++++++++++- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index dcf1e150aa46..88c95fd2a462 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -64,7 +64,8 @@ Temporal-backed `SessionExecution` in `packages/core/src/session/execution/`: - `packages/temporal/src/activities.ts`: the `runTurnStep` activity (heartbeats; forwards cancellation; injects the attempt's event-log owner token). -- `temporal.ts`: the `SessionExecution` layer + node: `wake` → `signalWithStart`, `resume` → +- `packages/temporal/src/executor.ts`: the `SessionExecution` layer + node: `wake` → + `signalWithStart`, `resume` → forced `signalWithStart`, `interrupt` → cancel signal; each drain runs one step of the local coordinator's loop (`SessionRunner.runStep`) in an activity against the durable event log. The Temporal client and an embedded worker are co-hosted in the server process (both run under bun). @@ -233,6 +234,7 @@ Verified live (dev server, `gpt-5-mini`, stepped mode on): answered with the file's contents. Worker affinity (below) skips the materialization on warm paths; the packs are the baseline that works without it. + Covered by tests rather than by a live run: - **A compaction restart keeps deferring.** The `deferTools` flag is threaded through both diff --git a/packages/temporal/src/l2-step.ts b/packages/temporal/src/l2-step.ts index c7bd3e23e7fc..efd12a86d2bc 100644 --- a/packages/temporal/src/l2-step.ts +++ b/packages/temporal/src/l2-step.ts @@ -10,6 +10,8 @@ // because a workflow cannot consume a stream. The tools of one step still run concurrently with each // other; what is lost is the overlap between the model and its own tools. +import { ActivityFailure, type ApplicationFailure } from "@temporalio/workflow" +import { HALTED_FAILURE_TYPE } from "./protocol" import type { StepDrainInput, StepDrainResult } from "./drain" import type { ModelCallDrainInput, @@ -19,6 +21,18 @@ import type { ToolCallDrainResult, } from "./l2-drain" +/** + * A halt the user asked for, as it looks once it has crossed an activity boundary. The runner raises + * it as an interrupt, `boundary.ts` throws it as an `ApplicationFailure`, and the SDK wraps that in + * one `ActivityFailure`. It is not a cancellation, so `isCancellation` says no, and a dispatcher + * that checks only that would treat a refusal as one failed tool and carry on. + * + * A `TimeoutFailure` or a `CancelledFailure` cause has no `type` field, so neither matches. + */ +export const isHaltFailure = (error: unknown) => + error instanceof ActivityFailure && + (error.cause as ApplicationFailure | undefined)?.type === HALTED_FAILURE_TYPE + /** The three activities a stepped turn drives. */ export interface SteppedActivities { readonly runModelCall: (input: ModelCallDrainInput) => Promise diff --git a/packages/temporal/src/workflow.ts b/packages/temporal/src/workflow.ts index f98c902e7842..be0664e8d2d6 100644 --- a/packages/temporal/src/workflow.ts +++ b/packages/temporal/src/workflow.ts @@ -18,12 +18,10 @@ import { CancellationScope, isCancellation, allHandlersFinished, - ActivityFailure, - ApplicationFailure, } from "@temporalio/workflow" import type { StepActivities, SteppedTurnActivities } from "./activities" -import { makeSteppedTurn } from "./l2-step" -import { HALTED_FAILURE_TYPE, SIGNALS, RESUME_UPDATE } from "./protocol" +import { isHaltFailure, makeSteppedTurn } from "./l2-step" +import { SIGNALS, RESUME_UPDATE } from "./protocol" import { makeSupervisor, type SupervisorRuntime } from "./supervisor" const activityOptions = { @@ -116,11 +114,7 @@ const steppedRuntime: SupervisorRuntime = { runTurnStep: makeSteppedTurn({ activities: { runModelCall, runToolCall, sealStep }, isCancellation, - // A halt crosses the activity boundary as an ApplicationFailure, so isCancellation is false for - // it and the dispatcher would treat the user's refusal as one failed tool. - isHalt: (error) => - error instanceof ActivityFailure && - (error.cause as ApplicationFailure | undefined)?.type === HALTED_FAILURE_TYPE, + isHalt: isHaltFailure, }), } diff --git a/packages/temporal/test/l2-step.test.ts b/packages/temporal/test/l2-step.test.ts index fb051bd331eb..343f9f032813 100644 --- a/packages/temporal/test/l2-step.test.ts +++ b/packages/temporal/test/l2-step.test.ts @@ -3,7 +3,10 @@ // pinned here is the orchestration: the owner token reaches every writer, a settled step dispatches // nothing, a failed tool still lets the step close, and an interrupt is not swallowed. import { describe, it, expect } from "bun:test" -import { makeSteppedTurn, type SteppedActivities } from "../src/l2-step" +import { isHaltFailure, makeSteppedTurn, type SteppedActivities } from "../src/l2-step" +import { runAtBoundary } from "../src/boundary" +import { Effect } from "effect" +import { ActivityFailure, ApplicationFailure, CancelledFailure, TimeoutFailure } from "@temporalio/workflow" import type { StepDrainInput, StepDrainResult } from "../src/activities" import type { ModelCallDrainResult, SealDrainInput, ToolCallDrainInput } from "../src/l2-drain" @@ -122,3 +125,34 @@ describe("stepped turn", () => { expect(seals).toHaveLength(0) }) }) + +// The bug this predicate exists for was a mismatch between what `boundary.ts` throws and what the +// dispatcher recognises. Injecting a fake predicate cannot catch that, so match against the real +// failure shapes. The negative cases are the point: a predicate that answered true for everything +// would pass the positive one alone. +describe("halt predicate", () => { + const wrap = (cause?: Error) => + new ActivityFailure("activity failed", "runToolCall", "1", 1 as never, undefined, cause) + + it("recognises what the activity boundary throws for a user halt", async () => { + // Built by running the boundary rather than by hand, so the two sides cannot drift: an interrupt + // with no abort is how a decline leaves the runner, and whatever that produces is what a + // dispatcher has to recognise. + const thrown = await runAtBoundary("ses_1", new AbortController().signal, Effect.interrupt).then( + () => undefined, + (error: unknown) => error, + ) + expect(thrown).toBeInstanceOf(ApplicationFailure) + expect(isHaltFailure(wrap(thrown as Error))).toBe(true) + }) + + it("says no to everything else that can come back from an activity", () => { + expect(isHaltFailure(wrap(new CancelledFailure("cancelled")))).toBe(false) + expect(isHaltFailure(wrap(new TimeoutFailure("timed out", undefined, 1 as never)))).toBe(false) + expect(isHaltFailure(wrap(ApplicationFailure.create({ type: "SessionRunError" })))).toBe(false) + expect(isHaltFailure(wrap())).toBe(false) + // Unwrapped, so not what a dispatcher ever sees. + expect(isHaltFailure(ApplicationFailure.create({ type: "SessionRunDeclined" }))).toBe(false) + expect(isHaltFailure(new Error("plain"))).toBe(false) + }) +}) From 62b8966262827eb27712c4593fc3ccf13be0f345 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 18:06:16 -0700 Subject: [PATCH 20/33] Wrapped the branch's lines to 100 columns. Comments and prose reflowed, code wrapped by hand in the shared files so the review fixes stay readable in the diff. Running prettier at 100 over `llm.ts` would have moved 231 lines that have nothing to do with this work. Seven long lines are left, all string literals where a break would hurt more than help. --- packages/core/src/event.ts | 14 +- packages/core/src/session/runner/index.ts | 11 +- packages/core/src/session/runner/llm.ts | 123 +++++++++++++----- packages/core/test/event.test.ts | 14 +- .../test/session-runner-model-call.test.ts | 114 ++++++++++------ packages/core/test/step-overlap-bench.test.ts | 68 +++++++--- packages/temporal/README.md | 48 ++++--- .../temporal/scripts/stream-tail-probe.ts | 20 ++- packages/temporal/src/activities.ts | 22 +++- packages/temporal/src/boundary.ts | 13 +- packages/temporal/src/drain.ts | 3 +- packages/temporal/src/executor.ts | 25 +++- packages/temporal/src/l2-drain.ts | 64 +++++---- packages/temporal/src/l2-step.ts | 16 ++- packages/temporal/src/queue.ts | 12 +- packages/temporal/src/workflow.ts | 9 +- packages/temporal/test/l2-step.test.ts | 40 +++++- ...ession-execution-temporal-contract.test.ts | 6 +- 18 files changed, 427 insertions(+), 195 deletions(-) diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index c458cc51ba4d..adac062fc013 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -1,6 +1,7 @@ export * as EventV2 from "./event" -import { Cause, Context, Duration, Effect, Layer, Option, PubSub, Queue, Schema, Stream } from "effect" +import { Cause, Context, Duration, Effect, Layer, Option, PubSub, Queue, Schema } from "effect" +import { Stream } from "effect" import { Event } from "@opencode-ai/schema/event" import type { Data, Definition, Payload } from "@opencode-ai/schema/event" import { and, asc, eq, gt, inArray } from "drizzle-orm" @@ -175,8 +176,9 @@ export const allBounded = (events: Interface, capacity: number) => export interface LayerOptions { readonly beforeAggregateRead?: (aggregateID: string) => Effect.Effect /** How often a durable tail re-reads on its own, on top of the in-process wake. The wake only - * fires for commits made in THIS process, so without a tick a subscriber cannot see events another - * process appended: a standalone worker's whole turn is invisible to a tail on the HTTP server. + * fires for commits made in THIS process, so without a tick a subscriber cannot see events + * another process appended: a standalone worker's whole turn is invisible to a tail on the + * HTTP server. * Set to 0 to disable and rely on the wake alone. */ readonly livePollInterval?: Duration.Input } @@ -196,7 +198,8 @@ export const layerWith = (options?: LayerOptions) => typed: new Map>(), } const projectors = new Map() - // TODO: Bind durable projectors to exact type+version before supporting incompatible historical payloads. + // TODO: Bind durable projectors to exact type+version before supporting incompatible + // historical payloads. const listeners = new Array() const { db } = yield* Database.Service @@ -630,7 +633,8 @@ export const layerWith = (options?: LayerOptions) => ) const historical = yield* read // Wake on either an in-process commit or the tick. A tick that finds nothing new reads - // zero rows and emits nothing, so an idle subscriber costs one indexed query per period. + // zero rows and emits nothing, so an idle subscriber costs one indexed query per + // period. const pollInterval = options?.livePollInterval ?? DEFAULT_LIVE_POLL const woken = Stream.fromSubscription(wakes) // haltStrategy "left" keeps the wake stream as what ends the tail. The tick never ends, diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index aa367bb91469..c0e821ec759a 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -72,7 +72,7 @@ export interface ToolCallInput { /** How a dispatched call ended. * - `settled`: the tool ran and its result is durable. - * - `already-settled`: the log already had a result, so nothing ran. This is the at-least-once case. + * - `already-settled`: the log already had a result, so nothing ran. The at-least-once case. * - `unknown`: a retry of a tool that does not declare itself repeatable. Reported to the model as * an unknown outcome rather than run a second time. */ export type ToolCallOutcome = "settled" | "already-settled" | "unknown" @@ -117,15 +117,16 @@ export interface Interface { /** Run exactly one step and report the next loop state, so a caller (e.g. a Temporal workflow) * can drive the turn one step at a time. Mirrors one iteration of `run`'s loop. */ readonly runStep: (input: StepInput) => Effect.Effect - /** Run the provider attempt of one step and stop, handing back the tool calls it asked for instead - * of running them. The caller dispatches each one and then seals the step. This is what puts the + /** Run the provider attempt of one step and stop, handing back the tool calls it asked for + * instead of running them. The caller dispatches each one and then seals the step. This is what + * puts the * model-to-tools loop in a durable executor's hands rather than inside a single activity. */ readonly runModelCall: (input: StepInput) => Effect.Effect /** Run one recorded tool call and publish its result. Safe to call twice for the same call: the * second sees the settled result and does nothing. */ readonly runToolCall: (input: ToolCallInput) => Effect.Effect - /** Close a step once its calls have been dispatched: snapshot, file diff, Step.Ended, and the loop - * decision. Safe to call twice: the second sees the step already closed and returns the same + /** Close a step once its calls have been dispatched: snapshot, file diff, Step.Ended, and the + * loop decision. Safe to call twice: the second sees the step already closed and returns the same * answer without publishing again. */ readonly sealStep: (input: SealStepInput) => Effect.Effect } diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index b5e1203471b4..b9c4645a8b1d 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -60,7 +60,8 @@ import { llmClient } from "../../effect/app-node-platform" * `SessionPrompt` monolith. Implement the unchecked items in small reviewed slices: * * - Session ownership and controls - * - [x] Coordinate one local active drain per Session; explicit resumes join and prompt wakeups coalesce. + * - [x] Coordinate one local active drain per Session; explicit resumes join and prompt wakeups + * coalesce. * - [ ] Replace local ownership with durable multi-node ownership when clustered. * - [ ] Mark busy, retrying, idle, interrupted, or terminal-failure status durably. * - [ ] Honor interruption and reject stale work after runtime attachment replacement. @@ -86,7 +87,8 @@ import { llmClient } from "../../effect/app-node-platform" * - [x] Start each recorded local call eagerly and await all settlements before continuation. * - [ ] Add scoped runtime context, progress updates, attachment normalization, * plugins, and cancellation settlement. - * - [x] Reload projected history and start the next explicit provider turn after local tool results. + * - [x] Reload projected history and start the next explicit provider turn after local tool + * results. * - [x] Continue for durable user steering accepted during an active provider turn. * - [ ] Continue for compaction or another continuation condition when required. * @@ -98,9 +100,11 @@ import { llmClient } from "../../effect/app-node-platform" * Use `llm.stream(request)` for each provider turn. Keep tool execution and continuation here. * Durable continuation recovery remains a separate future slice with an explicit retry policy. * - * The current slice loads V2 history, translates it, resolves a model through a core service, and persists one + * The current slice loads V2 history, translates it, resolves a model through a core service, and + * persists one * provider turn. Registry definitions are advertised, local tool calls are settled durably, and an - * explicit loop starts the next provider turn after local settlement. Configured agent step limits bound the loop. + * explicit loop starts the next provider turn after local settlement. Configured agent step limits + * bound the loop. */ const layer = Layer.effect( @@ -177,7 +181,8 @@ const layer = Layer.effect( return false }) - // Match V1: declining a user prompt halts the loop instead of becoming model-facing tool output. + // Match V1: declining a user prompt halts the loop instead of becoming model-facing tool + // output. const isUserDeclined = (cause: Cause.Cause) => cause.reasons.some( (reason) => @@ -211,8 +216,10 @@ const layer = Layer.effect( promotion: SessionInput.Delivery | undefined, step: number, recoverOverflow?: typeof compaction.compactAfterOverflow, - // When set, tool calls are recorded and returned instead of run, and the step is left open for - // whoever runs them. This is what lets a durable executor make each call its own unit of work. + // When set, tool calls are recorded and returned instead of run, and the step is left open + // for + // whoever runs them. This is what lets a durable executor make each call its own unit of + // work. deferTools = false, ) { const session = yield* getSession(sessionID) @@ -297,11 +304,18 @@ const layer = Layer.effect( } needsContinuation = true const assistantMessageID = yield* publisher.assistantMessageID(event.id) - // Tool.Called is already durable (the publish above), so handing the call back is enough - // for the caller to run it later. Nothing forks here, which is why the step ends when the + // Tool.Called is already durable (the publish above), so handing the call back is + // enough + // for the caller to run it later. Nothing forks here, which is why the step ends when + // the // stream does and the overlap between the model and its tools is lost. if (deferTools) { - deferred.push({ id: event.id, name: event.name, input: event.input, assistantMessageID }) + deferred.push({ + id: event.id, + name: event.name, + input: event.input, + assistantMessageID, + }) return } yield* Effect.uninterruptibleMask((restore) => @@ -432,7 +446,12 @@ const layer = Layer.effect( deferTools?: boolean, ) => Effect.Effect - const runAfterOverflowCompaction: RunTurn = Effect.fnUntraced(function* (sessionID, promotion, step, deferTools) { + const runAfterOverflowCompaction: RunTurn = Effect.fnUntraced(function* ( + sessionID, + promotion, + step, + deferTools, + ) { return yield* runTurnAttempt(sessionID, promotion, step, undefined, deferTools).pipe( Effect.catchDefect( Effect.fnUntraced(function* (defect) { @@ -440,20 +459,36 @@ const layer = Layer.effect( if (defect.transition._tag === "ContinueAfterOverflowCompaction") return yield* Effect.die("Post-compaction provider attempt cannot recover another overflow") yield* Effect.yieldNow - return yield* runAfterOverflowCompaction(sessionID, undefined, defect.transition.step, deferTools) + return yield* runAfterOverflowCompaction( + sessionID, + undefined, + defect.transition.step, + deferTools, + ) }), ), ) }) const runTurn: RunTurn = Effect.fnUntraced(function* (sessionID, promotion, step, deferTools) { - return yield* runTurnAttempt(sessionID, promotion, step, compaction.compactAfterOverflow, deferTools).pipe( + return yield* runTurnAttempt( + sessionID, + promotion, + step, + compaction.compactAfterOverflow, + deferTools, + ).pipe( Effect.catchDefect( Effect.fnUntraced(function* (defect) { if (!(defect instanceof TurnTransitionError)) return yield* Effect.die(defect) yield* Effect.yieldNow if (defect.transition._tag === "ContinueAfterOverflowCompaction") - return yield* runAfterOverflowCompaction(sessionID, undefined, defect.transition.step, deferTools) + return yield* runAfterOverflowCompaction( + sessionID, + undefined, + defect.transition.step, + deferTools, + ) return yield* runTurn(sessionID, undefined, defect.transition.step, deferTools) }), ), @@ -490,12 +525,15 @@ const layer = Layer.effect( }) // Resume a crashed step from the durable log instead of re-streaming it. A Temporal step retry - // re-invokes runStep on the same log; if the in-flight step already DISPATCHED tools (Tool.Called - // is recorded before the side effect runs, so a running/completed tool may have run), re-streaming + // re-invokes runStep on the same log; if the in-flight step already DISPATCHED tools + // (Tool.Called + // is recorded before the side effect runs, so a running/completed tool may have run), + // re-streaming // would re-run that side effect and duplicate the assistant message. Instead we close the step // from the log: keep completed tool results, fail the ones still unsettled (their result never // committed -- we can't know if they ran, so the model redoes them), and publish a synthesized - // Step.Ended. The model is NOT re-called. Returns undefined when there is nothing to finalize (a + // Step.Ended. The model is NOT re-called. Returns undefined when there is nothing to finalize + // (a // fresh step, or a partial with no dispatched tools, which is safe to re-stream). Token/cost // metering is 0 for the resumed step only; faithful metering would need a durable step-sealed // marker carrying the provider usage. @@ -510,7 +548,8 @@ const layer = Layer.effect( // At most one assistant is in flight (the projector supersedes older ones); it only exists at // step entry on a re-drive, never on a fresh step. const inFlight = context.findLast( - (message): message is SessionMessage.Assistant => message.type === "assistant" && !message.time.completed, + (message): message is SessionMessage.Assistant => + message.type === "assistant" && !message.time.completed, ) if (!inFlight) return undefined const toolParts = inFlight.content.filter( @@ -613,12 +652,14 @@ const layer = Layer.effect( // Close tools left pending/running by an interrupted attempt before every turn, not just the // first. A mid-turn re-drive (first=false, from a Temporal step retry) would otherwise // re-stream a request with a dangling tool_use and no tool_result, which the provider rejects - // -- a retry poison loop. This is a no-op on a healthy step (the prior step settled its tools). + // -- a retry poison loop. This is a no-op on a healthy step (the prior step settled its + // tools). yield* failInterruptedTools(input.sessionID, entryContext) return { kind: "run", promotion } as const }) - // What the loop does once a step's provider attempt and its tools are done: a pending steer wins + // What the loop does once a step's provider attempt and its tools are done: a pending steer + // wins // over a queued prompt, and either one keeps the turn going. Shared for the same reason as the // prologue, and read after the tools have run so work admitted during the step is seen. const stepContinuation = Effect.fn("SessionRunner.stepContinuation")(function* ( @@ -627,9 +668,15 @@ const layer = Layer.effect( step: number, ) { let needsContinuation = hadToolCalls - if (!needsContinuation) needsContinuation = yield* SessionInput.hasPending(db, sessionID, "steer") + if (!needsContinuation) + needsContinuation = yield* SessionInput.hasPending(db, sessionID, "steer") if (needsContinuation) - return { ran: true, continue: true, step: step + 1, promotion: "steer" as SessionInput.Delivery } + return { + ran: true, + continue: true, + step: step + 1, + promotion: "steer" as SessionInput.Delivery, + } const moreQueue = yield* SessionInput.hasPending(db, sessionID, "queue") if (moreQueue) return { ran: true, continue: true, step: 1, promotion: "queue" as SessionInput.Delivery } return { ran: true, continue: false, step: step + 1, promotion: undefined } @@ -638,17 +685,24 @@ const layer = Layer.effect( const sealStep = Effect.fn("SessionRunner.sealStep")(function* (input: SealStepInput) { const context = yield* getContext(input.sessionID) const inFlight = context.findLast( - (message): message is SessionMessage.Assistant => message.type === "assistant" && !message.time.completed, + (message): message is SessionMessage.Assistant => + message.type === "assistant" && !message.time.completed, ) // On a retry that lands after Step.Ended was published there is nothing open, but the loop // decision still has to come out the same, so it is read off the step we just closed. const carried = input.assistantMessageID const target = - (carried ? context.findLast((m): m is SessionMessage.Assistant => m.id === carried) : undefined) ?? + (carried + ? context.findLast((m): m is SessionMessage.Assistant => m.id === carried) + : undefined) ?? inFlight ?? - context.findLast((message): message is SessionMessage.Assistant => message.type === "assistant") + context.findLast( + (message): message is SessionMessage.Assistant => message.type === "assistant", + ) if (!target) return yield* stepContinuation(input.sessionID, false, input.step) - const toolParts = target.content.filter((part): part is SessionMessage.AssistantTool => part.type === "tool") + const toolParts = target.content.filter( + (part): part is SessionMessage.AssistantTool => part.type === "tool", + ) // A step continues so the model can see its tool results. Provider-executed calls need no // follow-up turn, so a step holding only those finalizes as a plain stop. const localTools = toolParts.some((part) => part.provider?.executed !== true) @@ -679,7 +733,12 @@ const layer = Layer.effect( // crash-resume path uses, which costs only the metering on that step. finish: input.settlement?.finish ?? (localTools ? "tool-calls" : "stop"), cost: 0, - tokens: input.settlement?.tokens ?? { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + tokens: input.settlement?.tokens ?? { + input: 0, + output: 0, + reasoning: 0, + cache: { read: 0, write: 0 }, + }, snapshot: endSnapshot, files, }) @@ -693,7 +752,8 @@ const layer = Layer.effect( const toolPartOf = (messages: ReadonlyArray, callID: string) => { for (const message of messages) { if (message.type !== "assistant") continue - for (const part of message.content) if (part.type === "tool" && part.id === callID) return part + for (const part of message.content) + if (part.type === "tool" && part.id === callID) return part } return undefined } @@ -719,7 +779,9 @@ const layer = Layer.effect( // before handing it over. Missing means the log moved under us (a fence), and running a tool // whose call is not recorded would leave an orphan result. if (!part || part.type !== "tool") - return yield* Effect.die(`Tool call ${input.call.id} is not recorded on session ${input.sessionID}`) + return yield* Effect.die( + `Tool call ${input.call.id} is not recorded on session ${input.sessionID}`, + ) // At-least-once: a duplicate dispatch landing after the result did must not run anything. if (part.state.status !== "pending" && part.state.status !== "running") return { outcome: "already-settled" } as ToolCallResult @@ -782,7 +844,8 @@ const layer = Layer.effect( const runModelCall = Effect.fn("SessionRunner.runModelCall")(function* (input: StepInput) { const prologue = yield* stepPrologue(input) - if (prologue.kind === "settled") return { kind: "settled", result: prologue.result } as ModelCallResult + if (prologue.kind === "settled") + return { kind: "settled", result: prologue.result } as ModelCallResult const result = yield* runTurn(input.sessionID, prologue.promotion, input.step, true) return { kind: "called", diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index 0933e9e9a3e3..ffe0ac361c68 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -1,5 +1,7 @@ import { describe, expect } from "bun:test" -import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Schema, Scope, Stream } from "effect" +import { + Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Schema, Scope, Stream, +} from "effect" import { EventV2 } from "@opencode-ai/core/event" import { Event } from "@opencode-ai/schema/event" import { Session } from "@opencode-ai/schema/session" @@ -1165,7 +1167,9 @@ describe("EventV2", () => { expect(String(stale)).toContain("Owner fence") expect(rows.map((row) => row.seq)).toEqual([0, 1, 2, 3]) expect(rows.map((row) => (row.data as { messageID: string }).messageID).sort()).toEqual( - ["model", "tool-a", "tool-b", "seal"].map((writer) => durableData(aggregateID, writer).messageID).sort(), + ["model", "tool-a", "tool-b", "seal"] + .map((writer) => durableData(aggregateID, writer).messageID) + .sort(), ) }), ) @@ -1198,7 +1202,8 @@ describe("EventV2", () => { it.live("ends a durable tail when the event layer is released under it", () => Effect.gen(function* () { const aggregateID = Session.ID.create() - // The layer needs its own closeable scope, and the consumer has to be forked into a scope that + // The layer needs its own closeable scope, and the consumer has to be forked into a scope + // that // OUTLIVES it. Fork into the test's own scope and closing that scope interrupts the consumer // before the finalizer runs, so a hung tail and a killed one look the same. const outer = yield* Effect.scope @@ -1213,7 +1218,8 @@ describe("EventV2", () => { yield* Effect.sleep("150 millis") yield* Scope.close(layerScope, Exit.void) - // The tick never ends on its own, so it must not become what holds the tail open: a subscriber + // The tick never ends on its own, so it must not become what holds the tail open: a + // subscriber // would outlive the layer and keep reading a database being torn down. const settled = yield* Fiber.await(tail).pipe(Effect.timeout("5 seconds"), Effect.exit) expect(Exit.isSuccess(settled)).toBe(true) diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index a4500e806f31..d6c38588cf7c 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -1,4 +1,5 @@ -// The model-only attempt (L2): runModelCall performs one provider attempt, records the tool calls it +// The model-only attempt (L2): runModelCall performs one provider attempt, records the tool calls +// it // asked for, and stops. The caller dispatches each call as its own unit of work and seals the step // afterwards, which is what puts the model-to-tools loop in a durable executor rather than inside a // single activity. These tests pin the three properties that split depends on: @@ -56,9 +57,16 @@ const model = OpenAIChat.route .model({ id: "gpt-4o-mini" }) const models = SessionRunnerModel.layerWith(() => Effect.succeed(model)) const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) -const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) -const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) -const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) +const skillGuidance = Layer.mock(SkillGuidance.Service, { + load: () => Effect.succeed(SystemContext.empty), +}) +const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { + load: () => Effect.succeed(SystemContext.empty), +}) +const config = Layer.succeed( + Config.Service, + Config.Service.of({ entries: () => Effect.succeed([]) }), +) const permission = Layer.mock(PermissionV2.Service, {}) const mockClient = (stream: LLMClientShape["stream"]) => @@ -104,7 +112,10 @@ const callsIdempotentTool: LLMClientShape["stream"] = () => // the log for a seal to find. The whole-step path survives it because Step.Ended mints one on the // way past; a seal running in another process has no publisher to mint with. const silent: LLMClientShape["stream"] = () => - Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })]) + Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.stepFinish({ index: 0, reason: "stop" }), + ]) // An answer and no tool call: the step is over as soon as the stream is, but the seal still has to // happen, and there is a real assistant message for it to complete. const textOnly: LLMClientShape["stream"] = () => @@ -162,7 +173,14 @@ const seedSession = Effect.gen(function* () { .pipe(Effect.orDie) yield* db .insert(SessionTable) - .values({ id: sessionID, project_id: Project.ID.global, slug: "t", directory: "/project", title: "t", version: "t" }) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "t", + directory: "/project", + title: "t", + version: "t", + }) .onConflictDoNothing() .run() .pipe(Effect.orDie) @@ -276,7 +294,8 @@ describe("SessionRunner model-only attempt", () => { if (result.kind !== "called") return expect(result.calls).toHaveLength(0) expect(result.settlement?.finish).toBe("stop") - // Sealing is uniform: even with nothing to dispatch, the step is closed by the seal, not here, + // Sealing is uniform: even with nothing to dispatch, the step is closed by the seal, not + // here, // so the answer is recorded but its message is still open. const message = assistant(yield* store.context(sessionID)) expect(message?.type).toBe("assistant") @@ -292,7 +311,13 @@ describe("SessionRunner model-only attempt", () => { const runner = yield* SessionRunner.Service const store = yield* SessionStore.Service - yield* runner.runStep({ sessionID, step: 2, promotion: undefined, first: false, force: false }) + yield* runner.runStep({ + sessionID, + step: 2, + promotion: undefined, + first: false, + force: false, + }) // The contrast that makes the split meaningful: the whole-step path dispatches and seals. expect(ran.write).toBe(1) @@ -357,42 +382,46 @@ describe("SessionRunner tool dispatch", () => { }), ) - harness(callsTool).effect("reports a retried side-effecting call as unknown instead of repeating it", () => - Effect.gen(function* () { - yield* seedSession - const ran = counters() - yield* registerProbes(ran) - const call = yield* deferOneCall - const runner = yield* SessionRunner.Service - const store = yield* SessionStore.Service - - // The first attempt died somewhere between dispatch and its result landing, so the log still - // shows the call in flight and nothing can say whether the write happened. - const result = yield* runner.runToolCall({ sessionID, call, retry: true }) - - expect(result.outcome).toBe("unknown") - expect(ran.write).toBe(0) - const part = toolPart(yield* store.context(sessionID), "call_probe") - expect(part?.type === "tool" ? part.state.status : undefined).toBe("error") - }), + harness(callsTool).effect( + "reports a retried side-effecting call as unknown instead of repeating it", + () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const call = yield* deferOneCall + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + // The first attempt died between dispatch and its result landing, so the log still + // shows the call in flight and nothing can say whether the write happened. + const result = yield* runner.runToolCall({ sessionID, call, retry: true }) + + expect(result.outcome).toBe("unknown") + expect(ran.write).toBe(0) + const part = toolPart(yield* store.context(sessionID), "call_probe") + expect(part?.type === "tool" ? part.state.status : undefined).toBe("error") + }), ) - harness(callsIdempotentTool).effect("re-runs a retried call that declares itself repeatable", () => - Effect.gen(function* () { - yield* seedSession - const ran = counters() - yield* registerProbes(ran) - const call = yield* deferOneCall - const runner = yield* SessionRunner.Service - const store = yield* SessionStore.Service - - const result = yield* runner.runToolCall({ sessionID, call, retry: true }) - - expect(result.outcome).toBe("settled") - expect(ran.read).toBe(1) - const part = toolPart(yield* store.context(sessionID), "call_probe") - expect(part?.type === "tool" ? part.state.status : undefined).toBe("completed") - }), + harness(callsIdempotentTool).effect( + "re-runs a retried call that declares itself repeatable", + () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const call = yield* deferOneCall + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + const result = yield* runner.runToolCall({ sessionID, call, retry: true }) + + expect(result.outcome).toBe("settled") + expect(ran.read).toBe(1) + const part = toolPart(yield* store.context(sessionID), "call_probe") + expect(part?.type === "tool" ? part.state.status : undefined).toBe("completed") + }), ) }) @@ -642,7 +671,6 @@ describe("SessionRunner model-only attempt under compaction", () => { yield* seeder.publish(LLMEvent.textEnd({ id: "old" })) yield* seeder.publish(LLMEvent.stepFinish({ index: 0, reason: "stop" })) - const runner = yield* SessionRunner.Service const result = yield* runner.runModelCall({ sessionID, diff --git a/packages/core/test/step-overlap-bench.test.ts b/packages/core/test/step-overlap-bench.test.ts index 08ca4a8227ba..c8bdc4ee93ec 100644 --- a/packages/core/test/step-overlap-bench.test.ts +++ b/packages/core/test/step-overlap-bench.test.ts @@ -67,9 +67,16 @@ const model = OpenAIChat.route .model({ id: "gpt-4o-mini" }) const models = SessionRunnerModel.layerWith(() => Effect.succeed(model)) const systemContext = AppNodeBuilder.build(SystemContextRegistry.node) -const skillGuidance = Layer.mock(SkillGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) -const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { load: () => Effect.succeed(SystemContext.empty) }) -const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed([]) })) +const skillGuidance = Layer.mock(SkillGuidance.Service, { + load: () => Effect.succeed(SystemContext.empty), +}) +const referenceGuidance = Layer.mock(ReferenceGuidance.Service, { + load: () => Effect.succeed(SystemContext.empty), +}) +const config = Layer.succeed( + Config.Service, + Config.Service.of({ entries: () => Effect.succeed([]) }), +) const permission = Layer.mock(PermissionV2.Service, {}) // One tool call, then a stream that keeps going for TAIL_MS. That trailing stream is the whole @@ -83,9 +90,18 @@ const withTail: LLMClientShape["stream"] = () => ]), Stream.concat( Stream.fromIterable( - Array.from({ length: TAIL_CHUNKS }, (_, i) => LLMEvent.textDelta({ id: "txt_1", text: `chunk ${i} ` })), - ).pipe(Stream.mapEffect((event) => Effect.sleep(Duration.millis(TAIL_CHUNK_MS)).pipe(Effect.as(event)))), - Stream.fromIterable([LLMEvent.textEnd({ id: "txt_1" }), LLMEvent.stepFinish({ index: 0, reason: "tool-calls" })]), + Array.from({ length: TAIL_CHUNKS }, (_, i) => + LLMEvent.textDelta({ id: "txt_1", text: `chunk ${i} ` }), + ), + ).pipe( + Stream.mapEffect((event) => + Effect.sleep(Duration.millis(TAIL_CHUNK_MS)).pipe(Effect.as(event)), + ), + ), + Stream.fromIterable([ + LLMEvent.textEnd({ id: "txt_1" }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + ]), ), ) @@ -108,14 +124,17 @@ const harness = testEffect( ApplicationTools.node, ]), [ - [LayerNodePlatform.llmClient, Layer.succeed( - LLMClient.Service, - LLMClient.Service.of({ - prepare: () => Effect.die("unused"), - generate: () => Effect.die("unused"), - stream: withTail, - }), - )], + [ + LayerNodePlatform.llmClient, + Layer.succeed( + LLMClient.Service, + LLMClient.Service.of({ + prepare: () => Effect.die("unused"), + generate: () => Effect.die("unused"), + stream: withTail, + }), + ), + ], [PermissionV2.node, permission], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], [SessionRunnerModel.node, models], @@ -140,7 +159,14 @@ const seed = (sessionID: SessionV2.ID) => .pipe(Effect.orDie) yield* db .insert(SessionTable) - .values({ id: sessionID, project_id: Project.ID.global, slug: "t", directory: "/project", title: "t", version: "t" }) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "t", + directory: "/project", + title: "t", + version: "t", + }) .onConflictDoNothing() .run() .pipe(Effect.orDie) @@ -169,7 +195,8 @@ describe.skipIf(!process.env.OPENCODE_OVERLAP_BENCH)("step overlap cost", () => const fused = SessionV2.ID.make("ses_bench_fused") yield* seed(fused) - // Whole step in one go: the tool forks the instant its call arrives, so it runs under the tail. + // Whole step in one go: the tool forks the instant its call arrives, so it runs under the + // tail. const fusedMs = yield* elapsed(runner.runStep({ sessionID: fused, ...step })) const split = SessionV2.ID.make("ses_bench_split") @@ -179,8 +206,13 @@ describe.skipIf(!process.env.OPENCODE_OVERLAP_BENCH)("step overlap cost", () => Effect.gen(function* () { const result = yield* runner.runModelCall({ sessionID: split, ...step }) if (result.kind !== "called") return - for (const call of result.calls) yield* runner.runToolCall({ sessionID: split, call, retry: false }) - yield* runner.sealStep({ sessionID: split, step: result.step, settlement: result.settlement }) + for (const call of result.calls) + yield* runner.runToolCall({ sessionID: split, call, retry: false }) + yield* runner.sealStep({ + sessionID: split, + step: result.step, + settlement: result.settlement, + }) }), ) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 88c95fd2a462..b7c1a3027a12 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -22,7 +22,8 @@ One env var picks the executor. `temporal` runs each session as a per-session Te with one activity per step (this package: `executor.ts` wires the client and worker, `supervisor.ts` is the loop, `workflow.ts` adapts it to the sandbox, `drain.ts` is the step body). The default runs in-process on the proven `SessionRunCoordinator` (core's `execution/local.ts`), the same lifecycle -the v1 server uses, with no server and no ports (see [Two modes, one runner](#two-modes-one-runner)). +the v1 server uses, with no server and no ports (see [Two modes, one +runner](#two-modes-one-runner)). What an executor must do is defined executably: core's conformance suite (`session/execution/conformance.ts`) runs the same wake/resume/interrupt scenarios against the local executor in core's tests and against this package through real workflows. @@ -109,7 +110,8 @@ runModelCall -> runToolCall (one per call, concurrent) -> sealStep `SessionRunner.runModelCall` performs the attempt, records each call as `Tool.Called`, and hands the calls back rather than running them. `runToolCall` settles one call. `sealStep` takes the end snapshot, diffs it against the start, and publishes `Step.Ended`. The loop between them is workflow -code, so a retry policy, a timeout, an approval or a budget can live where the model-to-tools handoff +code, so a retry policy, a timeout, an approval or a budget can live where the model-to-tools +handoff used to be. Each activity also carries its own bounds: sealing does not inherit a turn-sized backstop, and one tool waiting on a human no longer holds the attempt and its sibling tools under a single timeout. @@ -230,7 +232,8 @@ Verified live (dev server, `gpt-5-mini`, stepped mode on): - **A tool activity rebuilds a worktree it has never seen.** Each of the three drains calls `worktrees.ensure`, so a tool call landing on a worker without the project tree materializes it from the snapshot packs. Checked by deleting the entire working directory between two turns of a - live session: the next turn's tool activity rebuilt both files, the `read` completed, and the model + live session: the next turn's tool activity rebuilt both files, the `read` completed, and the + model answered with the file's contents. Worker affinity (below) skips the materialization on warm paths; the packs are the baseline that works without it. @@ -260,7 +263,8 @@ to satisfy the same one or the two Temporal modes drift and only the tested one temporal server start-dev --port 7237 --headless & cd packages/temporal # whole step per activity -OPENCODE_CONTRACT_TEMPORAL=1 bun test --timeout 180000 test/session-execution-temporal-contract.test.ts +OPENCODE_CONTRACT_TEMPORAL=1 \ + bun test --timeout 180000 test/session-execution-temporal-contract.test.ts # model call, one activity per tool, seal OPENCODE_CONTRACT_TEMPORAL=1 OPENCODE_TEMPORAL_STEPPED=1 bun test --timeout 180000 \ test/session-execution-temporal-contract.test.ts @@ -276,7 +280,8 @@ are carried. #### Worker affinity (`OPENCODE_TEMPORAL_WORKTREE_AFFINITY=1`) -Off by default. Without it every worker polls one queue, and a worker drawing a session whose tree it +Off by default. Without it every worker polls one queue, and a worker drawing a session whose tree +it has never seen rebuilds that tree from snapshot packs. That is the portable baseline and it works. Affinity avoids the rebuild by routing instead: the queue name is derived from the session's directory, and only workers serving that directory poll it. @@ -290,7 +295,8 @@ OPENCODE_TEMPORAL_WORKTREE_AFFINITY=1 OPENCODE_TEMPORAL_WORKTREE=/srv/trees/acme The queue is keyed on the session's `location.directory`, not the project root, because that is the tree `worktrees.ensure` has to produce and two sessions in one project can sit in different directories. Paths are resolved through `realpath` first: on macOS `/tmp/x` and `/private/tmp/x` are -one tree, and a client and a worker that disagreed would sit on two queues and the session would hang +one tree, and a client and a worker that disagreed would sit on two queues and the session would +hang with nothing to show for it. **This trades availability for latency, which is why it is opt-in.** With affinity on, a session @@ -345,7 +351,8 @@ log. `OPENCODE_SESSION_EXECUTION=temporal` runs each session as a per-session Te `sessionTurn` supervisor (`supervisor.ts`) loops a `runTurnStep` drain, so each step (one provider attempt + its tools) is its own activity with its own retry/timeout/visibility, reusing `SessionRunner.runStep` (one iteration of `run`'s loop). Anything else (the default) runs in-process -on the proven `SessionRunCoordinator` (`execution/local.ts`) -- no server, no worker, no ports -- which +on the proven `SessionRunCoordinator` (`execution/local.ts`) -- no server, no worker, no ports -- +which drives whole turns with `SessionRunner.run` and owns the wake/resume/interrupt lifecycle. That coordinator is the same one the v1 server uses and has direct lifecycle tests (`session-run-coordinator.test.ts`), so the default path reuses well-exercised code rather than a @@ -361,7 +368,8 @@ local modes were folded away in favor of the coordinator for local.) A per-step re-drive resumes from the durable event log rather than re-running work. `runStep` closes any tool left dangling by an interrupted attempt on every entry, not just the first. Without that, a mid-turn retry (`first=false`) re-streamed a request carrying a `tool_use` with no `tool_result`, -which the provider rejects, a retry poison loop. And if the crashed step had already dispatched tools +which the provider rejects, a retry poison loop. And if the crashed step had already dispatched +tools it is finalized from the log: completed tool results are kept, still-unsettled tools are failed, and a synthesized `Step.Ended` closes the step without re-calling the model. A tool caught in flight at the crash is handled by declared idempotency: a side-effect-free tool (`read`/`glob`/`grep`, marked @@ -426,7 +434,8 @@ have the session's worktree present (see "What resumes cross-host"). ### Durable permission asks A tool waiting for user approval used to park on an in-memory deferred: invisible outside the asking -process (a standalone worker's ask could never be answered) and gone on restart. A pending ask is now +process (a standalone worker's ask could never be answered) and gone on restart. A pending ask is +now also a row in the shared store (`permission_request`). The blocked `assert` races its local deferred against a poll of the row, so a reply from ANY process sharing the store (the HTTP server answering for a detached worker) unblocks it; `list`/`get`/`forSession` read the rows, so serve can show asks @@ -499,10 +508,14 @@ distinct worker identities. ### What resumes cross-host, and what does not -The runner rebuilds a session's LLM context purely from the shared DB (`SessionHistory.entriesForRunner` -then `toLLMMessages`); it never reads local disk to reconstruct context. So the **conversation** resumes -on any worker: messages, tool results (the bounded preview and structured output that the model sees), -prompt attachments (stored inline as `data:` URIs in the prompt), and credentials (`CredentialTable`) +The runner rebuilds a session's LLM context purely from the shared DB +(`SessionHistory.entriesForRunner` +then `toLLMMessages`); it never reads local disk to reconstruct context. So the **conversation** +resumes +on any worker: messages, tool results (the bounded preview and structured output that the model +sees), +prompt attachments (stored inline as `data:` URIs in the prompt), and credentials +(`CredentialTable`) all ride the shared store. **The project working tree** now rides the store too. After each step capture the runner ships @@ -517,9 +530,12 @@ packs are the portable baseline that works with neither. Host-local state that does NOT ride the DB, so it is not reconstructed on a different host: - **The snapshot store (`${data}/snapshot`) and the retained full tool-output files - (`${data}/tool-output`).** The runner never reads these to rebuild context: snapshot file-diffs are - best-effort (`Effect.catch` to `undefined`), and the model sees the bounded tool-output preview, not - the file. They only affect the diff/restore/revert features and full-output viewing. Point `${data}` + (`${data}/tool-output`).** The runner never reads these to rebuild context: snapshot file-diffs + are + best-effort (`Effect.catch` to `undefined`), and the model sees the bounded tool-output preview, + not + the file. They only affect the diff/restore/revert features and full-output viewing. Point + `${data}` (the XDG data dir) at shared storage to make them portable. ## Porting this pattern diff --git a/packages/temporal/scripts/stream-tail-probe.ts b/packages/temporal/scripts/stream-tail-probe.ts index 0cbfa9b9cff7..55fae4c6d7e2 100644 --- a/packages/temporal/scripts/stream-tail-probe.ts +++ b/packages/temporal/scripts/stream-tail-probe.ts @@ -1,8 +1,10 @@ // How much of a real provider's stream arrives AFTER it has asked for its first tool? // // That gap is the only thing a stepped executor gives up: a whole-step activity forks the tool -// immediately and runs it under the rest of the stream, while a split has to wait for the attempt to -// return. The bench measured the loss as min(tail, tool duration) with a dialled mock. This measures +// immediately and runs it under the rest of the stream, while a split has to wait for the attempt +// to +// return. The bench measured the loss as min(tail, tool duration) with a dialled mock. This +// measures // the tail itself, against the real API, so the mock's dial can be set to something honest. // // The measuring point matters. OpenCode forks on the `tool-call` event, which the protocol layer @@ -62,7 +64,8 @@ const PROBES: ReadonlyArray = [ { label: "single-call-terse", prompt: "What files are here? Use the bash tool once." }, { label: "three-calls", - prompt: "Read these three files: src/a.ts, src/b.ts, src/c.ts. Use the read_file tool, one call per file.", + prompt: + "Read these three files: src/a.ts, src/b.ts, src/c.ts. Use the read_file tool, one call per file.", }, { label: "five-calls", @@ -71,7 +74,8 @@ const PROBES: ReadonlyArray = [ }, { label: "narrate-then-call", - prompt: "Say one short sentence about what you are going to check, then run `git status` with the bash tool.", + prompt: + "Say one short sentence about what you are going to check, then run `git status` with the bash tool.", }, { label: "call-then-narrate", @@ -137,7 +141,8 @@ const streamResponses = async (model: string, prompt: string): Promise = return consume(res, started, (obj) => { const ids: string[] = [] // The fork point: arguments complete, so the call can actually be dispatched. - if (obj?.type === "response.function_call_arguments.done") ids.push(String(obj.item_id ?? obj.output_index)) + if (obj?.type === "response.function_call_arguments.done") + ids.push(String(obj.item_id ?? obj.output_index)) const text = obj?.type === "response.output_text.delta" ? String(obj.delta ?? "").length : 0 return { toolIds: ids, text } }) @@ -209,7 +214,10 @@ for (const probe of PROBES) { let after = 0 for (let i = 0; i < runs; i++) { try { - const r = api === "responses" ? await streamResponses(model, probe.prompt) : await streamChat(model, probe.prompt) + const r = + api === "responses" + ? await streamResponses(model, probe.prompt) + : await streamChat(model, probe.prompt) if (r.firstToolAt === undefined) continue tails.push(Math.round(r.lastAt - r.firstToolAt)) calls = Math.max(calls, r.calls) diff --git a/packages/temporal/src/activities.ts b/packages/temporal/src/activities.ts index 6403d4a2e47a..3e21f5bb5bc2 100644 --- a/packages/temporal/src/activities.ts +++ b/packages/temporal/src/activities.ts @@ -6,7 +6,8 @@ import { heartbeat, Context } from "@temporalio/activity" // The event-log owner token for one activity execution: run id + activity id + attempt. A Temporal -// retry of the SAME step gets a fresh attempt, so once the retry claims the log the previous attempt +// retry of the SAME step gets a fresh attempt, so once the retry claims the log the previous +// attempt // (if still running) is fenced out. The activity id is essential: activity attempt numbers restart // at 1 for every step, so a run-id+attempt token alone would repeat across steps (step 1 attempt 1 // and step 2 attempt 1 both mint `run#1`), letting a zombie attempt from an earlier step re-match @@ -19,7 +20,8 @@ export function ownerTokenFrom(run: string, activityId: string, attempt: number) function ownerToken(): string { const info = Context.current().info // runId/workflowType are typed optional on ActivityInfo; activityId is always present and already - // makes the token unique per execution, so an empty run prefix in the (degenerate) missing case is + // makes the token unique per execution, so an empty run prefix in the (degenerate) missing case + // is // harmless. const run = info.workflowExecution?.runId ?? info.workflowType ?? "" return ownerTokenFrom(run, info.activityId, info.attempt) @@ -63,7 +65,9 @@ export function makeStepActivities( ): StepActivities { return { async runTurnStep(input) { - return beating(() => stepDrain({ ...input, owner: ownerToken() }, Context.current().cancellationSignal)) + return beating(() => + stepDrain({ ...input, owner: ownerToken() }, Context.current().cancellationSignal), + ) }, } } @@ -74,7 +78,8 @@ export type SteppedTurnActivities = { sealStep(input: SealDrainInput): Promise } -// The three activities of a stepped step. Only the model call mints and claims an owner token: it is +// The three activities of a stepped step. Only the model call mints and claims an owner token: it +// is // the writer that supersedes the attempt before it, and the tool and seal activities publish under // the token it returned. Giving each of them its own token would make a step's writers fence each // other out of the log. @@ -89,7 +94,10 @@ export function makeSteppedTurnActivities(drains: { return { async runModelCall(input) { return beating(() => - drains.modelCallDrain({ ...input, owner: ownerToken() }, Context.current().cancellationSignal), + drains.modelCallDrain( + { ...input, owner: ownerToken() }, + Context.current().cancellationSignal, + ), ) }, async runToolCall(input) { @@ -97,7 +105,9 @@ export function makeSteppedTurnActivities(drains: { // workflow's: a second attempt of THIS call is a re-run of a tool whose result never landed. // The workflow could not compute this without reading non-deterministic state. const retry = Context.current().info.attempt > 1 - return beating(() => drains.toolCallDrain({ ...input, retry }, Context.current().cancellationSignal)) + return beating(() => + drains.toolCallDrain({ ...input, retry }, Context.current().cancellationSignal), + ) }, async sealStep(input) { return beating(() => drains.sealDrain(input, Context.current().cancellationSignal)) diff --git a/packages/temporal/src/boundary.ts b/packages/temporal/src/boundary.ts index 8aed9f35de29..6d93d42f6bea 100644 --- a/packages/temporal/src/boundary.ts +++ b/packages/temporal/src/boundary.ts @@ -1,10 +1,13 @@ // Crossing from Effect into an activity result. Every drain in this package ends here, so the way a // failure is classified is decided in one place: -// - an interrupt caused by the driver's own cancellation rethrows the abort reason, so the attempt +// - an interrupt caused by the driver's own cancellation rethrows the abort reason, so the +// attempt // records Cancelled rather than Failed -// - an interrupt with no cancellation is an internal halt (a user declining a permission) and must +// - an interrupt with no cancellation is an internal halt (a user declining a permission) and +// must // be non-retryable, or the supervisor re-drives a turn the user explicitly stopped -// - a genuine run error crosses non-retryable with the RunError encoded in `details`, so the caller +// - a genuine run error crosses non-retryable with the RunError encoded in `details`, so the +// caller // reconstructs the exact typed error instead of a string // Only crashes and task timeouts (never thrown here) go through the activity retry policy. @@ -26,7 +29,9 @@ export const runAtBoundary = async ( if (Cause.hasInterruptsOnly(cause)) { if (signal.aborted) throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") - const declined = encodeRunError(new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make(sessionID) })) + const declined = encodeRunError( + new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make(sessionID) }), + ) throw ApplicationFailure.create({ message: "session run halted (user declined)", type: HALTED_FAILURE_TYPE, diff --git a/packages/temporal/src/drain.ts b/packages/temporal/src/drain.ts index 129b89d70e96..8d4363022ce3 100644 --- a/packages/temporal/src/drain.ts +++ b/packages/temporal/src/drain.ts @@ -1,5 +1,6 @@ // The per-step drain body for the Temporal layer: it runs inside the runTurnStep activity. It wraps -// one SessionRunner.runStep, claims the event log for the attempt, ensures the worktree, and encodes +// one SessionRunner.runStep, claims the event log for the attempt, ensures the worktree, and +// encodes // the error for the activity boundary. Local mode does not use this: it runs whole turns through // SessionRunner.run on the SessionRunCoordinator (execution/local.ts). Both modes go through the // same SessionRunner and the same durable event log. diff --git a/packages/temporal/src/executor.ts b/packages/temporal/src/executor.ts index ee72ca178620..4cd267d16cf9 100644 --- a/packages/temporal/src/executor.ts +++ b/packages/temporal/src/executor.ts @@ -24,7 +24,8 @@ import { TemporalConfig } from "./config" import { WORKFLOW_TYPE, WORKFLOW_ID_PREFIX, workflowId } from "./protocol" // Classify an interrupt-signal delivery error. "already completed"/"not found" means an idle -// session's workflow has already closed -- nothing to interrupt, a no-op. Anything else is a genuine +// session's workflow has already closed -- nothing to interrupt, a no-op. Anything else is a +// genuine // control-plane failure: the user's stop was not delivered, and it must NOT be reported as success. export function classifyInterruptError(e: unknown): "ignore" | "fail" { const message = String((e as { message?: unknown })?.message ?? e) @@ -90,7 +91,9 @@ const layer = Layer.effect( // Worker connection (native) hosts the runTurnStep activity + the workflow. Skipped in // client-only role so serve can run without an embedded worker. if (HOST_WORKER) { - const { NativeConnection, Worker } = yield* Effect.tryPromise(() => import("@temporalio/worker")).pipe( + const { NativeConnection, Worker } = yield* Effect.tryPromise( + () => import("@temporalio/worker"), + ).pipe( Effect.catch(() => Effect.die( "The embedded Temporal worker is unavailable in this build. Run standalone workers " + @@ -122,7 +125,8 @@ const layer = Layer.effect( } // Worker-only process: it hosts activities but drives no workflows, so the client methods are - // unused. Return a service whose driving methods fail loudly if something unexpectedly calls them. + // unused. Return a service whose driving methods fail loudly if something unexpectedly calls + // them. if (!HOST_CLIENT) { yield* Effect.logInfo("SessionExecutionTemporal worker ready").pipe( Effect.annotateLogs({ @@ -157,7 +161,11 @@ const layer = Layer.effect( workflowId: workflowId(id), args: [ id, - { startWithWake: true, idleTimeout: IDLE_TIMEOUT, stepped: STEPPED } satisfies WF.SessionTurnOptions, + { + startWithWake: true, + idleTimeout: IDLE_TIMEOUT, + stepped: STEPPED, + } satisfies WF.SessionTurnOptions, ], signal: WF.wake, signalArgs: [], @@ -215,8 +223,10 @@ const layer = Layer.effect( const startOp = new WithStartWorkflowOperation(WORKFLOW_TYPE, { taskQueue: resumeQueue, workflowId: workflowId(id), - // startWithWake=false: a fresh resume-with-start must not manufacture a wake drain; - // its forced drain comes from the resume update. Ignored when USE_EXISTING joins a + // startWithWake=false: a fresh resume-with-start must not manufacture a wake + // drain; + // its forced drain comes from the resume update. Ignored when USE_EXISTING joins + // a // running workflow (which keeps its own state). args: [ id, @@ -256,7 +266,8 @@ const layer = Layer.effect( // An idle session's workflow has already completed; nothing to interrupt is fine. if (classifyInterruptError(e) === "ignore") return Effect.void // A genuine delivery failure must not read as success: the stop did not happen. The - // interface has no error channel, so surface it as a defect rather than a false success. + // interface has no error channel, so surface it as a defect rather than a false + // success. const message = String((e as { message?: unknown })?.message ?? e) return Effect.logError("session interrupt signal failed").pipe( Effect.annotateLogs({ sessionID: id, error: message }), diff --git a/packages/temporal/src/l2-drain.ts b/packages/temporal/src/l2-drain.ts index 702d5758026c..2a43766506df 100644 --- a/packages/temporal/src/l2-drain.ts +++ b/packages/temporal/src/l2-drain.ts @@ -78,7 +78,10 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra sessionID: string, owner: string, claim: boolean, - use: (runner: SessionRunner.Interface, session: SessionSchema.Info) => Effect.Effect, + use: ( + runner: SessionRunner.Interface, + session: SessionSchema.Info, + ) => Effect.Effect, ) => Effect.gen(function* () { const session = yield* store.get(SessionSchema.ID.make(sessionID)) @@ -86,7 +89,8 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra // reports it as nothing to do, and a resume waiting on this should resolve, not reject. if (!session) return undefined if (claim) yield* events.claim(session.id, owner) - // A worker taking this step on a host without the project tree rebuilds it from snapshot packs. + // A worker taking this step on a host without the project tree rebuilds it from snapshot + // packs. yield* worktrees.ensure(session.location.directory) return yield* SessionRunner.Service.use((runner) => use(runner, session)).pipe( Effect.provide(locations.get(session.location)), @@ -109,36 +113,40 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra force: input.force, }), ).pipe( - Effect.map((result): ModelCallDrainResult => - result === undefined - ? { - kind: "settled", - result: { ran: false, continue: false, step: input.step, promotion: null }, - } - : result.kind === "settled" - ? { - kind: "settled", - result: { - ran: result.result.ran, - continue: result.result.continue, - step: result.result.step, - promotion: result.result.promotion ?? null, - }, - } - : { - kind: "called", - step: result.step, - calls: result.calls, - settlement: result.settlement, - assistantMessageID: result.assistantMessageID, - needsContinuation: result.needsContinuation, - owner: input.owner, - }, + Effect.map( + (result): ModelCallDrainResult => + result === undefined + ? { + kind: "settled", + result: { ran: false, continue: false, step: input.step, promotion: null }, + } + : result.kind === "settled" + ? { + kind: "settled", + result: { + ran: result.result.ran, + continue: result.result.continue, + step: result.result.step, + promotion: result.result.promotion ?? null, + }, + } + : { + kind: "called", + step: result.step, + calls: result.calls, + settlement: result.settlement, + assistantMessageID: result.assistantMessageID, + needsContinuation: result.needsContinuation, + owner: input.owner, + }, ), ), ) - const toolCallDrain = async (input: ToolCallDrainInput, signal: AbortSignal): Promise => + const toolCallDrain = async ( + input: ToolCallDrainInput, + signal: AbortSignal, + ): Promise => runAtBoundary( input.sessionID, signal, diff --git a/packages/temporal/src/l2-step.ts b/packages/temporal/src/l2-step.ts index efd12a86d2bc..55950e0f9b98 100644 --- a/packages/temporal/src/l2-step.ts +++ b/packages/temporal/src/l2-step.ts @@ -1,5 +1,6 @@ // One step as three units of work instead of one: the provider attempt, each tool call it asks for, -// and the seal that closes it. This is the whole point of the split, and it is workflow code, so the +// and the seal that closes it. This is the whole point of the split, and it is workflow code, so +// the // model-to-tools loop lives where retries, timers, approvals and budgets can sit between the two. // // MUST stay pure, like supervisor.ts: this is bundled into the workflow sandbox, so no `effect`, no @@ -7,7 +8,8 @@ // // What this costs, stated plainly: a whole-step activity starts each tool the moment the model asks // for it, while the stream is still going. Here the attempt has to return before any tool starts, -// because a workflow cannot consume a stream. The tools of one step still run concurrently with each +// because a workflow cannot consume a stream. The tools of one step still run concurrently with +// each // other; what is lost is the overlap between the model and its own tools. import { ActivityFailure, type ApplicationFailure } from "@temporalio/workflow" @@ -22,7 +24,8 @@ import type { } from "./l2-drain" /** - * A halt the user asked for, as it looks once it has crossed an activity boundary. The runner raises + * A halt the user asked for, as it looks once it has crossed an activity boundary. The runner + * raises * it as an interrupt, `boundary.ts` throws it as an `ApplicationFailure`, and the SDK wraps that in * one `ActivityFailure`. It is not a cancellation, so `isCancellation` says no, and a dispatcher * that checks only that would treat a refusal as one failed tool and carry on. @@ -42,8 +45,8 @@ export interface SteppedActivities { export interface SteppedTurnDeps { readonly activities: SteppedActivities - /** Whether an error is the driver's cancellation. An interrupt has to end the turn, so it must not - * be swallowed the way a failed tool is. */ + /** Whether an error is the driver's cancellation. An interrupt has to end the turn, so it must + * not be swallowed the way a failed tool is. */ readonly isCancellation: (error: unknown) => boolean /** Whether an error is the user stopping the turn, like a declined permission. It arrives as an * ordinary activity failure, so without this it reads as one bad tool and the turn carries on @@ -52,7 +55,8 @@ export interface SteppedTurnDeps { } /** - * Drives one step and reports the next loop state, so it drops straight into the supervisor in place + * Drives one step and reports the next loop state, so it drops straight into the supervisor in + * place * of a whole-step activity. */ export const makeSteppedTurn = diff --git a/packages/temporal/src/queue.ts b/packages/temporal/src/queue.ts index edce9d31485b..8e12e33f7047 100644 --- a/packages/temporal/src/queue.ts +++ b/packages/temporal/src/queue.ts @@ -1,18 +1,21 @@ // Routing a session's work to a worker that already has its project tree. // -// Without affinity every worker polls one queue, and a worker that draws a session whose worktree it +// Without affinity every worker polls one queue, and a worker that draws a session whose worktree +// it // has never seen rebuilds it from snapshot packs (`session/execution/worktree.ts`). That is the // portable baseline and it works, but it costs a materialization on the first step a worker takes // for that session. Affinity avoids the cost by routing instead: the queue name is derived from the // session's directory, and only workers serving that directory poll it. // -// The trade is availability for latency, and it is the whole reason this is opt-in. With affinity on, +// The trade is availability for latency, and it is the whole reason this is opt-in. With affinity +// on, // a session whose worktree has no worker polling for it does not fall back to another worker; it // waits. Reconstruction is what makes any worker able to serve any session, and turning affinity on // is choosing not to use it. // // Deliberately NOT imported by workflow code: this reaches for node:crypto, which the Temporal -// sandbox does not have. The queue is chosen by the client that starts the workflow and by the worker +// sandbox does not have. The queue is chosen by the client that starts the workflow and by the +// worker // that polls, both of which are ordinary Node. import { createHash } from "node:crypto" import { resolve } from "node:path" @@ -37,5 +40,6 @@ const DIGEST_LENGTH = 12 */ export const queueForWorktree = (base: string, directory: string): string => { const canonical = resolve(directory).replace(/[/\\]+$/, "") - return `${base}-wt-${createHash("sha256").update(canonical).digest("hex").slice(0, DIGEST_LENGTH)}` + const digest = createHash("sha256").update(canonical).digest("hex").slice(0, DIGEST_LENGTH) + return `${base}-wt-${digest}` } diff --git a/packages/temporal/src/workflow.ts b/packages/temporal/src/workflow.ts index be0664e8d2d6..a32dca546c97 100644 --- a/packages/temporal/src/workflow.ts +++ b/packages/temporal/src/workflow.ts @@ -28,7 +28,8 @@ const activityOptions = { // The heartbeat is the liveness bound (it stops within seconds of a worker death and Temporal // re-drives). startToClose is only the backstop for a drain that hangs while its process stays // alive, so it must comfortably exceed any legitimate turn: long tool runs, many steps, or a - // human taking their time over a permission ask. 30 minutes proved far too tight -- it hard-killed + // human taking their time over a permission ask. 30 minutes proved far too tight -- it + // hard-killed // legitimate turns and each kill opened a short two-writer window until the zombie attempt // noticed its heartbeat rejection. startToCloseTimeout: "12 hours", @@ -58,7 +59,8 @@ const runtime: SupervisorRuntime = { // Short-circuit when the predicate already holds. Besides saving a round trip, this avoids a real // breakage: on @temporalio/workflow 1.21, calling condition(fn, timeout) when fn is already true // leaves the current CancellationScope cancelled, so the NEXT condition() throws CancelledFailure - // -- which the supervisor reads as an interrupt and the workflow completes without ever draining a + // -- which the supervisor reads as an interrupt and the workflow completes without ever draining + // a // turn. Checking fn() first keeps the timeout timer (and its scope) out of the already-true path. // A timed wait that does NOT use the SDK's condition(fn, timeout). On @temporalio/workflow 1.21 // that variant cancels its internal timer scope on resolve and the cancellation LEAKS into the @@ -99,7 +101,8 @@ const runtime: SupervisorRuntime = { cancelCurrentScope: () => activeDrainScope?.cancel(), isCancellation, // A per-turn interrupt cancels only the child drain scope; a real workflow cancellation cancels - // the root. Reliable now that the timed wait no longer cancels any scope (nothing contaminates the + // the root. Reliable now that the timed wait no longer cancels any scope (nothing contaminates + // the // root's consideredCancelled). isRootCancelled: () => rootScope?.consideredCancelled ?? false, allHandlersFinished, diff --git a/packages/temporal/test/l2-step.test.ts b/packages/temporal/test/l2-step.test.ts index 343f9f032813..15b8728ab9de 100644 --- a/packages/temporal/test/l2-step.test.ts +++ b/packages/temporal/test/l2-step.test.ts @@ -6,7 +6,12 @@ import { describe, it, expect } from "bun:test" import { isHaltFailure, makeSteppedTurn, type SteppedActivities } from "../src/l2-step" import { runAtBoundary } from "../src/boundary" import { Effect } from "effect" -import { ActivityFailure, ApplicationFailure, CancelledFailure, TimeoutFailure } from "@temporalio/workflow" +import { + ActivityFailure, + ApplicationFailure, + CancelledFailure, + TimeoutFailure, +} from "@temporalio/workflow" import type { StepDrainInput, StepDrainResult } from "../src/activities" import type { ModelCallDrainResult, SealDrainInput, ToolCallDrainInput } from "../src/l2-drain" @@ -15,13 +20,26 @@ class FakeHalt extends Error {} const isCancellation = (e: unknown) => e instanceof FakeCancel const isHalt = (e: unknown) => e instanceof FakeHalt -const INPUT: StepDrainInput = { sessionID: "ses_1", step: 2, promotion: null, first: false, force: false } +const INPUT: StepDrainInput = { + sessionID: "ses_1", + step: 2, + promotion: null, + first: false, + force: false, +} const SEALED: StepDrainResult = { ran: true, continue: true, step: 3, promotion: "steer" } -const call = (id: string, name = "probe_write") => ({ id, name, input: {}, assistantMessageID: "msg_1" }) +const call = (id: string, name = "probe_write") => ({ + id, + name, + input: {}, + assistantMessageID: "msg_1", +}) const fakes = ( model: ModelCallDrainResult, - onTool: (input: ToolCallDrainInput) => Promise<{ outcome: "settled" }> = async () => ({ outcome: "settled" }), + onTool: (input: ToolCallDrainInput) => Promise<{ outcome: "settled" }> = async () => ({ + outcome: "settled", + }), ) => { const tools: ToolCallDrainInput[] = [] const seals: SealDrainInput[] = [] @@ -58,7 +76,10 @@ describe("stepped turn", () => { kind: "called", step: 2, calls: [call("call_a"), call("call_b")], - settlement: { finish: "tool-calls", tokens: { input: 1, output: 2, reasoning: 0, cache: { read: 0, write: 0 } } }, + settlement: { + finish: "tool-calls", + tokens: { input: 1, output: 2, reasoning: 0, cache: { read: 0, write: 0 } }, + }, owner: "run-1:model-1:1", }) @@ -135,10 +156,15 @@ describe("halt predicate", () => { new ActivityFailure("activity failed", "runToolCall", "1", 1 as never, undefined, cause) it("recognises what the activity boundary throws for a user halt", async () => { - // Built by running the boundary rather than by hand, so the two sides cannot drift: an interrupt + // Built by running the boundary rather than by hand, so the two sides cannot drift: an + // interrupt // with no abort is how a decline leaves the runner, and whatever that produces is what a // dispatcher has to recognise. - const thrown = await runAtBoundary("ses_1", new AbortController().signal, Effect.interrupt).then( + const thrown = await runAtBoundary( + "ses_1", + new AbortController().signal, + Effect.interrupt, + ).then( () => undefined, (error: unknown) => error, ) diff --git a/packages/temporal/test/session-execution-temporal-contract.test.ts b/packages/temporal/test/session-execution-temporal-contract.test.ts index e0900dd951c0..dcc381a703ac 100644 --- a/packages/temporal/test/session-execution-temporal-contract.test.ts +++ b/packages/temporal/test/session-execution-temporal-contract.test.ts @@ -6,7 +6,8 @@ // opt-in: // // temporal server start-dev --port 7237 --headless & -// OPENCODE_CONTRACT_TEMPORAL=1 bun test --timeout 120000 test/session-execution-temporal-contract.test.ts +// OPENCODE_CONTRACT_TEMPORAL=1 \ +// bun test --timeout 120000 test/session-execution-temporal-contract.test.ts // // Without the opt-in the file registers nothing, so a plain `bun test` stays server-free. import { makeExecutionFor, runContract } from "@opencode-ai/core/session/execution/conformance" @@ -24,5 +25,6 @@ if (process.env.OPENCODE_CONTRACT_TEMPORAL === "1") { const stepped = process.env.OPENCODE_TEMPORAL_STEPPED === "1" // Imported dynamically because the driver reads its connection config at module load. const { SessionExecutionTemporal } = await import("@opencode-ai/temporal/executor") - runContract(stepped ? "temporal driver (stepped)" : "temporal driver", makeExecutionFor(SessionExecutionTemporal.node)) + const label = stepped ? "temporal driver (stepped)" : "temporal driver" + runContract(label, makeExecutionFor(SessionExecutionTemporal.node)) } From 97bf46d10b6e95c133dca9561818d4a1ed245c8b Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 18:10:13 -0700 Subject: [PATCH 21/33] Keyed affinity on the project tree, and guarded a double seal. A second review found the queue keyed on the session's directory while `worktrees.ensure` rebuilds the project tree. One tree split into a queue per subfolder, so a session started from a subdirectory waited on a queue nobody polls. Keys on the project worktree now. `step.ended` was the one event with no projector guard, and a step's writers share an owner token, so a late seal attempt could overwrite the end snapshot and file diff with a different instant. Also drops `toolPartOf`, dead since the point read, tightens a type guard that asserted an assistant message from an id match alone, and fixes the README where it described `realpath` resolution the code doesn't do. --- packages/core/src/session/message-updater.ts | 4 ++ packages/core/src/session/runner/llm.ts | 12 ++--- .../test/session-runner-model-call.test.ts | 51 +++++++++++++++++++ packages/temporal/README.md | 10 ++-- packages/temporal/src/executor.ts | 33 +++++++++--- 5 files changed, 90 insertions(+), 20 deletions(-) diff --git a/packages/core/src/session/message-updater.ts b/packages/core/src/session/message-updater.ts index 46118a89fe4b..de50688b748a 100644 --- a/packages/core/src/session/message-updater.ts +++ b/packages/core/src/session/message-updater.ts @@ -208,6 +208,10 @@ export function update(adapter: Adapter, event: SessionEvent.Event) { }, "session.next.step.ended": (event) => { return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { + // A step's writers share one owner token, so two seal attempts are both admitted. The + // tool cases guard on status; this one has to guard on the step already being closed, or + // a late attempt overwrites the end snapshot and file diff with a different instant. + if (draft.time.completed) return draft.time.completed = event.data.timestamp draft.finish = event.data.finish draft.cost = event.data.cost diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index b9c4645a8b1d..b2d1a4d3acb7 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -693,7 +693,9 @@ const layer = Layer.effect( const carried = input.assistantMessageID const target = (carried - ? context.findLast((m): m is SessionMessage.Assistant => m.id === carried) + ? context.findLast( + (m): m is SessionMessage.Assistant => m.type === "assistant" && m.id === carried, + ) : undefined) ?? inFlight ?? context.findLast( @@ -749,14 +751,6 @@ const layer = Layer.effect( ) }) - const toolPartOf = (messages: ReadonlyArray, callID: string) => { - for (const message of messages) { - if (message.type !== "assistant") continue - for (const part of message.content) - if (part.type === "tool" && part.id === callID) return part - } - return undefined - } const runToolCall = Effect.fn("SessionRunner.runToolCall")(function* (input: ToolCallInput) { const session = yield* getSession(input.sessionID) diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index d6c38588cf7c..e58c01b792ca 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -35,6 +35,8 @@ import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { SessionTable } from "@opencode-ai/core/session/sql" import { SessionStore } from "@opencode-ai/core/session/store" import { SessionMessage } from "@opencode-ai/core/session/message" +import { SessionEvent } from "@opencode-ai/core/session/event" +import { DateTime } from "effect" import { ConfigCompaction } from "@opencode-ai/core/config/compaction" import { SessionContextEpoch } from "@opencode-ai/core/session/context-epoch" import { createLLMEventPublisher } from "@opencode-ai/core/session/runner/publish-llm-event" @@ -770,3 +772,52 @@ describe("SessionRunner declined permission in a stepped turn", () => { }), ) }) + +// A step's writers share one owner token, so the fence does not separate two seal attempts. The +// projector is what holds the line, and step.ended was the one event in that window with no guard. +describe("SessionRunner duplicate seal", () => { + harness(callsTool).effect("keeps the first close when a late attempt lands", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + const model = yield* runner.runModelCall({ + sessionID, + step: 2, + promotion: undefined, + first: false, + force: false, + }) + if (model.kind !== "called" || !model.calls[0]) throw new Error("expected a deferred call") + yield* runner.runToolCall({ sessionID, call: model.calls[0], retry: false }) + const seal = { + sessionID, + step: model.step, + settlement: model.settlement, + assistantMessageID: model.assistantMessageID, + needsContinuation: model.needsContinuation, + } + yield* runner.sealStep(seal) + const closedAt = assistant(yield* store.context(sessionID)) + const first = String(closedAt?.type === "assistant" ? closedAt.time.completed : undefined) + + // A zombie attempt publishing its own Step.Ended under the same token is admitted by the + // fence, so the projection has to reject it. + const events = yield* EventV2.Service + yield* events.publish(SessionEvent.Step.Ended, { + sessionID, + timestamp: yield* DateTime.now, + assistantMessageID: SessionMessage.ID.make(model.assistantMessageID!), + finish: "stop", + cost: 0, + tokens: { input: 99, output: 99, reasoning: 0, cache: { read: 0, write: 0 } }, + }) + + const after = assistant(yield* store.context(sessionID)) + expect(String(after?.type === "assistant" ? after.time.completed : undefined)).toBe(first) + expect(after?.type === "assistant" ? after.tokens?.input : undefined).not.toBe(99) + }), + ) +}) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index b7c1a3027a12..1bbc81a80826 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -306,12 +306,14 @@ what makes any worker able to serve any session, and turning affinity on is choo Two consequences to plan for, both silent: - **A worker serves one tree.** In the default `role=both` deployment the embedded worker polls the - queue for the process directory, so every session in another directory has no poller. If you open - more than one project against one serve process, do not turn this on. + queue for the process directory, so a session in another project has no poller. Point + `OPENCODE_TEMPORAL_WORKTREE` at the project root, not at a subfolder, since the key is the project + worktree. - **Flipping the flag strands workflows already running.** A workflow keeps the task queue it started on for life, and its activities inherit it. Restarting workers with the flag changed - leaves in-flight sessions with nobody polling their queue; they retry rather than fail. Drain - before flipping, in either direction. + leaves in-flight sessions with nobody polling their queue. They do not fail; they stay `RUNNING` + forever, because the workflow task that would run their idle timer is never picked up either. + Drain before flipping, in either direction. Both processes log the queue they use (`pollQueue` on a worker, `worktree` on the client), so a mismatch shows up as two names in the logs rather than as a session that never runs. diff --git a/packages/temporal/src/executor.ts b/packages/temporal/src/executor.ts index 4cd267d16cf9..94987e603aa5 100644 --- a/packages/temporal/src/executor.ts +++ b/packages/temporal/src/executor.ts @@ -17,6 +17,9 @@ import { makeStepActivities, makeSteppedTurnActivities } from "./activities" import { makeDrains } from "./drain" import { makeL2Drains } from "./l2-drain" import { queueForWorktree } from "./queue" +import { Database } from "@opencode-ai/core/database/database" +import { ProjectTable } from "@opencode-ai/core/project/sql" +import { eq } from "drizzle-orm" import { WorktreeMaterializer } from "@opencode-ai/core/session/execution/worktree" import { toRunError } from "@opencode-ai/core/session/execution/run-error-codec" import * as WF from "./workflow" @@ -50,6 +53,7 @@ const layer = Layer.effect( Effect.gen(function* () { const store = yield* SessionStore.Service const locations = yield* LocationServiceMap.Service + const db = (yield* Database.Service).db // The app context the local drain runs in: providing it, then the per-location layer, supplies // SessionRunner and all of its dependencies. const ctx = yield* Effect.context() @@ -69,14 +73,23 @@ const layer = Layer.effect( // Which queue a worker polls. With affinity off this is the one shared queue and any worker can // draw any session, rebuilding the tree if it has to. const POLL_QUEUE = AFFINITY ? queueForWorktree(TASK_QUEUE, SERVED_WORKTREE) : TASK_QUEUE - // Which queue a session's workflow runs on. Reads the session because the tree that has to be - // present is its location, not the project root: two sessions under one project can sit in - // different directories. + // Which queue a session's workflow runs on. Keyed on the PROJECT worktree, not the session's + // directory: `worktrees.ensure` rebuilds the project tree, so keying on the directory a session + // happened to start in would split one physical tree across a queue per subdirectory, and a + // session started from a subfolder would wait on a queue nobody polls. const queueFor = (id: SessionSchema.ID) => AFFINITY - ? Effect.map(store.get(id), (session) => - session ? queueForWorktree(TASK_QUEUE, session.location.directory) : TASK_QUEUE, - ) + ? Effect.gen(function* () { + const session = yield* store.get(id) + if (!session) return TASK_QUEUE + const project = yield* db + .select({ worktree: ProjectTable.worktree }) + .from(ProjectTable) + .where(eq(ProjectTable.id, session.projectID)) + .get() + .pipe(Effect.orDie) + return project ? queueForWorktree(TASK_QUEUE, project.worktree) : TASK_QUEUE + }) : Effect.succeed(TASK_QUEUE) const events = yield* EventV2.Service const worktrees = yield* WorktreeMaterializer.Service @@ -282,5 +295,11 @@ const layer = Layer.effect( export const node = makeGlobalNode({ service: SessionExecution.Service, layer, - deps: [SessionStore.node, LocationServiceMap.node, EventV2.node, WorktreeMaterializer.node], + deps: [ + SessionStore.node, + LocationServiceMap.node, + EventV2.node, + WorktreeMaterializer.node, + Database.node, + ], }) From dcd6f4adbb5b961259235c6d9b8dde5534a66e27 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 19:02:37 -0700 Subject: [PATCH 22/33] Told the model why a dispatched tool call failed. A tool whose output could not be stored had already run, so retrying repeated the side effect and the seal finally reported the call as interrupted. The whole-step path reports the real reason and keeps going. --- packages/core/src/session/runner/index.ts | 4 +- packages/core/src/session/runner/llm.ts | 22 ++++++++++ .../test/session-runner-model-call.test.ts | 43 ++++++++++++++++++- 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index c0e821ec759a..920b50f8b03c 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -73,9 +73,11 @@ export interface ToolCallInput { /** How a dispatched call ended. * - `settled`: the tool ran and its result is durable. * - `already-settled`: the log already had a result, so nothing ran. The at-least-once case. + * - `failed`: the tool ran and could not be settled. The reason reaches the model, and the call is + * closed, because a repeat would not fix it. * - `unknown`: a retry of a tool that does not declare itself repeatable. Reported to the model as * an unknown outcome rather than run a second time. */ -export type ToolCallOutcome = "settled" | "already-settled" | "unknown" +export type ToolCallOutcome = "settled" | "already-settled" | "failed" | "unknown" export interface ToolCallResult { readonly outcome: ToolCallOutcome diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index b2d1a4d3acb7..973ebc96f2e6 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -813,7 +813,29 @@ const layer = Layer.effect( Effect.catchCause((cause) => isUserDeclined(cause) ? Effect.interrupt : Effect.failCause(cause), ), + // The tool itself ran: what failed is storing its output. Letting that fail the dispatch + // would retry a side effect that already happened and finally tell the model the call was + // interrupted, which is not what happened to it. The whole-step path reports the same + // reason the same way. + Effect.catch((error) => Effect.succeed({ failure: error })), ) + if ("failure" in settlement) { + const reason = + settlement.failure instanceof Error + ? settlement.failure.message + : String(settlement.failure) + yield* Effect.uninterruptible( + events.publish(SessionEvent.Tool.Failed, { + sessionID: input.sessionID, + timestamp: yield* DateTime.now, + assistantMessageID, + callID: input.call.id, + error: { type: "unknown", message: `Tool execution failed: ${reason}` }, + provider: { executed: false }, + }), + ) + return { outcome: "failed" } as ToolCallResult + } // The tool has run by here, so losing the result to an interrupt would hide a // side effect that already happened. yield* Effect.uninterruptible(emitToolResult(events, { diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index e58c01b792ca..5d4fa0627410 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -129,7 +129,21 @@ const textOnly: LLMClientShape["stream"] = () => LLMEvent.stepFinish({ index: 0, reason: "stop" }), ]) -const harness = (stream: LLMClientShape["stream"]) => +// A store that cannot keep what a tool produced. The tool itself still ran by then, which is the +// case a dispatch has to report honestly rather than as a call that never finished. +const failingOutputStore = Layer.mock(ToolOutputStore.Service, { + bound: () => + Effect.fail( + new ToolOutputStore.StorageError({ operation: "write", cause: new Error("disk full") }), + ), +}) + +const harness = ( + stream: LLMClientShape["stream"], + outputStore: + | typeof ToolOutputStore.nodeWithoutConfig + | Layer.Layer = ToolOutputStore.nodeWithoutConfig, +) => testEffect( AppNodeBuilder.build( LayerNode.group([ @@ -151,7 +165,7 @@ const harness = (stream: LLMClientShape["stream"]) => [ [LayerNodePlatform.llmClient, mockClient(stream)], [PermissionV2.node, permission], - [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [ToolOutputStore.node, outputStore], [SessionRunnerModel.node, models], [SystemContextRegistry.node, systemContext], [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })], @@ -406,6 +420,31 @@ describe("SessionRunner tool dispatch", () => { }), ) + harness(callsTool, failingOutputStore).effect( + "tells the model why a dispatched call failed", + () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const call = yield* deferOneCall + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + + const result = yield* runner.runToolCall({ sessionID, call, retry: false }) + + // The tool ran and only its output was lost. Letting that fail the dispatch would repeat + // the write on the next attempt, and the step would finally close the call as interrupted: + // a reason the model cannot act on, and not what happened to it. + expect(result.outcome).toBe("failed") + expect(ran.write).toBe(1) + const part = toolPart(yield* store.context(sessionID), "call_probe") + const failure = + part?.type === "tool" && part.state.status === "error" ? part.state.error : undefined + expect(failure?.message).toContain("disk full") + }), + ) + harness(callsIdempotentTool).effect( "re-runs a retried call that declares itself repeatable", () => From df61619258953ae0e8e86cc9c73093421f9465a5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 19:08:46 -0700 Subject: [PATCH 23/33] Read a started tool call from the log, not the attempt number. The attempt number counts every way a dispatch can die, including the ones that never reached the tool, so a call nobody had touched was reported to the model as an unknown outcome. Publishing Tool.Called at dispatch makes a running call mean exactly that, and a fenced dispatch now dies before the tool runs rather than after. --- packages/core/src/session/runner/index.ts | 13 +-- packages/core/src/session/runner/llm.ts | 29 ++++- .../src/session/runner/publish-llm-event.ts | 11 +- .../test/session-runner-model-call.test.ts | 104 +++++++++++++----- packages/core/test/step-overlap-bench.test.ts | 2 +- packages/temporal/src/activities.ts | 8 +- packages/temporal/src/l2-drain.ts | 9 +- 7 files changed, 118 insertions(+), 58 deletions(-) diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index 920b50f8b03c..5801424317d1 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -36,9 +36,9 @@ export interface StepResult { readonly promotion: SessionInput.Delivery | undefined } -/** A tool call the provider asked for, recorded as Tool.Called but not run, handed to the caller to - * dispatch. Every id comes from the provider or the publisher and is carried, never regenerated: a - * second run of the same step would mint different ones and the results would not match the log. */ +/** A tool call the provider asked for, recorded but not run, handed to the caller to dispatch. + * Every id comes from the provider or the publisher and is carried, never regenerated: a second run + * of the same step would mint different ones and the results would not match the log. */ export interface DeferredToolCall { readonly id: string readonly name: string @@ -65,9 +65,6 @@ export interface SealStepInput { export interface ToolCallInput { readonly sessionID: SessionSchema.ID readonly call: DeferredToolCall - /** True when this call is being dispatched again after a crash or a timeout, so its side effect - * may already have happened. The dispatcher knows this; the log cannot tell us. */ - readonly retry: boolean } /** How a dispatched call ended. @@ -75,8 +72,8 @@ export interface ToolCallInput { * - `already-settled`: the log already had a result, so nothing ran. The at-least-once case. * - `failed`: the tool ran and could not be settled. The reason reaches the model, and the call is * closed, because a repeat would not fix it. - * - `unknown`: a retry of a tool that does not declare itself repeatable. Reported to the model as - * an unknown outcome rather than run a second time. */ + * - `unknown`: a second dispatch of a tool that does not declare itself repeatable. Reported to the + * model as an unknown outcome rather than run again. */ export type ToolCallOutcome = "settled" | "already-settled" | "failed" | "unknown" export interface ToolCallResult { diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 973ebc96f2e6..944e7e851d6e 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -44,7 +44,7 @@ import { Service, } from "./index" import { SessionRunnerModel } from "./model" -import { createLLMEventPublisher, emitToolResult } from "./publish-llm-event" +import { createLLMEventPublisher, emitToolResult, record } from "./publish-llm-event" import { toLLMMessages } from "./to-llm-message" import { MAX_STEPS_PROMPT } from "./max-steps" import { DEFAULT_MAX_STEPS, REPEAT_LIMIT, REPEATED_CALLS_PROMPT, trailingIdenticalToolSteps } from "./loop-guard" @@ -281,6 +281,7 @@ const layer = Layer.effect( ...(session.model?.variant === undefined ? {} : { variant: session.model.variant }), }, snapshot: startSnapshot, + deferCalls: deferTools, }) const withPublication = Semaphore.makeUnsafe(1).withPermit const publish = (event: LLMEvent, outputPaths: ReadonlyArray = []) => @@ -769,7 +770,7 @@ const layer = Layer.effect( item.type === "tool" && item.id === input.call.id, ) : undefined - // The call has to be in the log already: the attempt that produced it published Tool.Called + // The call has to be in the log already: the attempt that produced it recorded its input // before handing it over. Missing means the log moved under us (a fence), and running a tool // whose call is not recorded would leave an orphan result. if (!part || part.type !== "tool") @@ -781,10 +782,13 @@ const layer = Layer.effect( return { outcome: "already-settled" } as ToolCallResult const agent = yield* agents.select(session.agent) const materialization = yield* tools.materialize(agent.info?.permissions) - // A retry cannot know whether the side effect happened, so only a tool that declares itself - // repeatable is run again. The rest are reported unknown and the model decides, because - // re-running the `git push` that may already have landed is the worse failure. - if (input.retry && !materialization.idempotent(input.call.name)) { + // `running` means a dispatch already published Tool.Called and was about to run the tool, so + // the side effect may have happened and nothing that reads the log afterwards can tell. Only + // a tool that declares itself repeatable runs again. The rest are reported unknown and the + // model decides, because re-running the `git push` that may already have landed is the worse + // failure. The attempt number would say the same thing far less precisely: it counts every + // way a dispatch can die, including the ones that never reached the tool. + if (part.state.status === "running" && !materialization.idempotent(input.call.name)) { yield* events.publish(SessionEvent.Tool.Failed, { sessionID: input.sessionID, timestamp: yield* DateTime.now, @@ -795,6 +799,19 @@ const layer = Layer.effect( }) return { outcome: "unknown" } as ToolCallResult } + // The durable record that this call is being run, published before the tool can do anything. + // It is also the last point a fenced dispatch dies at: under a superseded owner this publish + // fails and the tool never runs, instead of running and losing its result. + yield* events.publish(SessionEvent.Tool.Called, { + sessionID: input.sessionID, + timestamp: yield* DateTime.now, + assistantMessageID, + callID: input.call.id, + tool: input.call.name, + input: record(input.call.input), + // Deferred calls are never provider-executed: those are filtered out before the hand-off. + provider: { executed: false }, + }) const settlement = yield* materialization .settle({ sessionID: input.sessionID, diff --git a/packages/core/src/session/runner/publish-llm-event.ts b/packages/core/src/session/runner/publish-llm-event.ts index 3f8d08162e75..d634b86e8771 100644 --- a/packages/core/src/session/runner/publish-llm-event.ts +++ b/packages/core/src/session/runner/publish-llm-event.ts @@ -11,6 +11,10 @@ type Input = { readonly agent: string readonly model: ModelV2.Ref readonly snapshot?: string + /** Record the calls the provider asked for, but leave Tool.Called to whoever dispatches them. + * The log then says which calls were only asked for and which one a process had started, which + * is the difference between re-running a side effect and reporting it as unknown. */ + readonly deferCalls?: boolean } const safe = (value: number | undefined) => Math.max(0, Number.isFinite(value) ? (value ?? 0) : 0) @@ -34,7 +38,9 @@ export interface StepSettlement { readonly tokens: ReturnType } -const record = (value: unknown): Record => +/** The shape a tool call's input is recorded in. Exported so a dispatcher publishing Tool.Called + * for a deferred call records it exactly as the streaming path would. */ +export const record = (value: unknown): Record => typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) : { value } const message = (value: unknown) => { @@ -371,6 +377,9 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) tool.called = true tool.providerExecuted = event.providerExecuted === true tool.providerMetadata = event.providerMetadata + // A provider-executed call has already run, so it is never deferred and is recorded here + // whatever the caller asked for. + if (input.deferCalls && !tool.providerExecuted) return yield* events.publish(SessionEvent.Tool.Called, { sessionID: input.sessionID, timestamp: yield* timestamp, diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index 5d4fa0627410..ca1c2ff4c5bb 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -3,7 +3,7 @@ // asked for, and stops. The caller dispatches each call as its own unit of work and seals the step // afterwards, which is what puts the model-to-tools loop in a durable executor rather than inside a // single activity. These tests pin the three properties that split depends on: -// - the call is durable (Tool.Called) but its side effect has NOT run +// - the call is durable but not started, so its side effect has NOT run // - the provider-minted callID and the publisher's assistantMessageID are handed back, never // regenerated, so the dispatcher's result can be matched to the recorded call // - the step is left open (no Step.Ended), because the tools have not run yet @@ -109,6 +109,20 @@ const callsIdempotentTool: LLMClientShape["stream"] = () => LLMEvent.toolCall({ id: "call_probe", name: "probe_read", input: {} }), LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), ]) +// Calls the tools that die with their side effect in flight, which is what a worker crash leaves +// behind for the next dispatch to read. +const callsCrashingTool: LLMClientShape["stream"] = () => + Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call_probe", name: "probe_crashes", input: {} }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + ]) +const callsCrashingIdempotentTool: LLMClientShape["stream"] = () => + Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call_probe", name: "probe_crashes_read", input: {} }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + ]) // A provider turn that publishes nothing at all: no text, no reasoning, no tool call. The publisher // mints the assistant message lazily on first content, so after this stream there is no message in // the log for a seal to find. The whole-step path survives it because Step.Ended mints one on the @@ -202,9 +216,10 @@ const seedSession = Effect.gen(function* () { .pipe(Effect.orDie) }) -// Two probes that count their own executions, so "recorded but not run" and "not run twice" are -// checked against the tools themselves rather than only against the projection. `probe_read` -// declares itself repeatable; `probe_write` does not, which is what decides a retry's behaviour. +// Probes that count their own executions, so "recorded but not run" and "not run twice" are +// checked against the tools themselves rather than only against the projection. The read probes +// declare themselves repeatable; the write probes do not, which is what decides whether a second +// dispatch runs the tool again. const registerProbes = (ran: { write: number; read: number }) => Effect.gen(function* () { yield* (yield* ApplicationTools.Service).register({ @@ -237,6 +252,32 @@ const registerProbes = (ran: { write: number; read: number }) => return "read" }), }), + // Dying with the side effect already done is the case the log has to survive: the call is + // left in flight and no later reader can say whether the write landed. + probe_crashes: Tool.make({ + description: "crashing write probe", + input: Schema.Struct({}), + output: Schema.String, + execute: () => + Effect.gen(function* () { + ran.write += 1 + return yield* Effect.die(new Error("worker died")) + }), + }), + // The repeatable one, dying only on its first run so a second dispatch has a result to + // produce. + probe_crashes_read: Tool.make({ + description: "crashing read probe", + idempotent: true, + input: Schema.Struct({}), + output: Schema.String, + execute: () => + Effect.gen(function* () { + ran.read += 1 + if (ran.read === 1) return yield* Effect.die(new Error("worker died")) + return "read" + }), + }), }) }) @@ -284,8 +325,11 @@ describe("SessionRunner model-only attempt", () => { const context = yield* store.context(sessionID) const part = toolPart(context, "call_probe") - // Durably recorded and left mid-flight: this is what a dispatcher picks up. - expect(part?.type === "tool" ? part.state.status : undefined).toBe("running") + // Durably recorded and not started: whoever dispatches the call publishes Tool.Called, so + // `running` in the log means a process was about to run the tool. Recording it here instead + // would make every later dispatch of a crashed step report an unknown outcome for a tool + // nobody had touched. + expect(part?.type === "tool" ? part.state.status : undefined).toBe("pending") // The step stays open, so no Step.Ended and no completed assistant message. const message = assistant(context) expect(message?.type === "assistant" ? Boolean(message.time.completed) : true).toBe(false) @@ -346,9 +390,10 @@ describe("SessionRunner model-only attempt", () => { ) }) -// Dispatching one recorded call on its own. The policy under test is what happens on a retry, when -// the side effect may already have run and the log cannot say: repeatable tools run again, the rest -// are reported unknown so the model decides. Same rule the crash-resume path already follows. +// Dispatching one recorded call on its own. The policy under test is what happens when a dispatch +// has already started the tool and died, so the side effect may have run and nothing can say: +// repeatable tools run again, the rest are reported unknown so the model decides. Same rule the +// crash-resume path already follows, off the same evidence. describe("SessionRunner tool dispatch", () => { const deferOneCall = Effect.gen(function* () { const runner = yield* SessionRunner.Service @@ -372,7 +417,7 @@ describe("SessionRunner tool dispatch", () => { const runner = yield* SessionRunner.Service const store = yield* SessionStore.Service - const result = yield* runner.runToolCall({ sessionID, call, retry: false }) + const result = yield* runner.runToolCall({ sessionID, call }) expect(result.outcome).toBe("settled") expect(ran.write).toBe(1) @@ -389,17 +434,17 @@ describe("SessionRunner tool dispatch", () => { const call = yield* deferOneCall const runner = yield* SessionRunner.Service - yield* runner.runToolCall({ sessionID, call, retry: false }) + yield* runner.runToolCall({ sessionID, call }) // A duplicate dispatch: at-least-once delivery means this happens, and it must not re-run. - const second = yield* runner.runToolCall({ sessionID, call, retry: true }) + const second = yield* runner.runToolCall({ sessionID, call }) expect(second.outcome).toBe("already-settled") expect(ran.write).toBe(1) }), ) - harness(callsTool).effect( - "reports a retried side-effecting call as unknown instead of repeating it", + harness(callsCrashingTool).effect( + "reports a side-effecting call whose dispatch already started as unknown", () => Effect.gen(function* () { yield* seedSession @@ -409,12 +454,16 @@ describe("SessionRunner tool dispatch", () => { const runner = yield* SessionRunner.Service const store = yield* SessionStore.Service - // The first attempt died between dispatch and its result landing, so the log still - // shows the call in flight and nothing can say whether the write happened. - const result = yield* runner.runToolCall({ sessionID, call, retry: true }) + // A dispatch that died with the tool in flight. What it leaves in the log is the whole + // evidence a later one gets: the call recorded as running, and no result. + const crashed = yield* runner.runToolCall({ sessionID, call }).pipe(Effect.exit) + expect(Exit.isFailure(crashed)).toBe(true) + expect(ran.write).toBe(1) + + const result = yield* runner.runToolCall({ sessionID, call }) expect(result.outcome).toBe("unknown") - expect(ran.write).toBe(0) + expect(ran.write).toBe(1) const part = toolPart(yield* store.context(sessionID), "call_probe") expect(part?.type === "tool" ? part.state.status : undefined).toBe("error") }), @@ -431,7 +480,7 @@ describe("SessionRunner tool dispatch", () => { const runner = yield* SessionRunner.Service const store = yield* SessionStore.Service - const result = yield* runner.runToolCall({ sessionID, call, retry: false }) + const result = yield* runner.runToolCall({ sessionID, call }) // The tool ran and only its output was lost. Letting that fail the dispatch would repeat // the write on the next attempt, and the step would finally close the call as interrupted: @@ -445,8 +494,8 @@ describe("SessionRunner tool dispatch", () => { }), ) - harness(callsIdempotentTool).effect( - "re-runs a retried call that declares itself repeatable", + harness(callsCrashingIdempotentTool).effect( + "re-runs a started call that declares itself repeatable", () => Effect.gen(function* () { yield* seedSession @@ -456,10 +505,11 @@ describe("SessionRunner tool dispatch", () => { const runner = yield* SessionRunner.Service const store = yield* SessionStore.Service - const result = yield* runner.runToolCall({ sessionID, call, retry: true }) + yield* runner.runToolCall({ sessionID, call }).pipe(Effect.exit) + const result = yield* runner.runToolCall({ sessionID, call }) expect(result.outcome).toBe("settled") - expect(ran.read).toBe(1) + expect(ran.read).toBe(2) const part = toolPart(yield* store.context(sessionID), "call_probe") expect(part?.type === "tool" ? part.state.status : undefined).toBe("completed") }), @@ -502,7 +552,7 @@ describe("SessionRunner step seal", () => { const model = yield* deferOneCall const runner = yield* SessionRunner.Service const store = yield* SessionStore.Service - yield* runner.runToolCall({ sessionID, call: model.calls[0]!, retry: false }) + yield* runner.runToolCall({ sessionID, call: model.calls[0]! }) const result = yield* runner.sealStep({ sessionID, step: 2, settlement: model.settlement }) @@ -537,7 +587,7 @@ describe("SessionRunner step seal", () => { yield* registerProbes(ran) const model = yield* deferOneCall const runner = yield* SessionRunner.Service - yield* runner.runToolCall({ sessionID, call: model.calls[0]!, retry: false }) + yield* runner.runToolCall({ sessionID, call: model.calls[0]! }) const first = yield* runner.sealStep({ sessionID, step: 2, settlement: model.settlement }) // A seal that published Step.Ended and then died is retried. The loop decision has to survive @@ -800,7 +850,7 @@ describe("SessionRunner declined permission in a stepped turn", () => { if (model.kind !== "called" || !model.calls[0]) throw new Error("expected a deferred call") const exit = yield* runner - .runToolCall({ sessionID, call: model.calls[0], retry: false }) + .runToolCall({ sessionID, call: model.calls[0] }) .pipe(Effect.exit) // An interrupt is what the activity boundary turns into a non-retryable halt. A plain failure @@ -830,7 +880,7 @@ describe("SessionRunner duplicate seal", () => { force: false, }) if (model.kind !== "called" || !model.calls[0]) throw new Error("expected a deferred call") - yield* runner.runToolCall({ sessionID, call: model.calls[0], retry: false }) + yield* runner.runToolCall({ sessionID, call: model.calls[0] }) const seal = { sessionID, step: model.step, diff --git a/packages/core/test/step-overlap-bench.test.ts b/packages/core/test/step-overlap-bench.test.ts index c8bdc4ee93ec..f84fcc8fcdce 100644 --- a/packages/core/test/step-overlap-bench.test.ts +++ b/packages/core/test/step-overlap-bench.test.ts @@ -207,7 +207,7 @@ describe.skipIf(!process.env.OPENCODE_OVERLAP_BENCH)("step overlap cost", () => const result = yield* runner.runModelCall({ sessionID: split, ...step }) if (result.kind !== "called") return for (const call of result.calls) - yield* runner.runToolCall({ sessionID: split, call, retry: false }) + yield* runner.runToolCall({ sessionID: split, call }) yield* runner.sealStep({ sessionID: split, step: result.step, diff --git a/packages/temporal/src/activities.ts b/packages/temporal/src/activities.ts index 3e21f5bb5bc2..01ee68140f1e 100644 --- a/packages/temporal/src/activities.ts +++ b/packages/temporal/src/activities.ts @@ -101,13 +101,7 @@ export function makeSteppedTurnActivities(drains: { ) }, async runToolCall(input) { - // Whether the side effect may already have happened is the activity's own knowledge, not the - // workflow's: a second attempt of THIS call is a re-run of a tool whose result never landed. - // The workflow could not compute this without reading non-deterministic state. - const retry = Context.current().info.attempt > 1 - return beating(() => - drains.toolCallDrain({ ...input, retry }, Context.current().cancellationSignal), - ) + return beating(() => drains.toolCallDrain(input, Context.current().cancellationSignal)) }, async sealStep(input) { return beating(() => drains.sealDrain(input, Context.current().cancellationSignal)) diff --git a/packages/temporal/src/l2-drain.ts b/packages/temporal/src/l2-drain.ts index 2a43766506df..2439eaca1b64 100644 --- a/packages/temporal/src/l2-drain.ts +++ b/packages/temporal/src/l2-drain.ts @@ -45,9 +45,6 @@ export interface ToolCallDrainInput { readonly sessionID: string readonly call: DeferredToolCall readonly owner: string - /** Set by the executor from the activity attempt, not by the workflow: only the dispatcher knows - * whether this call already had a run whose result never landed. */ - readonly retry?: boolean } export interface ToolCallDrainResult { @@ -151,11 +148,7 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra input.sessionID, signal, inSession(input.sessionID, input.owner, false, (runner, session) => - runner.runToolCall({ - sessionID: session.id, - call: input.call, - retry: input.retry === true, - }), + runner.runToolCall({ sessionID: session.id, call: input.call }), ).pipe(Effect.map((result) => result ?? { outcome: "already-settled" as const })), ) From cc6966a77c65165421ac1e3fe9715474ac723336 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 19:20:38 -0700 Subject: [PATCH 24/33] Brought a worker's stale worktree forward before it drains. A tree was only rebuilt when it was missing, so a worker that ran an earlier step served files from that step. The store's newest capture is the target now, and a host-local note says whether this tree is behind it or holds work nothing else has. --- .../core/src/session/execution/worktree.ts | 108 ++++++++++++++---- packages/core/src/snapshot-sync.ts | 5 + packages/core/src/snapshot/tip.ts | 35 ++++++ .../core/test/worktree-materialize.test.ts | 80 ++++++++++++- 4 files changed, 204 insertions(+), 24 deletions(-) create mode 100644 packages/core/src/snapshot/tip.ts diff --git a/packages/core/src/session/execution/worktree.ts b/packages/core/src/session/execution/worktree.ts index 060af00577c7..9d3532d95b8d 100644 --- a/packages/core/src/session/execution/worktree.ts +++ b/packages/core/src/session/execution/worktree.ts @@ -1,31 +1,39 @@ export * as WorktreeMaterializer from "./worktree" -// Rebuilds a missing project worktree from the shared store before a drain runs. This closes the -// host-local gap in cross-host resume: file tools need the tree, and a fresh worker does not have -// it. The capture side (snapshot-sync.ts) ships each snapshot as an incremental git pack; this -// side indexes every pack for the worktree into a fresh repo and checks out the newest tree. -// Ignored files and dependencies are not captured, so a bootstrap step (install, build) stays the -// project's own concern. +// Brings the project worktree to the newest state in the shared store before a drain runs. This +// closes the host-local gap in cross-host resume: file tools need the tree, and a worker that never +// ran an earlier step either has no tree at all or has one from the step it last ran. The capture +// side (snapshot-sync.ts) ships each snapshot as an incremental git pack; this side indexes every +// pack for the worktree and checks out the newest tree. Ignored files and dependencies are not +// captured, so a bootstrap step (install, build) stays the project's own concern. +// +// A tree is only ever moved forward, and only when this host's own note (snapshot/tip.ts) says it +// is behind the store. What that leaves open: two workers running tools of the SAME step on two +// hosts still cannot see each other's writes, because those are not captured until the step is +// sealed. One worker per worktree is what makes a step's tools share a tree. import { rm, writeFile } from "node:fs/promises" import path from "path" import { Cause, Context, Effect, Layer } from "effect" import { ChildProcess } from "effect/unstable/process" -import { asc, desc, eq } from "drizzle-orm" +import { and, asc, desc, eq } from "drizzle-orm" import { Database } from "../../database/database" import { makeGlobalNode } from "../../effect/app-node" import { KeyedMutex } from "../../effect/keyed-mutex" import { FSUtil } from "../../fs-util" import { Git } from "../../git" +import { Global } from "../../global" import { AppProcess } from "../../process" import { AbsolutePath } from "../../schema" import { SnapshotPackTable } from "../../snapshot/sql" +import { readWorktreeTip, writeWorktreeTip } from "../../snapshot/tip" export interface Interface { /** - * Make sure the session's directory exists, rebuilding its worktree from stored snapshot packs - * when it does not. A directory with no stored packs, or one whose worktree root already - * exists, is left alone. Never fails the caller. + * Make sure the session's directory holds the newest state the shared store has for it, + * rebuilding its worktree from stored snapshot packs when it is missing or behind. A directory + * with no stored packs, and a tree this host has neither built nor captured from, are left + * alone. Never fails the caller. */ readonly ensure: (directory: string) => Effect.Effect } @@ -34,11 +42,15 @@ export class Service extends Context.Service()( "@opencode/v2/WorktreeMaterializer", ) {} +// HEAD of a rebuilt tree, which doubles as the mark that says the tree is ours to move. +const RESTORED = "refs/heads/opencode-restore" + const layer = Layer.effect( Service, Effect.gen(function* () { const fs = yield* FSUtil.Service const git = yield* Git.Service + const global = yield* Global.Service const proc = yield* AppProcess.Service const { db } = yield* Database.Service const locks = KeyedMutex.makeUnsafe() @@ -79,7 +91,7 @@ const layer = Layer.effect( .run( ChildProcess.make( "git", - ["--git-dir", repository.gitDirectory, "update-ref", "refs/heads/opencode-restore", tip.id], + ["--git-dir", repository.gitDirectory, "update-ref", RESTORED, tip.id], { cwd: worktree, extendEnv: true }, ), ) @@ -88,11 +100,12 @@ const layer = Layer.effect( .run( ChildProcess.make( "git", - ["--git-dir", repository.gitDirectory, "symbolic-ref", "HEAD", "refs/heads/opencode-restore"], + ["--git-dir", repository.gitDirectory, "symbolic-ref", "HEAD", RESTORED], { cwd: worktree, extendEnv: true }, ), ) .pipe(Effect.ignore) + yield* writeWorktreeTip(global.data, tip.worktree, tip.tree) yield* Effect.logInfo("materialized worktree from snapshot packs", { worktree: tip.worktree, packs: rows.length, @@ -100,9 +113,51 @@ const layer = Layer.effect( }) }) + // Whether the tree on this host is older than what the store holds. No note means the tree was + // neither built from packs nor captured from here, so it belongs to whoever put it there. A + // note the store has never seen means a capture that never shipped, so this host holds work + // nothing else has and must not be moved back to an older tree. + const behind = Effect.fnUntraced(function* (tip: typeof SnapshotPackTable.$inferSelect) { + const held = yield* readWorktreeTip(global.data, tip.worktree) + if (!held || held === tip.tree) return false + const shipped = yield* db + .select({ time: SnapshotPackTable.time_created }) + .from(SnapshotPackTable) + .where(and(eq(SnapshotPackTable.worktree, tip.worktree), eq(SnapshotPackTable.tree, held))) + .orderBy(desc(SnapshotPackTable.time_created)) + .limit(1) + .get() + .pipe(Effect.orDie) + return shipped !== undefined && shipped.time < tip.time_created + }) + + // Whether this tree is one we built from packs. A checkout the host already had is somebody's + // working copy: reading its captures is fine, but checking a stored tree out over it would + // rewrite files and HEAD under whoever owns it. + const rebuilt = (worktree: string) => + proc + .run( + ChildProcess.make( + "git", + [ + "--git-dir", + path.join(worktree, ".git"), + "rev-parse", + "--verify", + "--quiet", + RESTORED, + ], + { cwd: worktree, extendEnv: true }, + ), + ) + .pipe( + Effect.map((result) => result.exitCode === 0), + Effect.catchCause(() => Effect.succeed(false)), + ) + const ensure = Effect.fn("WorktreeMaterializer.ensure")(function* (directory: string) { - if (yield* fs.existsSafe(directory)) return - // The newest capture whose session ran in this directory decides which worktree to rebuild. + // The newest capture whose session ran in this directory decides which worktree to rebuild, + // and which state a tree that is already here has to be brought to. const tip = yield* db .select() .from(SnapshotPackTable) @@ -112,18 +167,31 @@ const layer = Layer.effect( .get() .pipe(Effect.orDie) if (!tip) return + const present = yield* fs.existsSafe(tip.worktree) + if (present) { + if (!(yield* behind(tip))) return + if (!(yield* rebuilt(tip.worktree))) { + yield* Effect.logWarning("worktree is behind the store and was not built from it", { + worktree: tip.worktree, + tree: tip.tree, + }) + return + } + } yield* locks.withLock(tip.worktree)( Effect.gen(function* () { - // Re-check inside the lock (a concurrent drain may have rebuilt it), and never touch a - // worktree root that already exists: something else owns that tree. - if (yield* fs.existsSafe(tip.worktree)) return + // Re-check inside the lock: a concurrent drain may have done this already. + if ((yield* fs.existsSafe(tip.worktree)) && !(yield* behind(tip))) return yield* materialize(tip).pipe( Effect.catchCauseIf( (cause) => !Cause.hasInterrupts(cause), (cause) => Effect.gen(function* () { - // A half-built tree would pass the exists check forever; remove what we created. - yield* Effect.promise(() => rm(tip.worktree, { recursive: true, force: true })) + // A half-built tree would pass the exists check forever, so what we created is + // removed. A tree that was already here is not ours to remove: a failed refresh + // leaves it as stale as it was. + if (!present) + yield* Effect.promise(() => rm(tip.worktree, { recursive: true, force: true })) yield* Effect.logWarning("failed to materialize worktree", { worktree: tip.worktree, cause, @@ -142,5 +210,5 @@ const layer = Layer.effect( export const node = makeGlobalNode({ service: Service, layer, - deps: [Database.node, FSUtil.node, Git.node, AppProcess.node], + deps: [Database.node, FSUtil.node, Git.node, Global.node, AppProcess.node], }) diff --git a/packages/core/src/snapshot-sync.ts b/packages/core/src/snapshot-sync.ts index edab2106d124..ede416f8300a 100644 --- a/packages/core/src/snapshot-sync.ts +++ b/packages/core/src/snapshot-sync.ts @@ -21,6 +21,7 @@ import { AppProcess } from "./process" import { AbsolutePath } from "./schema" import type { Snapshot } from "./snapshot" import { SnapshotPackTable } from "./snapshot/sql" +import { writeWorktreeTip } from "./snapshot/tip" import { Hash } from "./util/hash" export interface Interface { @@ -62,6 +63,10 @@ const layer = Layer.effect( ) const push = Effect.fn("SnapshotSync.push")(function* (tree: Snapshot.ID) { + // Noted before the packing, which is best-effort: what this host holds is true whether or not + // the pack reaches the store, and a note left behind would let a later drain check out an + // older tree over work only this host has. + if (source) yield* writeWorktreeTip(global.data, worktree, tree) yield* Effect.gen(function* () { if (!source) return const latest = yield* db diff --git a/packages/core/src/snapshot/tip.ts b/packages/core/src/snapshot/tip.ts new file mode 100644 index 000000000000..06e563a28eac --- /dev/null +++ b/packages/core/src/snapshot/tip.ts @@ -0,0 +1,35 @@ +// The state a host's worktree is known to hold, kept in that host's own data directory. +// +// The shared store says what the newest captured tree is; it cannot say whether THIS host has it. +// The git objects cannot say either: a capture writes its tree into the side snapshot repository +// while a materialization indexes packs into the project repository, so "the tree is present" has +// two different answers depending on which host produced it. A host-local note is the one place +// both paths can agree on. +// +// It is written whenever this host captures (snapshot-sync) or rebuilds (execution/worktree), and +// read to answer one question: is this tree behind the store, or ahead of it? Being wrong towards +// "ahead" only leaves a stale tree; being wrong towards "behind" checks out over work nothing else +// holds, so an unreadable or missing note always means "leave the tree alone". + +import { mkdir, readFile, writeFile } from "node:fs/promises" +import path from "path" +import { Effect } from "effect" +import { Hash } from "../util/hash" + +const noteFile = (data: string, worktree: string) => + path.join(data, "worktree-tip", `${Hash.fast(worktree)}.tree`) + +export const readWorktreeTip = (data: string, worktree: string) => + Effect.promise(() => + readFile(noteFile(data, worktree), "utf8").then( + (text) => text.trim() || undefined, + () => undefined, + ), + ) + +export const writeWorktreeTip = (data: string, worktree: string, tree: string) => + Effect.promise(async () => { + const file = noteFile(data, worktree) + await mkdir(path.dirname(file), { recursive: true }).catch(() => {}) + await writeFile(file, tree).catch(() => {}) + }) diff --git a/packages/core/test/worktree-materialize.test.ts b/packages/core/test/worktree-materialize.test.ts index 55274f129acd..beb0eba19186 100644 --- a/packages/core/test/worktree-materialize.test.ts +++ b/packages/core/test/worktree-materialize.test.ts @@ -33,9 +33,14 @@ const captureStack = (file: string, worktree: string, data: string) => [Global.node, Layer.succeed(Global.Service, Global.make({ data }))], ]) -// The resuming host: only the shared store, no location, no snapshot repo, no worktree. -const materializeStack = (file: string) => - AppNodeBuilder.build(WorktreeMaterializer.node, [[Database.node, Database.layerFromPath(file)]]) +// The resuming host: the shared store and its own data directory, no location, no snapshot repo, +// no worktree. The data directory is what makes two of these independent hosts: it holds the note +// saying which state that host's tree is at. +const materializeStack = (file: string, data: string) => + AppNodeBuilder.build(WorktreeMaterializer.node, [ + [Database.node, Database.layerFromPath(file)], + [Global.node, Layer.succeed(Global.Service, Global.make({ data }))], + ]) describe("WorktreeMaterializer", () => { it.live("rebuilds a deleted worktree from shared-store packs, incremental chain included", () => @@ -80,7 +85,7 @@ describe("WorktreeMaterializer", () => { // The fresh host: the tree is gone, only the shared store remains. yield* Effect.promise(() => rm(worktree, { recursive: true, force: true })) - const B = yield* Layer.build(materializeStack(file)) + const B = yield* Layer.build(materializeStack(file, path.join(root, "host-b-data"))) yield* WorktreeMaterializer.Service.use((w) => w.ensure(worktree)).pipe(Effect.provide(B)) const [tracked, untracked, extra] = yield* Effect.promise(() => @@ -104,6 +109,73 @@ describe("WorktreeMaterializer", () => { }), ) + it.live("moves a rebuilt tree forward, and leaves a tree it did not build alone", () => + Effect.gen(function* () { + const tmp = yield* Effect.promise(() => tmpdir()) + const root = realpathSync(tmp.path) + const worktree = path.join(root, "project") + const file = path.join(root, "shared.db") + const tracked = path.join(worktree, "tracked.txt") + const content = () => Effect.promise(() => readFile(tracked, "utf8")) + const put = (text: string) => Effect.promise(() => writeFile(tracked, text)) + yield* Effect.promise(async () => { + await mkdir(worktree, { recursive: true }) + await $`git init -q ${worktree}`.quiet() + await $`git -C ${worktree} config user.email t@t`.quiet() + await $`git -C ${worktree} config user.name t`.quiet() + await writeFile(tracked, "v1\n") + await $`git -C ${worktree} add .`.quiet() + await $`git -C ${worktree} commit -qm seed`.quiet() + }) + + const A = yield* Layer.build(captureStack(file, worktree, path.join(root, "host-a-data"))) + const ship = Effect.gen(function* () { + const tree = yield* Snapshot.Service.use((s) => s.capture()) + if (!tree) throw new Error("expected a capture to produce a tree") + yield* SnapshotSync.Service.use((s) => s.push(tree)) + }).pipe(Effect.provide(A)) + yield* ship + + // A checkout this host already had is somebody's working copy: however far behind the store + // it is, checking a stored tree out over it would rewrite files under whoever owns them. + const C = yield* Layer.build(materializeStack(file, path.join(root, "host-c-data"))) + yield* put("mine\n") + yield* WorktreeMaterializer.Service.use((w) => w.ensure(worktree)).pipe(Effect.provide(C)) + expect(yield* content()).toBe("mine\n") + + // Host B builds the tree from the store, which is what makes it B's to move. + yield* Effect.promise(() => rm(worktree, { recursive: true, force: true })) + const B = yield* Layer.build(materializeStack(file, path.join(root, "host-b-data"))) + const ensureB = WorktreeMaterializer.Service.use((w) => w.ensure(worktree)).pipe( + Effect.provide(B), + ) + yield* ensureB + expect(yield* content()).toBe("v1\n") + + // The store moves on while B's tree does not: another host captured a newer state, which is + // what a worker picking up a later step of the same session arrives to. + yield* Effect.sleep(10) + yield* put("v2\n") + yield* ship + yield* put("v1\n") + + yield* ensureB + + // Rebuilding only a missing tree is not enough: a present one is served as it was left, and + // the tools of the step read files from whichever step this worker last ran. + expect(yield* content()).toBe("v2\n") + + // A tree already at the newest state is left alone whatever is in it. This is what keeps the + // later tools of a step from checking out over what its earlier tools wrote, since nothing + // captures those until the step is sealed. + yield* put("uncaptured\n") + yield* ensureB + expect(yield* content()).toBe("uncaptured\n") + + yield* Effect.promise(() => tmp[Symbol.asyncDispose]()) + }), + ) + // The shared-store deployment uses the libsql backend, so the pack blob has to survive that // driver's parameter path too, not only bun's. it.live("round-trips a pack blob through the libsql backend", () => From 77dccec17d902cd3046e47535d30589300f5a73a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 19:21:41 -0700 Subject: [PATCH 25/33] Wrote down what a running tool call and a stale tree now mean. --- packages/temporal/README.md | 39 +++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 1bbc81a80826..61390a627b83 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -126,16 +126,23 @@ Two things are load-bearing and easy to get wrong: owner, so a step's writers have to share one. Only `runModelCall` claims; the tool and seal activities publish under the token it returns. A token per activity execution (right when the activity *is* the whole step) would make them fence each other out. -- **A retried call is not silently repeated.** Whether a side effect already happened is the - activity's knowledge, not the workflow's, so `retry` comes from the Temporal attempt number. On a - retry only a tool declaring `idempotent` runs again; anything else is reported to the model as an - unknown outcome. This is the rule the crash-resume path already followed. +- **A call that already started is not silently repeated.** Whether a side effect may have happened + is read off the log: `runToolCall` publishes `Tool.Called` before it runs the tool, so a call the + log shows as running is one a dispatch was already inside. Then only a tool declaring `idempotent` + runs again; anything else is reported to the model as an unknown outcome. This is the rule the + crash-resume path already followed, off the same evidence. + + The attempt number would answer the same question far less precisely. It counts every way a + dispatch can die, including the ones that never reached the tool, so a call nobody had touched + would come back to the model as an unknown outcome. Publishing the call at dispatch also moves the + fence in front of the side effect: under a superseded owner the publish fails and the tool never + runs, where before it ran and then lost its result. It is also what closes the zombie window, which the settled-result check on its own does not: that check is a read then a write, so an attempt that lost its heartbeat but kept running could race a retry past it. For the case that matters, a non-idempotent side effect running twice, it cannot - happen anyway, because the retry refuses to run the tool at all. What can still race is which - truthful outcome reaches the model, the zombie's real result or the retry's "unknown", and both + happen anyway, because the second dispatch refuses to run the tool at all. What can still race is + which truthful outcome reaches the model, the zombie's real result or the "unknown", and both describe something that did happen. Reporting success for a tool that never ran is not reachable. #### What it costs, measured @@ -522,12 +529,20 @@ all ride the shared store. **The project working tree** now rides the store too. After each step capture the runner ships the snapshot tree as an incremental git pack (`snapshot-sync.ts`, `snapshot_pack` table). Before a -drain runs, a worker missing the session's directory rebuilds the worktree from those packs -(`session/execution/worktree.ts`): uncommitted edits and untracked files included, checked out at -the same absolute path it was captured at (a uniform fleet layout). Ignored files and dependencies -are not captured, so a rebuilt tree may need an install step before `bash` behaves identically. -Worker affinity (below) or a shared volume skips the materialization latency on warm paths; the -packs are the portable baseline that works with neither. +drain runs, a worker whose tree is missing or older than the store's newest capture builds it from +those packs (`session/execution/worktree.ts`): uncommitted edits and untracked files included, +checked out at the same absolute path it was captured at (a uniform fleet layout). Ignored files +and dependencies are not captured, so a rebuilt tree may need an install step before `bash` behaves +identically. Worker affinity (below) or a shared volume skips the materialization latency on warm +paths; the packs are the portable baseline that works with neither. + +Two rules bound what that refresh may touch, because checking a stored tree out over the wrong one +destroys work. A tree is moved only when a host-local note (`snapshot/tip.ts`) says this host is +behind the store, so a host holding a capture that never shipped is left as it is. And it is moved +only when this host built the tree from packs, so a checkout the host already had, a developer's own +working copy, is never rewritten: that case is logged and left alone. What stays open is the tools +of ONE step running on two hosts, since nothing captures their writes until the step is sealed. +Affinity is what keeps a step's tools on one tree. Host-local state that does NOT ride the DB, so it is not reconstructed on a different host: From e4cc70f9002f193586f8ec35bc1d433e148e256a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 19:31:00 -0700 Subject: [PATCH 26/33] Held every driver to running a tool call once. The contract only covered turns with no tools, so the piece the split actually changes, one activity per call instead of one for the step, had no scenario in the suite that both modes have to pass. --- .../core/src/session/execution/conformance.ts | 82 ++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/packages/core/src/session/execution/conformance.ts b/packages/core/src/session/execution/conformance.ts index a9386a7ed724..fc0fd16cdfae 100644 --- a/packages/core/src/session/execution/conformance.ts +++ b/packages/core/src/session/execution/conformance.ts @@ -24,6 +24,8 @@ import { SessionProjector } from "@opencode-ai/core/session/projector" import { SessionExecution } from "@opencode-ai/core/session/execution" import { SessionExecutionLocal } from "@opencode-ai/core/session/execution/local" import { SessionRunnerModel, ModelNotSelectedError } from "@opencode-ai/core/session/runner/model" +import { ApplicationTools } from "@opencode-ai/core/tool/application-tools" +import { Tool } from "@opencode-ai/core/tool/tool" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { SessionTable } from "@opencode-ai/core/session/sql" import { SessionStore } from "@opencode-ai/core/session/store" @@ -39,7 +41,7 @@ import { Auth } from "@opencode-ai/llm/route" import { describe, expect } from "bun:test" import { realpathSync } from "node:fs" import { tmpdir } from "node:os" -import { Cause, Context, DateTime, Effect, Exit, Layer, Stream } from "effect" +import { Cause, Context, DateTime, Effect, Exit, Layer, Schema, Stream } from "effect" import { testEffect } from "../../testing/effect" // The per-location service build resolves the session directory on disk, so it must exist. @@ -77,13 +79,50 @@ const countingModel = () => { return { requests, stream } } +// Asks for one tool on the first request and answers on the second, which is the smallest turn that +// makes a driver dispatch a tool and come back to the model with its result. +const toolCallingModel = () => { + const requests: number[] = [] + const stream: LLMClientShape["stream"] = () => { + requests.push(1) + if (requests.length > 1) + return Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.stepFinish({ index: 0, reason: "stop" }), + ]) + return Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call_contract", name: "contract_probe", input: {} }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + ]) + } + return { requests, stream } +} + +// Counts its own executions, so "ran exactly once" is checked against the tool rather than only +// against the projection. +const probeTool = (ran: { count: number }) => + Tool.make({ + description: "contract probe", + input: Schema.Struct({}), + output: Schema.String, + execute: () => + Effect.sync(() => { + ran.count += 1 + return "probed" + }), + }) + // The executor under test, built as its own graph over the shared database file (the same way the // serve process builds it), with the model/LLM mocked. Any SessionExecution node with the standard // dependency set (the local coordinator, the Temporal driver) plugs in here. export const makeExecutionFor = (node: typeof SessionExecutionLocal.node) => (stream: LLMClientShape["stream"], models = okModels) => - AppNodeBuilder.build(node, [ + // The tool registry is named in the group so a scenario can register a probe into the same + // graph the driver runs on. One graph, one instance: what the test registers is what the + // dispatch finds. + AppNodeBuilder.build(LayerNode.group([node, ApplicationTools.node]), [ [LayerNodePlatform.llmClient, mockClient(stream)], [PermissionV2.node, permission], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], @@ -273,5 +312,44 @@ export const runContract = (label: string, makeExec: ReturnType + withIdleOverride( + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + const graph = yield* Layer.build(makeExec(stream)) + yield* Context.get(graph, ApplicationTools.Service).register({ + contract_probe: probeTool(ran), + }) + const exec = Context.get(graph, SessionExecution.Service) + yield* exec.wake(sessionID) + // Waiting on the turn, not on the step: the first step's assistant is completed the + // moment the step is sealed, while the follow-up provider call is still to come. + yield* until(exec.active, (active) => !active.has(sessionID)) + // Where the drivers could drift: a step's model call, its tool and its close are one + // activity in one mode and three in the other, so the tool has to be dispatched once, + // settled in the log, and answered by a second provider turn either way. + expect(ran.count).toBe(1) + expect(requests).toHaveLength(2) + const store = yield* SessionStore.Service + const context = yield* store.context(sessionID) + const parts = context.flatMap((message) => + message.type === "assistant" ? message.content : [], + ) + const call = parts.findLast( + (part): part is SessionMessage.AssistantTool => + part.type === "tool" && part.id === "call_contract", + ) + expect(call?.state.status).toBe("completed") + }), + ), + 60000, + ) + } }) } From 4567c2e010061522ec7645cd253f45ee84084353 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 19:31:19 -0700 Subject: [PATCH 27/33] Noted the tool scenario in the contract suite. --- packages/temporal/README.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 61390a627b83..3259775019e8 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -277,13 +277,18 @@ OPENCODE_CONTRACT_TEMPORAL=1 OPENCODE_TEMPORAL_STEPPED=1 bun test --timeout 1800 test/session-execution-temporal-contract.test.ts ``` -Both pass 4/4. Running it the first time was worth the effort: **stepped mode failed 2 of the 4**, -on a case none of the live testing had reached. `countingModel` streams a step start and a step -finish and no content at all, so the publisher never mints an assistant message. The whole-step path -survives that because it mints one inside `Step.Ended` via `startAssistant()`; the seal, running in -another process with no publisher, searched the projection, found nothing, and left the turn open -forever. The fix carries the assistant message id out of the attempt the same way the tool call ids -are carried. +Both modes pass, and so does the local coordinator in core's own `bun test`. Running it the first +time was worth the effort: **stepped mode failed two scenarios**, on a case none of the live testing +had reached. `countingModel` streams a step start and a step finish and no content at all, so the +publisher never mints an assistant message. The whole-step path survives that because it mints one +inside `Step.Ended` via `startAssistant()`; the seal, running in another process with no publisher, +searched the projection, found nothing, and left the turn open forever. The fix carries the +assistant message id out of the attempt the same way the tool call ids are carried. + +One scenario is there for this split in particular: a turn that asks for a tool, runs it, and goes +back to the model with the result. Everything else in the suite settles without a tool, so the piece +the split actually changes, one activity per call rather than one for the whole step, would have had +no scenario that both modes must pass. #### Worker affinity (`OPENCODE_TEMPORAL_WORKTREE_AFFINITY=1`) From 885c2ee2100ea79100c8df35197cd737a46a432a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 26 Aug 2026 19:45:17 -0700 Subject: [PATCH 28/33] Capped the typecheck concurrency in fork CI. A full-width turbo run had several tsgo processes killed for memory on a hosted runner, which reads as a code failure. --- .github/workflows/test-fork.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-fork.yml b/.github/workflows/test-fork.yml index fc0311ce1046..32324733e45e 100644 --- a/.github/workflows/test-fork.yml +++ b/.github/workflows/test-fork.yml @@ -53,9 +53,11 @@ jobs: git config --global user.email "bot@opencode.ai" git config --global user.name "opencode" + # Concurrency capped: a full-width turbo run puts enough `tsgo` processes on a hosted runner + # at once that several are killed for memory, which reads as a code failure and is not one. - name: Typecheck timeout-minutes: 20 - run: bun typecheck + run: bun turbo typecheck --concurrency=2 - name: Unit tests (core) timeout-minutes: 30 From 270530342fcbc6e8577b5b4447e896480b476790 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 27 Aug 2026 11:54:19 -0400 Subject: [PATCH 29/33] Turned the live event tick on only where another process writes. The tick was on for every deployment, so a single process paid one query per second per subscribed session for events that cannot exist. The durable executor's composition root asks for it now, and OPENCODE_EVENT_POLL_MS overrides either way. --- packages/core/src/event.ts | 27 +++++++++++++++++++--- packages/core/src/flag/flag.ts | 6 +++++ packages/core/test/event.test.ts | 39 ++++++++++++++++---------------- packages/server/src/routes.ts | 15 ++++++++---- 4 files changed, 60 insertions(+), 27 deletions(-) diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index adac062fc013..9d3145ae96d4 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -7,6 +7,7 @@ import type { Data, Definition, Payload } from "@opencode-ai/schema/event" import { and, asc, eq, gt, inArray } from "drizzle-orm" import { Database } from "./database/database" import { EventSequenceTable, EventTable } from "./event/sql" +import { Flag } from "./flag/flag" import { Location } from "./location" import { makeGlobalNode } from "./effect/app-node" import { isDeepStrictEqual } from "node:util" @@ -179,15 +180,25 @@ export interface LayerOptions { * fires for commits made in THIS process, so without a tick a subscriber cannot see events * another process appended: a standalone worker's whole turn is invisible to a tail on the * HTTP server. - * Set to 0 to disable and rely on the wake alone. */ + * Zero (the default) relies on the wake alone, which is the whole story for a deployment that + * runs in one process. */ readonly livePollInterval?: Duration.Input } /** Chosen to be well under what a person notices in a transcript while staying one cheap indexed * read per subscribed session. In-process commits still wake instantly; this only catches what the - * wake cannot see. */ + * wake cannot see, so it is worth its cost only where another process writes: see `pollingNode`. */ const DEFAULT_LIVE_POLL = Duration.seconds(1) +// An operator's override, in milliseconds, for either node. Read at layer build rather than at +// import, so a test or a CLI that sets it late still gets it. +const configuredPoll = (fallback: Duration.Input): Duration.Input => { + const raw = Flag.OPENCODE_EVENT_POLL_MS + if (raw === undefined) return fallback + const millis = Number(raw) + return Number.isFinite(millis) && millis >= 0 ? millis : fallback +} + export const layerWith = (options?: LayerOptions) => Layer.effect( Service, @@ -635,7 +646,7 @@ export const layerWith = (options?: LayerOptions) => // Wake on either an in-process commit or the tick. A tick that finds nothing new reads // zero rows and emits nothing, so an idle subscriber costs one indexed query per // period. - const pollInterval = options?.livePollInterval ?? DEFAULT_LIVE_POLL + const pollInterval = configuredPoll(options?.livePollInterval ?? Duration.zero) const woken = Stream.fromSubscription(wakes) // haltStrategy "left" keeps the wake stream as what ends the tail. The tick never ends, // so the default ("both") would leave a subscriber hanging past the layer's own @@ -684,3 +695,13 @@ export const layerWith = (options?: LayerOptions) => const layer = layerWith() export const node = makeGlobalNode({ service: Service, layer: layer, deps: [Database.node] }) + +// For a deployment where the process serving subscribers is not the only one appending: a serve +// process driving sessions on standalone workers. The wake is published in-process, so without the +// tick a worker's whole turn is invisible to a tail on the server. Composition roots that know they +// are in that shape swap this in for `node`; everything else keeps the wake alone. +export const pollingNode = makeGlobalNode({ + service: Service, + layer: layerWith({ livePollInterval: DEFAULT_LIVE_POLL }), + deps: [Database.node], +}) diff --git a/packages/core/src/flag/flag.ts b/packages/core/src/flag/flag.ts index 20364dccdfdb..7b586bcb7db8 100644 --- a/packages/core/src/flag/flag.ts +++ b/packages/core/src/flag/flag.ts @@ -79,4 +79,10 @@ export const Flag = { get OPENCODE_CLIENT() { return process.env["OPENCODE_CLIENT"] ?? "cli" }, + // Milliseconds between a live event tail's own re-reads, `0` to rely on the in-process wake + // alone. Only a deployment where another process appends to the log needs it, so it is off unless + // the composition root asks for it; this overrides either way. + get OPENCODE_EVENT_POLL_MS() { + return process.env["OPENCODE_EVENT_POLL_MS"] + }, } diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index ffe0ac361c68..d45b2311faca 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -1174,28 +1174,29 @@ describe("EventV2", () => { }), ) - it.live("tails events another writer appended, which no in-process wake can announce", () => + it.live("tails another writer's events only when the layer asked for a tick", () => Effect.gen(function* () { - const events = yield* EventV2.Service - const aggregateID = Session.ID.create() - yield* events.publish(DurableMessage, durableData(aggregateID, "seed")) - // A second service over the SAME database with its OWN pubsub: exactly what a standalone - // worker is to the HTTP server. Its commits cannot reach this process's wake map, so if the - // tail only listened for wakes it would sit on the seed forever. - const other = yield* EventV2.Service.pipe(Effect.provide(EventV2.layerWith())) - - const tail = yield* events - .durable({ aggregateID, after: -1 }) - .pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped) - yield* Effect.sleep("100 millis") - yield* other.publish(DurableMessage, durableData(aggregateID, "from-elsewhere")) + // worker is to the HTTP server. Its commits cannot reach the reader's wake map, so a tail + // that only listens for wakes sits there forever. + const tailSees = (options: EventV2.LayerOptions) => + Effect.gen(function* () { + const reader = yield* EventV2.Service.pipe(Effect.provide(EventV2.layerWith(options))) + const writer = yield* EventV2.Service.pipe(Effect.provide(EventV2.layerWith())) + const aggregateID = Session.ID.create() + const tail = yield* reader + .durable({ aggregateID, after: -1 }) + .pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped) + yield* Effect.sleep("50 millis") + yield* writer.publish(DurableMessage, durableData(aggregateID, "from-elsewhere")) + const collected = yield* Fiber.join(tail).pipe(Effect.timeoutOption("1 second")) + return Option.isSome(collected) + }) - const collected = yield* Fiber.join(tail).pipe(Effect.timeout("10 seconds")) - expect(collected.map((event) => (event.data as { messageID: string }).messageID)).toEqual([ - durableData(aggregateID, "seed").messageID, - durableData(aggregateID, "from-elsewhere").messageID, - ]) + expect(yield* tailSees({ livePollInterval: "50 millis" })).toBe(true) + // Off unless asked for. One process wakes its own subscribers, so a tick there is a query per + // second per subscribed session for events that cannot exist. + expect(yield* tailSees({})).toBe(false) }), ) diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index 94117d000399..96e558065e2a 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -59,11 +59,15 @@ export function createServiceLayer() { // (execution/local.ts) -- no server, no ports. Both drive SessionRunner over the same durable // event log; the local coordinator owns the wake/resume/interrupt lifecycle and is shared with // the v1 server path, so it is the well-exercised default. - const executionNode = - process.env.OPENCODE_SESSION_EXECUTION === "temporal" - ? SessionExecutionTemporal.node - : SessionExecutionLocal.node - return AppNodeBuilder.build(applicationServices, [[SessionExecution.node, executionNode]]) + const temporal = process.env.OPENCODE_SESSION_EXECUTION === "temporal" + const executionNode = temporal ? SessionExecutionTemporal.node : SessionExecutionLocal.node + return AppNodeBuilder.build(applicationServices, [ + [SessionExecution.node, executionNode], + // Only the durable executor can put a turn in another process, and only then does a live tail + // need to re-read on its own: the wake it otherwise runs on is published in-process. Local mode + // keeps the wake alone, so the default deployment pays nothing for a case it does not have. + [EventV2.node, temporal ? EventV2.pollingNode : EventV2.node], + ]) } // The context for a standalone worker (src/worker.ts). Same services as serve, but with @@ -73,6 +77,7 @@ export function createWorkerLayer() { // A standalone worker only makes sense in temporal mode. return AppNodeBuilder.build(LayerNode.group([applicationServices, SessionExecution.node]), [ [SessionExecution.node, SessionExecutionTemporal.node], + [EventV2.node, EventV2.pollingNode], ]) } From 9ded3ce56ce0b7e40301bc77f2eef08e519ac68e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 27 Aug 2026 11:57:32 -0400 Subject: [PATCH 30/33] Named a refused run instead of inferring it from an interrupt. The boundary read any interrupt with no cancellation as the user declining, so a runner that stopped itself for another reason reported a decision nobody made. A dispatch raises the refusal under its own type now, and the whole-step path states that its interrupt means the same thing. --- packages/core/src/session/runner/llm.ts | 17 ++++--- .../test/session-runner-model-call.test.ts | 9 ++-- packages/temporal/src/boundary.ts | 45 +++++++++++++------ packages/temporal/src/drain.ts | 3 ++ packages/temporal/test/l2-step.test.ts | 40 ++++++++++++----- 5 files changed, 81 insertions(+), 33 deletions(-) diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 944e7e851d6e..8242697cff5b 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -26,6 +26,7 @@ import { ToolRegistry } from "../../tool/registry" import { ToolOutputStore } from "../../tool-output-store" import { SessionContextEpoch } from "../context-epoch" import { SessionCompaction } from "../compaction" +import { SessionRunDeclinedError } from "../error" import { SessionEvent } from "../event" import { SessionHistory } from "../history" import { SessionInput } from "../input" @@ -823,18 +824,20 @@ const layer = Layer.effect( input: input.call.input, }), }) - // A decline is the user stopping the turn, not a tool that failed. It has to reach the - // boundary as an interrupt, or the dispatcher treats it as one bad tool, seals the step and - // the agent carries on past a refusal. .pipe( - Effect.catchCause((cause) => - isUserDeclined(cause) ? Effect.interrupt : Effect.failCause(cause), - ), // The tool itself ran: what failed is storing its output. Letting that fail the dispatch // would retry a side effect that already happened and finally tell the model the call was // interrupted, which is not what happened to it. The whole-step path reports the same - // reason the same way. + // reason the same way. A decline is a defect, so it passes through this untouched. Effect.catch((error) => Effect.succeed({ failure: error })), + // A decline is the user stopping the turn, not a tool that failed. It is named here + // rather than raised as a bare interrupt, so nothing downstream has to infer from the + // absence of a cancellation what the user meant. + Effect.catchCause((cause) => + isUserDeclined(cause) + ? Effect.fail(new SessionRunDeclinedError({ sessionID: input.sessionID })) + : Effect.failCause(cause), + ), ) if ("failure" in settlement) { const reason = diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index ca1c2ff4c5bb..bb1829f2b565 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -36,6 +36,7 @@ import { SessionTable } from "@opencode-ai/core/session/sql" import { SessionStore } from "@opencode-ai/core/session/store" import { SessionMessage } from "@opencode-ai/core/session/message" import { SessionEvent } from "@opencode-ai/core/session/event" +import { SessionRunDeclinedError } from "@opencode-ai/core/session/error" import { DateTime } from "effect" import { ConfigCompaction } from "@opencode-ai/core/config/compaction" import { SessionContextEpoch } from "@opencode-ai/core/session/context-epoch" @@ -853,11 +854,13 @@ describe("SessionRunner declined permission in a stepped turn", () => { .runToolCall({ sessionID, call: model.calls[0] }) .pipe(Effect.exit) - // An interrupt is what the activity boundary turns into a non-retryable halt. A plain failure - // reads as one bad tool and the turn continues. + // Named, not inferred: the activity boundary turns this error into a non-retryable halt, + // where a plain tool failure reads as one bad tool and the turn continues. Raising a bare + // interrupt instead would leave the boundary to guess the user's decision from the absence + // of a cancellation. expect(Exit.isFailure(exit)).toBe(true) if (!Exit.isFailure(exit)) return - expect(Cause.hasInterrupts(exit.cause)).toBe(true) + expect(Cause.squash(exit.cause)).toBeInstanceOf(SessionRunDeclinedError) }), ) }) diff --git a/packages/temporal/src/boundary.ts b/packages/temporal/src/boundary.ts index 6d93d42f6bea..579f52024096 100644 --- a/packages/temporal/src/boundary.ts +++ b/packages/temporal/src/boundary.ts @@ -1,14 +1,11 @@ // Crossing from Effect into an activity result. Every drain in this package ends here, so the way a // failure is classified is decided in one place: // - an interrupt caused by the driver's own cancellation rethrows the abort reason, so the -// attempt -// records Cancelled rather than Failed -// - an interrupt with no cancellation is an internal halt (a user declining a permission) and -// must -// be non-retryable, or the supervisor re-drives a turn the user explicitly stopped +// attempt records Cancelled rather than Failed +// - a run the user refused (a declined permission) crosses non-retryable under its own type, or +// the supervisor re-drives a turn the user explicitly stopped // - a genuine run error crosses non-retryable with the RunError encoded in `details`, so the -// caller -// reconstructs the exact typed error instead of a string +// caller reconstructs the exact typed error instead of a string // Only crashes and task timeouts (never thrown here) go through the activity retry policy. import { Cause, Effect, Exit } from "effect" @@ -18,10 +15,32 @@ import { SessionRunDeclinedError } from "@opencode-ai/core/session/error" import { encodeRunError } from "@opencode-ai/core/session/execution/run-error-codec" import { HALTED_FAILURE_TYPE } from "./protocol" +export interface BoundaryOptions { + /** Whether an interrupt the body raises on its own, with nothing cancelling it, means the user + * refused the run. The whole-step path says yes: it turns a decline into an interrupt to halt + * the loop, which is the local behaviour V1 defined and its own tests pin. The stepped path says + * no, because its dispatch names the decline, so an interrupt there is the runner stopping for + * some other reason and reading it as a refusal would put words in the user's mouth. */ + readonly declineIsInterrupt?: boolean +} + +const halted = (sessionID: string, declined?: SessionRunDeclinedError) => { + const encoded = encodeRunError( + declined ?? new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make(sessionID) }), + ) + return ApplicationFailure.create({ + message: "session run halted (user declined)", + type: HALTED_FAILURE_TYPE, + nonRetryable: true, + details: encoded === undefined ? undefined : [encoded], + }) +} + export const runAtBoundary = async ( sessionID: string, signal: AbortSignal, body: Effect.Effect, + options?: BoundaryOptions, ): Promise => { const exit = await Effect.runPromiseExit(body, { signal }) if (Exit.isSuccess(exit)) return exit.value @@ -29,17 +48,17 @@ export const runAtBoundary = async ( if (Cause.hasInterruptsOnly(cause)) { if (signal.aborted) throw signal.reason instanceof Error ? signal.reason : new Error("session run interrupted") - const declined = encodeRunError( - new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make(sessionID) }), - ) + if (options?.declineIsInterrupt) throw halted(sessionID) + // Nothing cancelled this run and nothing named a reason: the runner stopped itself, which a + // session handed to a runner bound to another location does. throw ApplicationFailure.create({ - message: "session run halted (user declined)", - type: HALTED_FAILURE_TYPE, + message: "session run interrupted with no cancellation", + type: "SessionRunInterrupted", nonRetryable: true, - details: declined === undefined ? undefined : [declined], }) } const squashed = Cause.squash(cause) as { _tag?: string; message?: string } + if (squashed instanceof SessionRunDeclinedError) throw halted(sessionID, squashed) const encoded = encodeRunError(squashed) throw ApplicationFailure.create({ message: squashed?.message ?? Cause.pretty(cause), diff --git a/packages/temporal/src/drain.ts b/packages/temporal/src/drain.ts index 8d4363022ce3..d3788c787200 100644 --- a/packages/temporal/src/drain.ts +++ b/packages/temporal/src/drain.ts @@ -71,6 +71,9 @@ export const makeDrains = ({ store, locations, ctx, events, worktrees }: DrainDe ).pipe(Effect.provide(locations.get(session.location))) return { ran: r.ran, continue: r.continue, step: r.step, promotion: r.promotion ?? null } }).pipe(Effect.provideService(EventV2.EventOwner, input.owner), Effect.provide(ctx), Effect.scoped), + // A whole step turns a declined permission into an interrupt to halt its own loop, so an + // interrupt with nothing cancelling it is that refusal and nothing else. + { declineIsInterrupt: true }, ) return { stepDrain } diff --git a/packages/temporal/test/l2-step.test.ts b/packages/temporal/test/l2-step.test.ts index 15b8728ab9de..2006467e1087 100644 --- a/packages/temporal/test/l2-step.test.ts +++ b/packages/temporal/test/l2-step.test.ts @@ -5,6 +5,8 @@ import { describe, it, expect } from "bun:test" import { isHaltFailure, makeSteppedTurn, type SteppedActivities } from "../src/l2-step" import { runAtBoundary } from "../src/boundary" +import { SessionRunDeclinedError } from "@opencode-ai/core/session/error" +import { SessionSchema } from "@opencode-ai/core/session/schema" import { Effect } from "effect" import { ActivityFailure, @@ -155,23 +157,41 @@ describe("halt predicate", () => { const wrap = (cause?: Error) => new ActivityFailure("activity failed", "runToolCall", "1", 1 as never, undefined, cause) - it("recognises what the activity boundary throws for a user halt", async () => { - // Built by running the boundary rather than by hand, so the two sides cannot drift: an - // interrupt - // with no abort is how a decline leaves the runner, and whatever that produces is what a - // dispatcher has to recognise. - const thrown = await runAtBoundary( - "ses_1", - new AbortController().signal, - Effect.interrupt, - ).then( + const throwsFrom = ( + body: Effect.Effect, + options?: { declineIsInterrupt: true }, + ) => + runAtBoundary("ses_1", new AbortController().signal, body, options).then( () => undefined, (error: unknown) => error, ) + + it("recognises what the activity boundary throws for a named refusal", async () => { + // Built by running the boundary rather than by hand, so the two sides cannot drift: a dispatch + // reports a decline as this error, and whatever the boundary makes of it is what a dispatcher + // has to recognise. + const thrown = await throwsFrom( + Effect.fail(new SessionRunDeclinedError({ sessionID: SessionSchema.ID.make("ses_1") })), + ) expect(thrown).toBeInstanceOf(ApplicationFailure) expect(isHaltFailure(wrap(thrown as Error))).toBe(true) }) + it("recognises a whole step's refusal, which arrives as an interrupt", async () => { + const thrown = await throwsFrom(Effect.interrupt, { declineIsInterrupt: true }) + expect(isHaltFailure(wrap(thrown as Error))).toBe(true) + }) + + it("does not read an unexplained interrupt as a refusal", async () => { + // The stepped path names its refusals, so an interrupt with nothing cancelling it is the + // runner stopping for another reason. Calling that a decline would report a user decision + // nobody made. + const thrown = await throwsFrom(Effect.interrupt) + expect(thrown).toBeInstanceOf(ApplicationFailure) + expect((thrown as ApplicationFailure).type).toBe("SessionRunInterrupted") + expect(isHaltFailure(wrap(thrown as Error))).toBe(false) + }) + it("says no to everything else that can come back from an activity", () => { expect(isHaltFailure(wrap(new CancelledFailure("cancelled")))).toBe(false) expect(isHaltFailure(wrap(new TimeoutFailure("timed out", undefined, 1 as never)))).toBe(false) From 1e23cce233e2504ed942b44a340e8ced2e06350e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 27 Aug 2026 12:08:38 -0400 Subject: [PATCH 31/33] Closed a tool call that a stop cut short. A whole step closes the tools it opened on its way out; a call that is its own activity had nobody to do it, so an interrupted turn left the call recorded as running until the next prompt. A cancellation that hands the call to another attempt still leaves it alone. --- .../core/src/session/execution/conformance.ts | 53 +++++++++++++++++++ packages/core/src/session/runner/index.ts | 4 ++ packages/core/src/session/runner/llm.ts | 46 +++++++++++----- .../test/session-runner-model-call.test.ts | 35 ++++++++++++ packages/temporal/src/activities.ts | 27 ++++++++-- packages/temporal/src/l2-drain.ts | 17 +++++- 6 files changed, 164 insertions(+), 18 deletions(-) diff --git a/packages/core/src/session/execution/conformance.ts b/packages/core/src/session/execution/conformance.ts index fc0fd16cdfae..c5e1c4b93c58 100644 --- a/packages/core/src/session/execution/conformance.ts +++ b/packages/core/src/session/execution/conformance.ts @@ -113,6 +113,22 @@ const probeTool = (ran: { count: number }) => }), }) +// A tool that never finishes, so a turn can be interrupted with one in flight. +const hangingTool = () => + Tool.make({ + description: "contract probe that never finishes", + input: Schema.Struct({}), + output: Schema.String, + execute: () => Effect.never, + }) + +const toolParts = (messages: ReadonlyArray) => + messages.flatMap((message) => + message.type === "assistant" + ? message.content.filter((part): part is SessionMessage.AssistantTool => part.type === "tool") + : [], + ) + // The executor under test, built as its own graph over the shared database file (the same way the // serve process builds it), with the model/LLM mocked. Any SessionExecution node with the standard // dependency set (the local coordinator, the Temporal driver) plugs in here. @@ -351,5 +367,42 @@ export const runContract = (label: string, makeExec: ReturnType + withIdleOverride( + Effect.gen(function* () { + yield* seedSession(sessionID) + yield* seedPrompt(sessionID) + const graph = yield* Layer.build(makeExec(stream)) + yield* Context.get(graph, ApplicationTools.Service).register({ + contract_probe: hangingTool(), + }) + const exec = Context.get(graph, SessionExecution.Service) + const store = yield* SessionStore.Service + yield* exec.wake(sessionID) + yield* until(store.context(sessionID), (context) => + toolParts(context).some((part) => part.state.status === "running"), + ) + + yield* exec.interrupt(sessionID) + + // A call left running is not a cosmetic loose end: the transcript shows a tool still + // going, and the next turn has to reconstruct what happened to it. A whole step closes + // the tools it opened on its way out, and a step whose calls are their own units of + // work has to end up in the same place. + yield* until(store.context(sessionID), (context) => + toolParts(context).every((part) => part.state.status !== "running"), + ) + const closed = toolParts(yield* store.context(sessionID)) + expect(closed.map((part) => part.state.status)).toEqual(["error"]) + yield* until(exec.active, (active) => !active.has(sessionID)) + }), + ), + 60000, + ) + } }) } diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index 5801424317d1..bdcdc9835395 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -124,6 +124,10 @@ export interface Interface { /** Run one recorded tool call and publish its result. Safe to call twice for the same call: the * second sees the settled result and does nothing. */ readonly runToolCall: (input: ToolCallInput) => Effect.Effect + /** Close a call a stop cut short, so it does not sit in the log as running until the next turn. + * A whole step closes the tools it opened on its way out; a dispatch that is its own unit of work + * has to be told. A call that never started, and one that already settled, are left alone. */ + readonly failToolCall: (input: ToolCallInput) => Effect.Effect /** Close a step once its calls have been dispatched: snapshot, file diff, Step.Ended, and the * loop decision. Safe to call twice: the second sees the step already closed and returns the same * answer without publishing again. */ diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 8242697cff5b..df79bfc3a1cc 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -754,23 +754,25 @@ const layer = Layer.effect( }) + // A point read, not the whole projected history. The call carries the message that owns it, so + // decoding every message to find one part would make a step cost O(session) per tool instead + // of O(1). Message ids are looked up on their own, so the session has to be checked too: a call + // from another session must not resolve here. + const recordedCall = Effect.fnUntraced(function* (input: ToolCallInput) { + const owner = yield* store.message(SessionMessage.ID.make(input.call.assistantMessageID)) + if (owner?.sessionID !== input.sessionID || owner.message.type !== "assistant") + return undefined + // findLast to agree with the projector, which writes tool updates the same way. + return owner.message.content.findLast( + (item): item is SessionMessage.AssistantTool => + item.type === "tool" && item.id === input.call.id, + ) + }) + const runToolCall = Effect.fn("SessionRunner.runToolCall")(function* (input: ToolCallInput) { const session = yield* getSession(input.sessionID) const assistantMessageID = SessionMessage.ID.make(input.call.assistantMessageID) - // A point read, not the whole projected history. The call carries the message - // that owns it, so decoding every message to find one part would make a step - // cost O(session) per tool instead of O(1). - // Message ids are looked up on their own, so the session has to be checked too: a call from - // another session must not resolve here. - const owner = yield* store.message(assistantMessageID) - const part = - owner?.sessionID === input.sessionID && owner.message.type === "assistant" - ? // findLast to agree with the projector, which writes tool updates the same way. - owner.message.content.findLast( - (item): item is SessionMessage.AssistantTool => - item.type === "tool" && item.id === input.call.id, - ) - : undefined + const part = yield* recordedCall(input) // The call has to be in the log already: the attempt that produced it recorded its input // before handing it over. Missing means the log moved under us (a fence), and running a tool // whose call is not recorded would leave an orphan result. @@ -871,6 +873,21 @@ const layer = Layer.effect( return { outcome: "settled" } as ToolCallResult }) + const failToolCall = Effect.fn("SessionRunner.failToolCall")(function* (input: ToolCallInput) { + const part = yield* recordedCall(input) + // Only a call a dispatch had started is closed. One that never reached its tool is left + // pending for the next turn's entry check, and a settled one keeps the result it earned. + if (part?.state.status !== "running") return + yield* events.publish(SessionEvent.Tool.Failed, { + sessionID: input.sessionID, + timestamp: yield* DateTime.now, + assistantMessageID: SessionMessage.ID.make(input.call.assistantMessageID), + callID: input.call.id, + error: { type: "unknown", message: "Tool execution interrupted" }, + provider: { executed: false }, + }) + }) + const runStep = Effect.fn("SessionRunner.runStep")(function* (input: StepInput) { const prologue = yield* stepPrologue(input) if (prologue.kind === "settled") return prologue.result @@ -898,6 +915,7 @@ const layer = Layer.effect( runStep, runModelCall, runToolCall, + failToolCall, sealStep, }) }), diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index bb1829f2b565..b5527599eca1 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -495,6 +495,41 @@ describe("SessionRunner tool dispatch", () => { }), ) + harness(callsCrashingTool).effect("closes a started call that a stop cut short", () => + Effect.gen(function* () { + yield* seedSession + const ran = counters() + yield* registerProbes(ran) + const call = yield* deferOneCall + const runner = yield* SessionRunner.Service + const store = yield* SessionStore.Service + const recorded = Effect.map(store.context(sessionID), (context) => { + const part = toolPart(context, "call_probe") + return part?.type === "tool" ? part.state : undefined + }) + + // Nothing has started this call, so it is left for the next turn's entry check: closing it + // here would report a tool the model asked for as having been cut short. + yield* runner.failToolCall({ sessionID, call }) + expect((yield* recorded)?.status).toBe("pending") + + // A dispatch died with the tool in flight, which is what the log shows after a stop lands + // mid-tool: the call recorded as running, with no result. + yield* runner.runToolCall({ sessionID, call }).pipe(Effect.exit) + expect((yield* recorded)?.status).toBe("running") + + yield* runner.failToolCall({ sessionID, call }) + const closed = yield* recorded + expect(closed?.status).toBe("error") + const reason = closed?.status === "error" ? closed.error.message : "" + expect(reason).toBe("Tool execution interrupted") + + // A call that is already terminal keeps what it has, whatever lands afterwards. + yield* runner.failToolCall({ sessionID, call }) + expect(yield* recorded).toEqual(closed) + }), + ) + harness(callsCrashingIdempotentTool).effect( "re-runs a started call that declares itself repeatable", () => diff --git a/packages/temporal/src/activities.ts b/packages/temporal/src/activities.ts index 01ee68140f1e..8ce8b1310b06 100644 --- a/packages/temporal/src/activities.ts +++ b/packages/temporal/src/activities.ts @@ -3,7 +3,13 @@ // crash is detected quickly, and forwards Temporal cancellation as an AbortSignal so an interrupt // turns into Effect fiber interruption inside the runner. -import { heartbeat, Context } from "@temporalio/activity" +import { heartbeat, CancelledFailure, Context } from "@temporalio/activity" + +// The SDK's cancel reasons that mean nothing is coming back for this call: the workflow asked to +// stop, or it is gone. The rest (a worker shutting down, a pause, a heartbeat timeout, a reset) +// hand the same call to another attempt. They reach an activity as the reason on the abort, which +// every SDK version sets, and as `cancellationDetails`, which only a server that sends them does. +const ENDS_THE_TURN = ["CANCELLED", "NOT_FOUND"] // The event-log owner token for one activity execution: run id + activity id + attempt. A Temporal // retry of the SAME step gets a fresh attempt, so once the retry claims the log the previous @@ -88,7 +94,11 @@ export function makeSteppedTurnActivities(drains: { input: ModelCallDrainInput & { readonly owner: string }, signal: AbortSignal, ) => Promise - toolCallDrain: (input: ToolCallDrainInput, signal: AbortSignal) => Promise + toolCallDrain: ( + input: ToolCallDrainInput, + signal: AbortSignal, + turnEnded: () => boolean, + ) => Promise sealDrain: (input: SealDrainInput, signal: AbortSignal) => Promise }): SteppedTurnActivities { return { @@ -101,7 +111,18 @@ export function makeSteppedTurnActivities(drains: { ) }, async runToolCall(input) { - return beating(() => drains.toolCallDrain(input, Context.current().cancellationSignal)) + const context = Context.current() + // An attempt that is being handed on must leave the call as it found it, or its successor + // reads a closed call and never runs the tool. A superseded attempt learns of it the same way + // a stopped one does; closing a call its successor is re-running costs the model that result, + // which is the smaller loss. + const turnEnded = () => { + const details = context.cancellationDetails + if (details) return details.cancelRequested || details.notFound + const reason: unknown = context.cancellationSignal.reason + return reason instanceof CancelledFailure && ENDS_THE_TURN.includes(reason.message ?? "") + } + return beating(() => drains.toolCallDrain(input, context.cancellationSignal, turnEnded)) }, async sealStep(input) { return beating(() => drains.sealDrain(input, Context.current().cancellationSignal)) diff --git a/packages/temporal/src/l2-drain.ts b/packages/temporal/src/l2-drain.ts index 2439eaca1b64..b61d721eb3b9 100644 --- a/packages/temporal/src/l2-drain.ts +++ b/packages/temporal/src/l2-drain.ts @@ -143,12 +143,27 @@ export const makeL2Drains = ({ store, locations, ctx, events, worktrees }: L2Dra const toolCallDrain = async ( input: ToolCallDrainInput, signal: AbortSignal, + /** Whether a cancellation means the turn is over rather than this attempt being handed on. + * Only the first closes the call: a worker shutting down leaves it for the next attempt, which + * has to be free to decide whether the tool may run again. */ + turnEnded: () => boolean = () => false, ): Promise => runAtBoundary( input.sessionID, signal, inSession(input.sessionID, input.owner, false, (runner, session) => - runner.runToolCall({ sessionID: session.id, call: input.call }), + runner.runToolCall({ sessionID: session.id, call: input.call }).pipe( + // A stop landing mid-tool leaves the call recorded as running, where a whole step closes + // the tools it opened before it returns. Nothing else closes it until the next turn's + // entry check, so a transcript would show the call still going long after the stop. + Effect.onInterrupt(() => + turnEnded() + ? runner + .failToolCall({ sessionID: session.id, call: input.call }) + .pipe(Effect.ignore) + : Effect.void, + ), + ), ).pipe(Effect.map((result) => result ?? { outcome: "already-settled" as const })), ) From e602f432afef1269a59a3703849039e787737279 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 27 Aug 2026 12:10:40 -0400 Subject: [PATCH 32/33] Reported the calls a step did not settle. A dispatch that decided not to run a tool, or lost its result, read as an ordinary success everywhere an operator looks. The outcome each one already returns says which call it was. --- packages/core/src/session/runner/index.ts | 5 ++- packages/temporal/src/l2-step.ts | 22 ++++++++++++- packages/temporal/src/workflow.ts | 3 ++ packages/temporal/test/l2-step.test.ts | 38 +++++++++++++++++++++-- 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index bdcdc9835395..2a63cf5ae9db 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -67,7 +67,10 @@ export interface ToolCallInput { readonly call: DeferredToolCall } -/** How a dispatched call ended. +/** How a dispatched call ended. It is the dispatch's own account of itself: the log says what the + * model was told, not whether a tool was skipped on purpose or its result was lost. A durable + * executor keeps it with the rest of the call's record, and reports the ones that are not `settled` + * where an operator reads about the turn. * - `settled`: the tool ran and its result is durable. * - `already-settled`: the log already had a result, so nothing ran. The at-least-once case. * - `failed`: the tool ran and could not be settled. The reason reaches the model, and the call is diff --git a/packages/temporal/src/l2-step.ts b/packages/temporal/src/l2-step.ts index 55950e0f9b98..60aaefd27240 100644 --- a/packages/temporal/src/l2-step.ts +++ b/packages/temporal/src/l2-step.ts @@ -52,6 +52,10 @@ export interface SteppedTurnDeps { * ordinary activity failure, so without this it reads as one bad tool and the turn carries on * past the refusal. */ readonly isHalt: (error: unknown) => boolean + /** Where a step says what became of the calls it dispatched. A dispatch that decided not to run + * a tool, or could not keep its result, is a step's most surprising outcome and the least + * visible: it reads as an ordinary success everywhere else. */ + readonly log?: (message: string, attributes: Record) => void } /** @@ -60,7 +64,7 @@ export interface SteppedTurnDeps { * of a whole-step activity. */ export const makeSteppedTurn = - ({ activities, isCancellation, isHalt }: SteppedTurnDeps) => + ({ activities, isCancellation, isHalt, log }: SteppedTurnDeps) => async (input: StepDrainInput): Promise => { const model = await activities.runModelCall(input) // A crashed step finalized from the log, or the recovery gate finding no work: the step is over @@ -80,6 +84,22 @@ export const makeSteppedTurn = if (isCancellation(outcome.reason) || isHalt(outcome.reason)) throw outcome.reason } + // A dispatch that settled its call needs no telling. The rest are what an operator is looking + // for when a turn did something unexpected: a tool reported as unknown ran or did not, and + // nothing else in this workflow's history says which call that was. + const unsettled = model.calls + .map((call, index) => { + const result = dispatched[index] + return { + call: call.id, + tool: call.name, + outcome: result?.status === "fulfilled" ? result.value.outcome : "errored", + } + }) + .filter((entry) => entry.outcome !== "settled") + if (unsettled.length > 0) + log?.("step did not settle every call it dispatched", { step: model.step, calls: unsettled }) + return activities.sealStep({ sessionID: input.sessionID, step: model.step, diff --git a/packages/temporal/src/workflow.ts b/packages/temporal/src/workflow.ts index a32dca546c97..a395a39dd674 100644 --- a/packages/temporal/src/workflow.ts +++ b/packages/temporal/src/workflow.ts @@ -18,6 +18,7 @@ import { CancellationScope, isCancellation, allHandlersFinished, + log, } from "@temporalio/workflow" import type { StepActivities, SteppedTurnActivities } from "./activities" import { isHaltFailure, makeSteppedTurn } from "./l2-step" @@ -118,6 +119,8 @@ const steppedRuntime: SupervisorRuntime = { activities: { runModelCall, runToolCall, sealStep }, isCancellation, isHalt: isHaltFailure, + // The SDK's logger, so a line carries its workflow and run id and is suppressed on replay. + log: (message, attributes) => log.info(message, attributes), }), } diff --git a/packages/temporal/test/l2-step.test.ts b/packages/temporal/test/l2-step.test.ts index 2006467e1087..f92ad5abddd4 100644 --- a/packages/temporal/test/l2-step.test.ts +++ b/packages/temporal/test/l2-step.test.ts @@ -15,7 +15,12 @@ import { TimeoutFailure, } from "@temporalio/workflow" import type { StepDrainInput, StepDrainResult } from "../src/activities" -import type { ModelCallDrainResult, SealDrainInput, ToolCallDrainInput } from "../src/l2-drain" +import type { + ModelCallDrainResult, + SealDrainInput, + ToolCallDrainInput, + ToolCallDrainResult, +} from "../src/l2-drain" class FakeCancel extends Error {} class FakeHalt extends Error {} @@ -39,7 +44,7 @@ const call = (id: string, name = "probe_write") => ({ const fakes = ( model: ModelCallDrainResult, - onTool: (input: ToolCallDrainInput) => Promise<{ outcome: "settled" }> = async () => ({ + onTool: (input: ToolCallDrainInput) => Promise = async () => ({ outcome: "settled", }), ) => { @@ -98,6 +103,35 @@ describe("stepped turn", () => { expect(result).toEqual(SEALED) }) + it("reports the calls it did not settle, and says nothing when they all settled", async () => { + const lines: Array<{ message: string; attributes: Record }> = [] + const log = (message: string, attributes: Record) => + lines.push({ message, attributes }) + const model: ModelCallDrainResult = { + kind: "called", + step: 2, + calls: [call("call_a"), call("call_b", "probe_read")], + owner: "own", + } + const settling = fakes(model) + await makeSteppedTurn({ ...settling, isCancellation, isHalt, log })(INPUT) + // Nothing surprising happened, so nothing is said about it. + expect(lines).toHaveLength(0) + + const skipping = fakes(model, async (input) => ({ + outcome: input.call.id === "call_a" ? "unknown" : "settled", + })) + await makeSteppedTurn({ ...skipping, isCancellation, isHalt, log })(INPUT) + + // Which call was skipped, and why, is the dispatch's own knowledge: the log records what the + // model was told, and the workflow's history records an activity that succeeded. + expect(lines).toHaveLength(1) + expect(lines[0]?.attributes).toEqual({ + step: 2, + calls: [{ call: "call_a", tool: "probe_write", outcome: "unknown" }], + }) + }) + it("seals even when a tool call fails outright", async () => { const { activities, tools, seals } = fakes( { kind: "called", step: 2, calls: [call("call_a"), call("call_b")], owner: "own" }, From de37977995aa590ee0e527002e8acf2df4028ed7 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 27 Aug 2026 12:12:00 -0400 Subject: [PATCH 33/33] Wrote down what a stop, a refusal and the tick now do. --- packages/temporal/README.md | 48 +++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 3259775019e8..d2d8ac5a6f95 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -144,6 +144,19 @@ Two things are load-bearing and easy to get wrong: happen anyway, because the second dispatch refuses to run the tool at all. What can still race is which truthful outcome reaches the model, the zombie's real result or the "unknown", and both describe something that did happen. Reporting success for a tool that never ran is not reachable. +- **A stop closes the calls it cut short.** A whole step closes the tools it opened on its way out. + A call that is its own activity has nobody to do that, so an interrupted turn used to leave it + recorded as running until the next prompt: a transcript showing a tool still going, and an entry + check left to work out what happened to it. The dispatch closes its own call when the cancellation + means the turn is over. When it means this attempt is being handed on (a worker shutting down, a + pause, a heartbeat timeout), the call is left exactly as it was, or the next attempt would read a + closed call and never run the tool. +- **A refusal is named, not inferred.** A declined permission crosses the activity boundary as its + own error type. The whole-step path still raises it as an interrupt, because halting the local + loop that way is the behaviour V1 defined and its tests pin, so that drain says so at the boundary + (`declineIsInterrupt`). What the boundary no longer does is read every interrupt with nothing + cancelling it as a refusal: a runner that stopped itself for another reason used to be reported as + a decision the user never made. #### What it costs, measured @@ -230,6 +243,14 @@ Verified live (dev server, `gpt-5-mini`, stepped mode on): returned 204, the child process died, the call was closed as `Tool execution interrupted` so the next attempt is not a poisoned request, the workflow stayed RUNNING, and the next prompt answered normally. + + What the stop does not wait for is the tools. A step's dispatch runs under `allSettled` with the + activity default of `WAIT_CANCELLATION_COMPLETED`, which reads like the turn ends only once the + slowest tool has acknowledged. History says otherwise: with a tool that never finishes, the + workflow recorded `ActivityTaskCancelRequested` and moved on in the same second, and completed two + seconds later with that activity still running. The stop costs one workflow task, not the slowest + cleanup. The tool activity closes its own call on the way out, so the log is not left waiting on + it either. - **An approval holds only the tool that is waiting.** Reading a `*.env` file parks on the default agent's `ask` rule. While the human deliberated, `runModelCall` was **completed** and `runToolCall` was the only outstanding activity. Replying `once` completed the tool and the turn. @@ -285,10 +306,12 @@ inside `Step.Ended` via `startAssistant()`; the seal, running in another process searched the projection, found nothing, and left the turn open forever. The fix carries the assistant message id out of the attempt the same way the tool call ids are carried. -One scenario is there for this split in particular: a turn that asks for a tool, runs it, and goes -back to the model with the result. Everything else in the suite settles without a tool, so the piece -the split actually changes, one activity per call rather than one for the whole step, would have had -no scenario that both modes must pass. +Two scenarios are there for this split in particular. One is a turn that asks for a tool, runs it, +and goes back to the model with the result; the other interrupts a turn with a tool in flight and +demands that no call is left running. Everything else in the suite settles without a tool, so the +piece the split actually changes, one activity per call rather than one for the whole step, would +have had no scenario that both modes must pass. The interrupt one was the more useful of the two: +stepped mode failed it until the dispatch learned to close its own call. #### Worker affinity (`OPENCODE_TEMPORAL_WORKTREE_AFFINITY=1`) @@ -351,12 +374,17 @@ serve process writes itself. A UI attached to serve saw a prompt admitted and th Token deltas are not part of this. `Text.Delta`, `Reasoning.Delta` and `Tool.Input.Delta` are live-only and never reach the durable log, so what crosses a process boundary is block-level: -`step.started`, `tool.called`, `tool.success`, `step.ended`. The durable tail now also re-reads on a -tick (`LayerOptions.livePollInterval`, default one second, 0 to disable). In-process commits -still wake it instantly, so latency is unchanged where it already -worked; the tick only catches what the wake cannot see. An idle subscriber costs one indexed read -per period, and a tick with nothing new emits nothing. The same split now delivers the worker's -`step.started`, `tool.called`, `tool.success` and `step.ended` to a live subscriber. +`step.started`, `tool.called`, `tool.success`, `step.ended`. A durable tail can also re-read on a +tick (`LayerOptions.livePollInterval`). In-process commits still wake it instantly, so latency is +unchanged where it already worked; the tick only catches what the wake cannot see. An idle +subscriber costs one indexed read per period, and a tick with nothing new emits nothing. With it, +the same split delivers the worker's `step.started`, `tool.called`, `tool.success` and `step.ended` +to a live subscriber. + +It is off unless a composition root asks for it, and only the durable executor's does +(`EventV2.pollingNode`, wired in `server/src/routes.ts`). A deployment running in one process wakes +its own subscribers, so a tick there would be a query per second per subscribed session for events +that cannot exist. `OPENCODE_EVENT_POLL_MS` overrides either way, `0` to turn it off. ### Two modes, one runner