From 542c87ba03c88457825dd07c4ffbb7a8ee72740d Mon Sep 17 00:00:00 2001 From: Mike Olson Date: Mon, 20 Jul 2026 11:47:41 -0400 Subject: [PATCH] fix(orchestrator): Harden Grok v2 runtime lifecycle Settle root turns from provider signals and preserve post-settle continuations. Track asynchronous subagent and monitor work, allow image prompts, and keep steered messages visible. Separate soft steering from hard Stop while containing and reaping native process trees. --- .github/workflows/ci.yml | 10 + apps/server/scripts/acp-mock-agent.ts | 437 +- apps/server/scripts/acp-thread-spawn-helper.c | 37 + .../Adapters/AcpAdapterV2.test.ts | 6423 ++++++++++++++++- .../orchestration-v2/Adapters/AcpAdapterV2.ts | 3384 ++++++++- .../Adapters/GrokAdapterV2.test.ts | 255 +- .../Adapters/GrokAdapterV2.ts | 82 +- .../src/orchestration-v2/EffectWorker.test.ts | 40 +- .../src/orchestration-v2/EffectWorker.ts | 33 +- .../src/orchestration-v2/ProjectionStore.ts | 2 + .../src/orchestration-v2/ProviderAdapter.ts | 2 + .../ProviderContinuationRequests.ts | 4 + .../ProviderContinuationService.test.ts | 193 + .../ProviderContinuationService.ts | 13 +- .../ProviderSessionManager.test.ts | 93 + .../ProviderSessionManager.ts | 40 +- .../ProviderTurnControlService.ts | 48 +- .../ProviderTurnStartService.ts | 17 + .../RunExecutionService.test.ts | 190 +- .../orchestration-v2/RunExecutionService.ts | 72 +- .../SelectionRestart.integration.test.ts | 7 +- .../message_steering/cursor_output.ts | 11 +- .../fixtures/message_steering/grok_output.ts | 11 +- .../provider/acp/AcpJsonRpcConnection.test.ts | 34 + .../src/provider/acp/AcpRuntimeModel.test.ts | 93 + .../src/provider/acp/AcpRuntimeModel.ts | 30 + .../acp/AcpSessionRuntime.processTree.test.ts | 1030 +++ .../src/provider/acp/AcpSessionRuntime.ts | 1383 +++- .../src/provider/acp/GrokAcpSupport.test.ts | 24 + .../server/src/provider/acp/GrokAcpSupport.ts | 21 + .../src/provider/acp/XAiAcpExtension.test.ts | 1143 ++- .../src/provider/acp/XAiAcpExtension.ts | 1029 ++- .../web/src/components/ChatView.logic.test.ts | 104 + apps/web/src/components/ChatView.logic.ts | 17 + apps/web/src/components/ChatView.tsx | 11 +- .../src/components/chat/MessagesTimeline.tsx | 34 +- .../src/state/orchestrationV2Projection.ts | 1 + packages/effect-acp/src/client.ts | 10 + packages/effect-acp/src/protocol.test.ts | 200 + packages/effect-acp/src/protocol.ts | 225 +- .../src/orchestrationV2Timeline.test.ts | 35 +- .../shared/src/orchestrationV2Timeline.ts | 20 +- 42 files changed, 15897 insertions(+), 951 deletions(-) create mode 100644 apps/server/scripts/acp-thread-spawn-helper.c create mode 100644 apps/server/src/orchestration-v2/ProviderContinuationService.test.ts create mode 100644 apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21fbce026f5..e01bc5d2096 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,16 @@ jobs: - name: Checkout uses: actions/checkout@v6 + # Blacksmith boots GitHub's Ubuntu runner image (gcc is usually present), + # but ACP process-tree live tests compile a small pthread fixture with `cc` + # and soft-skip when it is missing. Install build-essential so that path + # always runs in CI instead of silently no-oping. + - name: Install C toolchain for process-tree fixtures + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends build-essential + command -v cc + - name: Setup Vite+ uses: voidzero-dev/setup-vp@v1 with: diff --git a/apps/server/scripts/acp-mock-agent.ts b/apps/server/scripts/acp-mock-agent.ts index dce0c261aa3..4494593707c 100644 --- a/apps/server/scripts/acp-mock-agent.ts +++ b/apps/server/scripts/acp-mock-agent.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node // @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; import * as Effect from "effect/Effect"; @@ -17,13 +18,36 @@ const emitToolCalls = process.env.T3_ACP_EMIT_TOOL_CALLS === "1"; const emitInterleavedAssistantToolCalls = process.env.T3_ACP_EMIT_INTERLEAVED_ASSISTANT_TOOL_CALLS === "1"; const emitGenericToolPlaceholders = process.env.T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS === "1"; +const emitPostSettleMonitorFlow = process.env.T3_ACP_EMIT_POST_SETTLE_MONITOR_FLOW === "1"; +const emitInTurnTaskOutputThenLateDuplicate = + process.env.T3_ACP_EMIT_IN_TURN_TASKOUTPUT_THEN_LATE_DUPLICATE === "1"; +const injectedReportTriggerPath = process.env.T3_ACP_INJECTED_REPORT_TRIGGER_PATH; const emitAskQuestion = process.env.T3_ACP_EMIT_ASK_QUESTION === "1"; +const emitElicitation = process.env.T3_ACP_EMIT_ELICITATION === "1"; +const emitUrlElicitation = process.env.T3_ACP_EMIT_URL_ELICITATION === "1"; const emitXAiAskUserQuestion = process.env.T3_ACP_EMIT_XAI_ASK_USER_QUESTION === "1"; const emitXAiPromptCompleteThenHang = process.env.T3_ACP_EMIT_XAI_PROMPT_COMPLETE_THEN_HANG === "1"; const emitForeignSessionUpdates = process.env.T3_ACP_EMIT_FOREIGN_SESSION_UPDATES === "1"; const hangPromptForever = process.env.T3_ACP_HANG_PROMPT_FOREVER === "1"; +const hangAfterPermission = process.env.T3_ACP_HANG_AFTER_PERMISSION === "1"; const hangFirstPromptForever = process.env.T3_ACP_HANG_FIRST_PROMPT_FOREVER === "1"; const emitLateUpdateAfterCancel = process.env.T3_ACP_EMIT_LATE_UPDATE_AFTER_CANCEL === "1"; +const emitTaskBackgroundedAfterCancel = + process.env.T3_ACP_EMIT_TASK_BACKGROUNDED_AFTER_CANCEL === "1"; +const residualCallbackResponseLogPath = process.env.T3_ACP_RESIDUAL_CALLBACK_RESPONSE_LOG_PATH; +const residualCallbackTriggerPath = process.env.T3_ACP_RESIDUAL_CALLBACK_TRIGGER_PATH; +const exitAfterResidualCallbacks = process.env.T3_ACP_EXIT_AFTER_RESIDUAL_CALLBACKS === "1"; +const emitRunningCommandThenHang = process.env.T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG === "1"; +const emitRunningCommandThenHangOnFirstPrompt = + process.env.T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG_FIRST_PROMPT === "1"; +const emitEmptySuccessfulBash = process.env.T3_ACP_EMIT_EMPTY_SUCCESSFUL_BASH === "1"; +const emitEmptySuccessfulBashThenHang = + process.env.T3_ACP_EMIT_EMPTY_SUCCESSFUL_BASH_THEN_HANG === "1"; +const exitOnCancel = process.env.T3_ACP_EXIT_ON_CANCEL === "1"; +const runningCommandIgnoresTerm = process.env.T3_ACP_RUNNING_COMMAND_IGNORE_TERM === "1"; +const runningCommandPidPath = process.env.T3_ACP_RUNNING_COMMAND_PID_PATH; +const runningCommandSeparateSession = process.env.T3_ACP_RUNNING_COMMAND_SEPARATE_SESSION === "1"; +const exitAfterRunningCommandLaunch = process.env.T3_ACP_EXIT_AFTER_RUNNING_COMMAND_LAUNCH === "1"; const omitXAiPromptCompleteStopReason = process.env.T3_ACP_OMIT_XAI_PROMPT_COMPLETE_STOP_REASON === "1"; const failLoadSession = process.env.T3_ACP_FAIL_LOAD_SESSION === "1"; @@ -83,6 +107,11 @@ function writeJsonRpcNotification(method: string, params: unknown): void { process.stdout.write(`${JSON.stringify({ jsonrpc: "2.0", method, params })}\n`); } +function logResidualCallbackResponse(kind: string): void { + if (!residualCallbackResponseLogPath) return; + NodeFS.appendFileSync(residualCallbackResponseLogPath, `${kind}\n`, "utf8"); +} + process.once("SIGTERM", () => { logExit("SIGTERM"); process.exit(0); @@ -481,6 +510,9 @@ const program = Effect.gen(function* () { Effect.gen(function* () { const cancelledSessionId = String(sessionId ?? "mock-session-1"); cancelledSessions.add(cancelledSessionId); + if (exitOnCancel) { + return yield* Effect.sync(() => process.exit(0)); + } if (emitLateUpdateAfterCancel) { yield* Effect.sleep("50 millis"); yield* Effect.sync(() => { @@ -493,6 +525,36 @@ const program = Effect.gen(function* () { }); }); } + if (emitTaskBackgroundedAfterCancel) { + // Grok cancel-as-detach: the foreground command is re-run as a + // background task that later completes on its own. + yield* Effect.sync(() => { + writeJsonRpcNotification("_x.ai/task_backgrounded", { + sessionId: cancelledSessionId, + update: { + sessionUpdate: "task_backgrounded", + tool_call_id: "task-bg-1", + task_id: "task-bg-1", + command: "sleep 30", + }, + }); + }); + yield* Effect.sleep("1200 millis") + .pipe( + Effect.andThen( + Effect.sync(() => { + writeJsonRpcNotification("_x.ai/task_completed", { + sessionId: cancelledSessionId, + update: { + sessionUpdate: "task_completed", + task_snapshot: { task_id: "task-bg-1", command: "sleep 30" }, + }, + }); + }), + ), + ) + .pipe(Effect.forkDetach); + } }), ); @@ -501,6 +563,80 @@ const program = Effect.gen(function* () { const requestedSessionId = String(request.sessionId ?? sessionId); promptCount += 1; + if (residualCallbackTriggerPath !== undefined) { + yield* Effect.gen(function* () { + while (!(yield* Effect.sync(() => NodeFS.existsSync(residualCallbackTriggerPath)))) { + yield* Effect.sleep("20 millis"); + } + yield* Effect.sync(() => { + writeJsonRpcNotification("session/update", { + sessionId: requestedSessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "residual assistant callback" }, + }, + }); + writeJsonRpcNotification("session/update", { + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: "residual-tool-call", + title: "Residual tool callback", + kind: "other", + status: "pending", + rawInput: {}, + }, + }); + writeJsonRpcNotification("session/update", { + sessionId: requestedSessionId, + update: { + sessionUpdate: "plan", + entries: [ + { content: "Residual plan callback", priority: "high", status: "pending" }, + ], + }, + }); + }); + yield* agent.client + .requestPermission({ + sessionId: requestedSessionId, + toolCall: { + toolCallId: "residual-permission", + title: "Residual permission callback", + }, + options: [{ optionId: "allow", name: "Allow", kind: "allow_once" }], + }) + .pipe( + Effect.exit, + Effect.tap(() => Effect.sync(() => logResidualCallbackResponse("permission"))), + Effect.ignore, + Effect.forkDetach, + ); + yield* agent.client + .elicit({ + sessionId: requestedSessionId, + message: "Residual elicitation callback", + mode: "form", + requestedSchema: { + type: "object", + properties: { + approved: { type: "boolean", title: "Approved" }, + }, + }, + }) + .pipe( + Effect.exit, + Effect.tap(() => Effect.sync(() => logResidualCallbackResponse("elicitation"))), + Effect.ignore, + Effect.forkDetach, + ); + if (exitAfterResidualCallbacks) { + yield* Effect.sleep("100 millis"); + return yield* Effect.sync(() => process.exit(0)); + } + }).pipe(Effect.forkDetach); + } + if (Number.isFinite(promptDelayMs) && promptDelayMs > 0) { yield* Effect.sleep(`${promptDelayMs} millis`); } @@ -562,10 +698,126 @@ const program = Effect.gen(function* () { return yield* Effect.never; } - if (hangPromptForever || (hangFirstPromptForever && promptCount === 1)) { + if ( + hangPromptForever || + (hangFirstPromptForever && promptCount === 1) || + (emitEmptySuccessfulBashThenHang && promptCount === 2) + ) { return yield* Effect.never; } + if ( + emitRunningCommandThenHang || + (emitRunningCommandThenHangOnFirstPrompt && promptCount === 1) + ) { + const toolCallId = "tool-call-running-1"; + if (runningCommandPidPath !== undefined) { + const command = runningCommandIgnoresTerm + ? 'trap "" TERM; bash -c \'trap "" TERM; while :; do sleep 1; done\' & child=$!; printf "%s %s\\n" "$$" "$child" > "$1"; wait "$child"' + : 'sleep 120 & child=$!; printf "%s %s\\n" "$$" "$child" > "$1"; wait "$child"'; + if (runningCommandSeparateSession) { + const launcher = [ + 'const { spawn } = require("node:child_process");', + "const child = spawn(process.argv[1], process.argv.slice(2), { stdio: 'ignore' });", + "child.once('exit', (code, signal) => process.exitCode = code ?? (signal ? 1 : 0));", + ].join(" "); + const detachedCommand = runningCommandIgnoresTerm + ? 'trap "" TERM; bash -c \'trap "" TERM; while :; do sleep 1; done\' & child=$!; printf "%s %s %s\\n" "$PPID" "$$" "$child" > "$1"; wait "$child"' + : 'sleep 120 & child=$!; printf "%s %s %s\\n" "$PPID" "$$" "$child" > "$1"; wait "$child"'; + // Nested bash publishes "$PPID $$ $child" once it starts. Do not + // write the launcher PID alone here: that races with bash and can + // clobber the triple that interrupt tests wait for. + const detachedLauncher = NodeChildProcess.spawn( + process.execPath, + ["-e", launcher, "bash", "-c", detachedCommand, "bash", runningCommandPidPath], + { detached: true, stdio: "ignore" }, + ); + detachedLauncher.unref(); + } else { + NodeChildProcess.spawn("bash", ["-c", command, "bash", runningCommandPidPath], { + stdio: "ignore", + }); + } + } + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call", + toolCallId, + title: "Terminal", + kind: "execute", + status: "pending", + rawInput: { + command: ["sleep", "120"], + }, + }, + }); + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId, + title: "Terminal", + kind: "execute", + status: "in_progress", + rawInput: { + command: ["sleep", "120"], + }, + // Grok-like mid-stream Bash re-report: exit_code 0 while still running. + rawOutput: { type: "Bash", exit_code: 0 }, + }, + }); + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-call-output-1", + title: "get_command_or_subagent_output", + kind: "other", + status: "pending", + rawInput: { task_id: "task-running-1" }, + }, + }); + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-output-1", + status: "in_progress", + rawOutput: { task_id: "task-running-1", status: "running" }, + }, + }); + if (exitAfterRunningCommandLaunch) { + yield* Effect.sleep("100 millis"); + return yield* Effect.sync(() => process.exit(0)); + } + // Stay open until session/cancel so interrupt tests can observe a running tool. + while (!cancelledSessions.has(requestedSessionId)) { + yield* Effect.sleep("25 millis"); + } + cancelledSessions.delete(requestedSessionId); + return { stopReason: "cancelled" }; + } + + if (emitEmptySuccessfulBash || (emitEmptySuccessfulBashThenHang && promptCount === 1)) { + const update = { + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-empty-success-1", + title: "Terminal", + kind: "execute", + status: "completed", + rawInput: { command: "true" }, + rawOutput: { type: "Bash", exit_code: 0 }, + }, + } as const; + yield* agent.client.sessionUpdate(update); + yield* Effect.sleep("25 millis"); + yield* agent.client.sessionUpdate(update); + return { stopReason: "end_turn" }; + } + if (emitXAiPromptCompleteThenHang) { writeJsonRpcNotification("session/update", { sessionId: requestedSessionId, @@ -674,6 +926,32 @@ const program = Effect.gen(function* () { return { stopReason: "end_turn" }; } + if (emitElicitation) { + yield* agent.client.elicit({ + sessionId: requestedSessionId, + message: "Approve this request?", + mode: "form", + requestedSchema: { + type: "object", + properties: { + approved: { type: "boolean", title: "Approved" }, + }, + }, + }); + return { stopReason: "end_turn" }; + } + + if (emitUrlElicitation) { + yield* agent.client.elicit({ + sessionId: requestedSessionId, + message: "Open authentication page", + mode: "url", + url: "https://example.com/auth", + elicitationId: "url-elicitation-1", + }); + return { stopReason: "end_turn" }; + } + if (emitToolCalls) { const toolCallId = "tool-call-1"; @@ -732,6 +1010,10 @@ const program = Effect.gen(function* () { cancelledSessions.delete(requestedSessionId) || permission.outcome.outcome === "cancelled"; + if (hangAfterPermission) { + return yield* Effect.never; + } + yield* agent.client.sessionUpdate({ sessionId: requestedSessionId, update: { @@ -798,6 +1080,159 @@ const program = Effect.gen(function* () { return { stopReason: "end_turn" }; } + // In-turn monitor + TaskOutput hydrate, then a late post-finalize + // duplicate terminal TaskOutput for the same task. Exercises the + // already-handled short-circuit: must not pin hasPendingBackgroundWork. + if (emitInTurnTaskOutputThenLateDuplicate) { + const monitorToolCallId = "tool-call-monitor-1"; + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: monitorToolCallId, + title: "Monitor: mock background task", + kind: "execute", + status: "pending", + rawInput: {}, + }, + }); + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: monitorToolCallId, + status: "in_progress", + }, + }); + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-call-fetch-1", + title: "get_command_or_subagent_output", + kind: "other", + status: "pending", + rawInput: { task_id: "task-monitor-1" }, + }, + }); + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-fetch-1", + status: "completed", + rawOutput: { output: "MONITOR_LISTING_TOKEN" }, + }, + }); + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "Monitor listing ready in-turn." }, + }, + }); + // After deferred finalize (~2s) clears activeTurn, re-emit a terminal + // TaskOutput for the same task so bufferPostSettleWake sees + // alreadyHandledToolUpdate with a non-empty wake path. + yield* Effect.gen(function* () { + yield* Effect.sleep("2500 millis"); + yield* Effect.sync(() => { + writeJsonRpcNotification("session/update", { + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-fetch-1", + title: "get_command_or_subagent_output", + kind: "other", + status: "completed", + rawOutput: { output: "MONITOR_LISTING_TOKEN_LATE" }, + }, + }); + }); + }).pipe(Effect.forkDetach); + return { stopReason: "end_turn" }; + } + + if (emitPostSettleMonitorFlow) { + const monitorToolCallId = "tool-call-monitor-1"; + + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: monitorToolCallId, + title: "Monitor: mock background task", + kind: "execute", + status: "pending", + rawInput: {}, + }, + }); + + yield* agent.client.sessionUpdate({ + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: monitorToolCallId, + status: "in_progress", + }, + }); + + // After the prompt settles, replay the CLI-injected monitor-event + // turn: end notice, TaskOutput hydration, then (once the trigger + // file exists) the report chunk. Detached fiber on the real clock; + // it outlives the prompt handler. + yield* Effect.gen(function* () { + yield* Effect.sleep("150 millis"); + yield* Effect.sync(() => { + writeJsonRpcNotification("session/update", { + sessionId: requestedSessionId, + update: { + sessionUpdate: "user_message_chunk", + content: { + type: "text", + text: 'Monitor "task-monitor-1" ended: [monitor ended: exit 0]', + }, + }, + }); + writeJsonRpcNotification("session/update", { + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-call-fetch-1", + title: "get_command_or_subagent_output", + kind: "other", + status: "pending", + rawInput: { task_id: "task-monitor-1" }, + }, + }); + writeJsonRpcNotification("session/update", { + sessionId: requestedSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-fetch-1", + status: "completed", + rawOutput: { output: "MONITOR_LISTING_TOKEN" }, + }, + }); + }); + if (injectedReportTriggerPath === undefined) return; + while (!(yield* Effect.sync(() => NodeFS.existsSync(injectedReportTriggerPath)))) { + yield* Effect.sleep("20 millis"); + } + yield* Effect.sync(() => { + writeJsonRpcNotification("session/update", { + sessionId: requestedSessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "Monitor finished. MONITOR_REPORT_TOKEN" }, + }, + }); + }); + }).pipe(Effect.forkDetach); + + return { stopReason: "end_turn" }; + } + if (emitAskQuestion) { yield* agent.client.extRequest("cursor/ask_question", { toolCallId: "ask-question-tool-call-1", diff --git a/apps/server/scripts/acp-thread-spawn-helper.c b/apps/server/scripts/acp-thread-spawn-helper.c new file mode 100644 index 00000000000..f5b84ee6d98 --- /dev/null +++ b/apps/server/scripts/acp-thread-spawn-helper.c @@ -0,0 +1,37 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include + +static const char *pid_path; + +static void *spawn_child(void *unused) { + (void)unused; + pid_t child = fork(); + if (child == 0) { + execlp("sleep", "sleep", "120", NULL); + _exit(127); + } + if (child < 0) return NULL; + FILE *file = fopen(pid_path, "w"); + if (file != NULL) { + fprintf(file, "%ld %d\n", syscall(SYS_gettid), child); + fclose(file); + } + waitpid(child, NULL, 0); + return NULL; +} + +int main(int argc, char **argv) { + if (argc != 2) return 2; + pid_path = argv[1]; + pthread_t worker; + if (pthread_create(&worker, NULL, spawn_child, NULL) != 0) return 3; + if (pthread_join(worker, NULL) != 0) return 4; + return 0; +} diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index ffdb8d15afd..e9d0d20b95e 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -1,3 +1,7 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; + import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; import { @@ -14,32 +18,58 @@ import { ThreadId, type OrchestrationV2ProviderThread, } from "@t3tools/contracts"; -import * as DateTime from "effect/DateTime"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import type * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; +import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; +import * as Scope from "effect/Scope"; import * as TestClock from "effect/testing/TestClock"; import { ChildProcessSpawner } from "effect/unstable/process"; +import * as EffectAcpErrors from "effect-acp/errors"; import type * as EffectAcpProtocol from "effect-acp/protocol"; +import type * as EffectAcpSchema from "effect-acp/schema"; import { ServerConfig } from "../../config.ts"; import * as AcpSessionRuntime from "../../provider/acp/AcpSessionRuntime.ts"; +import { + normalizeXAiAcpToolCallState, + registerXAiBackgroundTaskTracking, +} from "../../provider/acp/XAiAcpExtension.ts"; import { layer as idAllocatorLayer, IdAllocatorV2 } from "../IdAllocator.ts"; import { ProviderAdapterV2RuntimePolicy, + type ProviderAdapterV2Event, type ProviderAdapterV2TurnInput, } from "../ProviderAdapter.ts"; +import type { ProviderContinuationRequest } from "../ProviderContinuationRequests.ts"; import { AcpProviderCapabilitiesV2, + acpCanonicalJson, + acpClaimNativeTransportRequest, + acpNativeUserInputRequestMatches, + acpPostSettleContinuationOfferEvidence, + acpPostSettleMonitorPromptShouldSuppress, + acpPostSettleWakeEvidence, + acpPostSettleWakeShouldBuffer, + acpProjectedCommandExitCode, makeAcpAdapterV2, + type AcpAdapterV2ExtensionContext, type AcpAdapterV2Flavor, + type AcpAdapterV2RuntimeInput, } from "./AcpAdapterV2.ts"; const serverConfigLayer = ServerConfig.layerTest(process.cwd(), { @@ -50,14 +80,202 @@ const testLayer = Layer.mergeAll(NodeServices.layer, idAllocatorLayer, serverCon const ACP_TEST_DRIVER = ProviderDriverKind.make("acp-test"); const decodeUnknownJson = Schema.decodeUnknownOption(Schema.UnknownFromJsonString); +describe("acpProjectedCommandExitCode", () => { + const successOutput = { type: "Bash", exit_code: 0 }; + const failedOutput = { type: "Bash", exit_code: 1 }; + + it("omits exit codes for non-terminal and interrupted tool statuses", () => { + assert.equal(acpProjectedCommandExitCode("pending", successOutput), undefined); + assert.equal(acpProjectedCommandExitCode("running", successOutput), undefined); + assert.equal(acpProjectedCommandExitCode("interrupted", successOutput), undefined); + }); + + it("projects real exit codes only for completed and failed tools", () => { + assert.equal(acpProjectedCommandExitCode("completed", successOutput), 0); + assert.equal(acpProjectedCommandExitCode("completed", failedOutput), 1); + assert.equal(acpProjectedCommandExitCode("failed", failedOutput), 1); + assert.equal(acpProjectedCommandExitCode("completed", {}), undefined); + }); +}); + +const taskkillPlatformError = (method: string) => + PlatformError.systemError({ _tag: "Unknown", module: "taskkill-test", method }); + +function makeTaskkillSpawner(input: { + readonly exitCode?: number; + readonly exitFailure?: boolean; + readonly output?: string; + readonly outputFailure?: boolean; + readonly spawnFailure?: boolean; + readonly commands?: Array<{ readonly command: string; readonly args: ReadonlyArray }>; +}) { + return ChildProcessSpawner.make((command) => { + const value = command as unknown as { + readonly command: string; + readonly args: ReadonlyArray; + }; + input.commands?.push({ command: value.command, args: value.args }); + if (input.spawnFailure === true) return Effect.fail(taskkillPlatformError("spawn")); + const output = input.outputFailure + ? Stream.fail(taskkillPlatformError("output")) + : Stream.encodeText(Stream.make(input.output ?? "")); + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1234), + exitCode: input.exitFailure + ? Effect.fail(taskkillPlatformError("exitCode")) + : Effect.succeed(ChildProcessSpawner.ExitCode(input.exitCode ?? 0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.empty, + stderr: Stream.empty, + all: output, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ); + }); +} + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +const waitForProcesses = (pids: ReadonlyArray) => + Effect.gen(function* () { + while (!pids.every(processExists)) { + yield* Effect.sleep("10 millis"); + } + }).pipe(Effect.timeoutOption("2 seconds")); + +const waitForProcessesToExit = (pids: ReadonlyArray) => + Effect.gen(function* () { + while (pids.some(processExists)) { + yield* Effect.sleep("10 millis"); + } + }).pipe(Effect.timeoutOption("2 seconds")); + +function linuxProcessStart(pid: number): string | undefined { + try { + const stat = NodeFS.readFileSync(`/proc/${pid}/stat`, "utf8"); + const commandEnd = stat.lastIndexOf(")"); + return commandEnd < 0 + ? undefined + : stat + .slice(commandEnd + 2) + .trim() + .split(/\s+/)[19]; + } catch { + return undefined; + } +} + +function cleanupPublishedDetachedFixture(path: string): void { + let published: Array; + try { + published = NodeFS.readFileSync(path, "utf8") + .trim() + .split(/\s+/) + .map(Number) + .filter((pid) => Number.isSafeInteger(pid) && pid > 1); + } catch { + return; + } + const roots = published.filter((pid) => { + try { + return NodeFS.readFileSync(`/proc/${pid}/cmdline`, "utf8").includes(path); + } catch { + return false; + } + }); + const owned = new Map(); + const pending = [...roots]; + while (pending.length > 0) { + const pid = pending.shift(); + if (pid === undefined || owned.has(pid)) continue; + const start = linuxProcessStart(pid); + if (start === undefined) continue; + owned.set(pid, start); + try { + pending.push( + ...NodeFS.readFileSync(`/proc/${pid}/task/${pid}/children`, "utf8") + .trim() + .split(/\s+/) + .filter(Boolean) + .map(Number), + ); + } catch { + // The process already exited. + } + } + for (const [pid, start] of [...owned.entries()].toReversed()) { + if (linuxProcessStart(pid) !== start) continue; + try { + process.kill(pid, "SIGKILL"); + } catch { + // The exact fixture process already exited. + } + } +} + +const waitForPublishedProcessIds = ( + fileSystem: FileSystem.FileSystem, + path: string, + count: number, +) => + Effect.gen(function* () { + while (true) { + const ids = (yield* fileSystem.readFileString(path)) + .trim() + .split(/\s+/) + .filter(Boolean) + .map(Number); + if (ids.length === count && ids.every((pid) => Number.isSafeInteger(pid) && pid > 1)) { + return ids; + } + yield* Effect.sleep("10 millis"); + } + }).pipe(Effect.timeoutOption("2 seconds")); + function makeMockRuntime(input: { readonly childProcessSpawner: ChildProcessSpawner.ChildProcessSpawner["Service"]; readonly mockAgentPath: string; - readonly environment?: Readonly>; + readonly environment?: + | Readonly> + | ((runtimeOrdinal: number) => Readonly>); readonly protocolEvents?: Queue.Queue; + readonly ownDescendantProcessGroups?: boolean; + readonly ownDetachedProcessGroup?: boolean; + readonly processGroupPlatform?: NodeJS.Platform; + readonly processGroupTerminationGrace?: Duration.Input; + readonly linuxCgroupController?: AcpSessionRuntime.AcpSessionRuntimeOptions["linuxCgroupController"]; + readonly posixProcessTreeController?: AcpSessionRuntime.AcpSessionRuntimeOptions["posixProcessTreeController"]; + readonly windowsProcessTreeTerminator?: AcpSessionRuntime.AcpSessionRuntimeOptions["windowsProcessTreeTerminator"]; + readonly wrapCancel?: ( + cancel: AcpSessionRuntime.AcpSessionRuntime["Service"]["cancel"], + ) => AcpSessionRuntime.AcpSessionRuntime["Service"]["cancel"]; + readonly wrapOutgoingResponse?: ( + onOutgoingResponse: NonNullable, + ) => NonNullable; + readonly wrapIncomingRequest?: ( + onIncomingRequest: NonNullable, + ) => NonNullable; + readonly wrapRuntime?: ( + runtime: AcpSessionRuntime.AcpSessionRuntime["Service"], + runtimeOrdinal: number, + ) => AcpSessionRuntime.AcpSessionRuntime["Service"]; }): AcpAdapterV2Flavor["makeRuntime"] { + let runtimeOrdinal = 0; return (runtimeInput) => Effect.gen(function* () { + runtimeOrdinal += 1; const protocolEvents = input.protocolEvents; const protocolLogging = protocolEvents === undefined @@ -73,23 +291,66 @@ function makeMockRuntime(input: { const context = yield* Layer.build( AcpSessionRuntime.layer({ ...runtimeInput, + ...(input.ownDetachedProcessGroup === undefined + ? {} + : { ownDetachedProcessGroup: input.ownDetachedProcessGroup }), + ...(input.ownDescendantProcessGroups === undefined + ? {} + : { ownDescendantProcessGroups: input.ownDescendantProcessGroups }), + ...(input.ownDetachedProcessGroup === true + ? { processGroupPlatform: input.processGroupPlatform ?? "linux" } + : {}), + ...(input.processGroupTerminationGrace === undefined + ? {} + : { processGroupTerminationGrace: input.processGroupTerminationGrace }), + ...(input.linuxCgroupController === undefined + ? {} + : { linuxCgroupController: input.linuxCgroupController }), + ...(input.posixProcessTreeController === undefined + ? {} + : { posixProcessTreeController: input.posixProcessTreeController }), + ...(input.windowsProcessTreeTerminator === undefined + ? {} + : { windowsProcessTreeTerminator: input.windowsProcessTreeTerminator }), protocolLogging, spawn: { command: process.execPath, args: [input.mockAgentPath], cwd: runtimeInput.cwd, - env: { T3_ACP_SESSION_LIFECYCLE: "1", ...input.environment }, + env: { + T3_ACP_SESSION_LIFECYCLE: "1", + ...(typeof input.environment === "function" + ? input.environment(runtimeOrdinal) + : input.environment), + }, }, authMethodId: "test", + ...(input.wrapIncomingRequest === undefined || + runtimeInput.onIncomingRequest === undefined + ? {} + : { + onIncomingRequest: input.wrapIncomingRequest(runtimeInput.onIncomingRequest), + }), + ...(input.wrapOutgoingResponse === undefined || + runtimeInput.onOutgoingResponse === undefined + ? {} + : { + onOutgoingResponse: input.wrapOutgoingResponse(runtimeInput.onOutgoingResponse), + }), }).pipe( Layer.provide( Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, input.childProcessSpawner), ), ), ); - return yield* Effect.service(AcpSessionRuntime.AcpSessionRuntime).pipe( + const runtime = yield* Effect.service(AcpSessionRuntime.AcpSessionRuntime).pipe( Effect.provide(context), ); + const wrapped = { + ...runtime, + ...(input.wrapCancel === undefined ? {} : { cancel: input.wrapCancel(runtime.cancel) }), + }; + return input.wrapRuntime?.(wrapped, runtimeOrdinal) ?? wrapped; }); } @@ -107,6 +368,20 @@ function rawProtocolMethod(event: EffectAcpProtocol.AcpProtocolLogEvent): string return undefined; } +const pollProtocolMethods = (events: Queue.Queue) => + Effect.gen(function* () { + const methods: string[] = []; + let polled = 0; + let event = yield* Queue.poll(events); + while (Option.isSome(event) && polled < 256) { + polled += 1; + const method = rawProtocolMethod(event.value); + if (method !== undefined) methods.push(method); + event = yield* Queue.poll(events); + } + return methods; + }); + function makeTurnInput(input: { readonly threadId: ThreadId; readonly providerThread: OrchestrationV2ProviderThread; @@ -165,7 +440,131 @@ function makeTurnInput(input: { } describe("AcpAdapterV2", () => { - it.effect("negotiates and executes optional native session forks through the ACP runtime", () => + it.live("cleans detached fixtures when an assertion aborts the test scope", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + let published: ReadonlyArray = []; + const failed = yield* Effect.scoped( + Effect.gen(function* () { + const commandPidPath = yield* fileSystem.makeTempFileScoped({ + prefix: "t3-acp-forced-failure-command-", + }); + yield* Effect.addFinalizer(() => + Effect.sync(() => cleanupPublishedDetachedFixture(commandPidPath)), + ); + const fixture = NodeChildProcess.spawn( + "bash", + [ + "-c", + 'sleep 120 & child=$!; printf "%s %s\\n" "$$" "$child" > "$1"; wait "$child"', + "bash", + commandPidPath, + ], + { detached: true, stdio: "ignore" }, + ); + fixture.unref(); + published = Option.getOrThrow( + yield* waitForPublishedProcessIds(fileSystem, commandPidPath, 2), + ); + return yield* Effect.fail("forced assertion failure"); + }), + ).pipe(Effect.exit); + + assert.isTrue(Exit.isFailure(failed)); + assert.isTrue( + Option.isSome(yield* waitForProcessesToExit(published)), + "detached cleanup finalizer must reap the Bash and sleep fixture", + ); + }).pipe(Effect.provide(testLayer)), + ); + + it("matches xAI native request identities exactly across shared prefixes", () => { + const request = { + nativeMethod: "x.ai/ask_user_question", + nativeRequestId: "1", + nativeSessionId: "session-1", + }; + assert.isTrue( + acpNativeUserInputRequestMatches(request, { + method: "x.ai/ask_user_question", + payload: { sessionId: "session-1", toolCallId: 1 }, + }), + ); + assert.isFalse( + acpNativeUserInputRequestMatches(request, { + method: "x.ai/ask_user_question", + payload: { sessionId: "session-1", toolCallId: "10", note: "request 1" }, + }), + ); + for (const incomplete of [ + { ...request, nativeMethod: "" }, + { ...request, nativeRequestId: "" }, + { ...request, nativeSessionId: "" }, + ]) { + assert.isFalse( + acpNativeUserInputRequestMatches(incomplete, { + method: "x.ai/ask_user_question", + payload: { sessionId: "session-1", toolCallId: "1" }, + }), + ); + } + assert.isFalse( + acpNativeUserInputRequestMatches(request, { + method: "_x.ai/ask_user_question", + payload: { sessionId: "session-1", toolCallId: "1" }, + }), + ); + assert.isFalse( + acpNativeUserInputRequestMatches(request, { + method: "x.ai/ask_user_question", + payload: { + method: "x.ai/ask_user_question", + params: { sessionId: "session-10", toolCallId: "1" }, + }, + }), + ); + }); + + it("claims concurrent identical native requests in per-runtime sequence order", () => { + const requests = [ + { generation: 2, requestId: "second", sequence: 8, identity: "shared" }, + { generation: 1, requestId: "stale", sequence: 1, identity: "shared" }, + { generation: 2, requestId: "first", sequence: 7, identity: "shared" }, + { generation: 2, requestId: "other", sequence: 6, identity: "other" }, + ]; + const [firstId, afterFirst] = acpClaimNativeTransportRequest( + requests, + 2, + (request) => request.identity === "shared", + ); + const [secondId, afterSecond] = acpClaimNativeTransportRequest( + afterFirst, + 2, + (request) => request.identity === "shared", + ); + + assert.equal(firstId, "first"); + assert.equal(secondId, "second"); + assert.deepEqual( + afterSecond.map((request) => request.requestId), + ["stale", "other"], + ); + }); + + it("canonicalizes nested elicitation schemas independently of object key order", () => { + assert.equal( + acpCanonicalJson({ + type: "object", + properties: { answer: { type: "string", title: "Answer", enum: ["a", "b"] } }, + }), + acpCanonicalJson({ + properties: { answer: { enum: ["a", "b"], title: "Answer", type: "string" } }, + type: "object", + }), + ); + }); + + it.live("replaces an unexpectedly terminated ACP runtime before the next turn", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; @@ -175,23 +574,33 @@ describe("AcpAdapterV2", () => { const mockAgentPath = yield* path.fromFileUrl( new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); - const makeRuntime = makeMockRuntime({ childProcessSpawner, mockAgentPath }); - - const instanceId = ProviderInstanceId.make("acp-test"); + const runtimeInputs: AcpAdapterV2RuntimeInput[] = []; + let runtimeOrdinalSeen = 0; + const baseMakeRuntime = makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = runtimeOrdinal; + return {}; + }, + }); + const instanceId = ProviderInstanceId.make("acp-test-unexpected-termination"); const adapter = makeAcpAdapterV2({ crypto: yield* Crypto.Crypto, instanceId, flavor: { driver: ACP_TEST_DRIVER, capabilities: AcpProviderCapabilitiesV2, - makeRuntime, + makeRuntime: (runtimeInput) => + Effect.sync(() => { + runtimeInputs.push(runtimeInput); + }).pipe(Effect.andThen(baseMakeRuntime(runtimeInput))), }, fileSystem, idAllocator, serverConfig, }); - const sourceThreadId = ThreadId.make("thread-acp-native-fork-source"); - const targetThreadId = ThreadId.make("thread-acp-native-fork-target"); + const threadId = ThreadId.make("thread-acp-unexpected-termination"); const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ runtimeMode: "full-access", interactionMode: "default", @@ -199,33 +608,43 @@ describe("AcpAdapterV2", () => { }); const modelSelection = { instanceId, model: "default" } as const; const runtime = yield* adapter.openSession({ - threadId: sourceThreadId, - providerSessionId: ProviderSessionId.make("provider-session-acp-native-fork"), + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-unexpected-termination"), modelSelection, runtimePolicy, }); - - assert.isTrue(runtime.providerSession.capabilities.threads.canForkThread); - assert.isTrue(runtime.providerSession.capabilities.threads.canReadThreadSnapshot); - - const sourceProviderThread = yield* runtime.ensureThread({ - threadId: sourceThreadId, + const providerThread = yield* runtime.ensureThread({ + threadId, modelSelection, runtimePolicy, }); - const forkedProviderThread = yield* runtime.forkThread({ - sourceProviderThread, - targetThreadId, - }); + assert.equal(runtimeOrdinalSeen, 1); + yield* runtimeInputs[0]!.onTermination!( + new EffectAcpErrors.AcpTransportError({ + detail: "Injected unexpected writer termination", + cause: "test", + }), + ); - assert.equal(sourceProviderThread.nativeThreadRef?.nativeId, "mock-session-1"); - assert.equal(forkedProviderThread.nativeThreadRef?.nativeId, "mock-session-1-fork"); - assert.equal(forkedProviderThread.appThreadId, targetThreadId); - assert.equal(forkedProviderThread.forkedFrom?.providerThreadId, sourceProviderThread.id); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + yield* runtime.events.pipe( + Stream.filter((event) => event.type === "turn.terminal"), + Stream.runHead, + ); + assert.equal(runtimeOrdinalSeen, 2); + assert.lengthOf(runtimeInputs, 2); }).pipe(Effect.provide(testLayer), Effect.scoped), ); - it.effect("rejects requested options that the active ACP session does not expose", () => + it.live("reaps detached native work when the provider exits before explicit teardown", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; @@ -235,45 +654,79 @@ describe("AcpAdapterV2", () => { const mockAgentPath = yield* path.fromFileUrl( new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); - const instanceId = ProviderInstanceId.make("acp-test"); + const commandPidPath = yield* fileSystem.makeTempFileScoped({ + prefix: "t3-acp-provider-exit-command-", + }); + yield* Effect.addFinalizer(() => + Effect.sync(() => cleanupPublishedDetachedFixture(commandPidPath)), + ); + const instanceId = ProviderInstanceId.make("acp-test-provider-exit"); const adapter = makeAcpAdapterV2({ crypto: yield* Crypto.Crypto, instanceId, flavor: { driver: ACP_TEST_DRIVER, capabilities: AcpProviderCapabilitiesV2, - makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath }), + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + ownDescendantProcessGroups: true, + ownDetachedProcessGroup: true, + processGroupTerminationGrace: 0, + environment: { + T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG: "1", + T3_ACP_EXIT_AFTER_RUNNING_COMMAND_LAUNCH: "1", + T3_ACP_RUNNING_COMMAND_PID_PATH: commandPidPath, + T3_ACP_RUNNING_COMMAND_SEPARATE_SESSION: "1", + }, + }), }, fileSystem, idAllocator, serverConfig, }); - const threadId = ThreadId.make("thread-acp-unsupported-option"); + const threadId = ThreadId.make("thread-acp-provider-exit-running-command"); const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ runtimeMode: "full-access", interactionMode: "default", cwd: process.cwd(), }); - const error = yield* adapter - .openSession({ - threadId, - providerSessionId: ProviderSessionId.make("provider-session-acp-unsupported-option"), - modelSelection: { + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-provider-exit"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime + .startTurn( + makeTurnInput({ + threadId, + providerThread, instanceId, - model: "default", - options: [{ id: "missing-option", value: "high" }], - }, - runtimePolicy, - }) - .pipe(Effect.flip); + runtimePolicy, + now: yield* DateTime.now, + }), + ) + .pipe(Effect.exit, Effect.forkScoped); - assert.equal(error._tag, "ProviderAdapterOpenSessionError"); - assert.include(String(error.cause), "does not expose requested configuration option(s)"); - assert.include(String(error.cause), "missing-option"); + const published = yield* waitForPublishedProcessIds(fileSystem, commandPidPath, 3); + assert.isTrue(Option.isSome(published), "detached fixture must publish all process IDs"); + const pids = Option.getOrThrow(published); + assert.isTrue( + Option.isSome(yield* waitForProcessesToExit(pids)), + "provider termination must reap the detached launcher, Bash, and sleep processes", + ); }).pipe(Effect.provide(testLayer), Effect.scoped), ); - it.effect("reconfigures a loaded ACP session from its own active setup metadata", () => + it.live("surfaces reduced guarantee when delegated cgroup containment is unavailable", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; @@ -283,101 +736,51 @@ describe("AcpAdapterV2", () => { const mockAgentPath = yield* path.fromFileUrl( new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); - const protocolEvents = yield* Queue.bounded(256); - const instanceId = ProviderInstanceId.make("acp-test"); + let containment: + | AcpSessionRuntime.AcpSessionRuntime["Service"]["processContainment"] + | undefined; + const instanceId = ProviderInstanceId.make("acp-test-cgroup-unavailable"); const adapter = makeAcpAdapterV2({ crypto: yield* Crypto.Crypto, instanceId, flavor: { driver: ACP_TEST_DRIVER, capabilities: AcpProviderCapabilitiesV2, - makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath, protocolEvents }), + makeRuntime: makeMockRuntime({ + childProcessSpawner, + linuxCgroupController: null, + mockAgentPath, + ownDescendantProcessGroups: true, + ownDetachedProcessGroup: true, + wrapRuntime: (runtime) => { + containment = runtime.processContainment; + return runtime; + }, + }), }, fileSystem, idAllocator, serverConfig, }); - const firstThreadId = ThreadId.make("thread-acp-active-setup:first"); - const secondThreadId = ThreadId.make("thread-acp-active-setup:second"); + const threadId = ThreadId.make("thread-acp-cgroup-unavailable"); const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ runtimeMode: "full-access", interactionMode: "default", cwd: process.cwd(), }); - const initialSelection = { instanceId, model: "default" } satisfies ModelSelection; - const alternateSelection = { - instanceId, - model: "grok-mock-alt", - } satisfies ModelSelection; - const originalSelection = { instanceId, model: "grok-build" } satisfies ModelSelection; + const modelSelection = { instanceId, model: "default" } as const; const runtime = yield* adapter.openSession({ - threadId: firstThreadId, - providerSessionId: ProviderSessionId.make("provider-session-acp-active-setup"), - modelSelection: initialSelection, - runtimePolicy, - }); - const firstProviderThread = yield* runtime.ensureThread({ - threadId: firstThreadId, - modelSelection: initialSelection, - runtimePolicy, - }); - const now = yield* DateTime.now; - yield* runtime.startTurn( - makeTurnInput({ - threadId: firstThreadId, - providerThread: firstProviderThread, - instanceId, - runtimePolicy, - modelSelection: alternateSelection, - now, - }), - ); - yield* runtime.events.pipe( - Stream.filter((event) => event.type === "turn.terminal"), - Stream.runHead, - ); - - const secondProviderThread: OrchestrationV2ProviderThread = { - ...firstProviderThread, - id: ProviderThreadId.make("provider-thread-acp-active-setup:second"), - appThreadId: secondThreadId, - nativeThreadRef: { - driver: ACP_TEST_DRIVER, - nativeId: "mock-session-2", - strength: "strong", - }, - status: "idle", - }; - yield* runtime.resumeThread({ - providerThread: secondProviderThread, - modelSelection: alternateSelection, + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-cgroup-unavailable"), + modelSelection, runtimePolicy, }); - yield* runtime.startTurn( - makeTurnInput({ - threadId: secondThreadId, - providerThread: secondProviderThread, - instanceId, - runtimePolicy, - modelSelection: originalSelection, - now, - ordinal: 2, - }), - ); - yield* runtime.events.pipe( - Stream.filter((event) => event.type === "turn.terminal"), - Stream.runHead, - ); - - const setModelRequests = Array.from(yield* Queue.takeAll(protocolEvents)).filter( - (event) => - event.direction === "outgoing" && rawProtocolMethod(event) === "session/set_model", - ); - assert.lengthOf(setModelRequests, 2); + yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy }); + assert.equal(containment, "process-ledger-reduced-guarantee"); }).pipe(Effect.provide(testLayer), Effect.scoped), ); - it.effect("cancels pending permission requests while interrupting an ACP turn", () => + it.live("cleans a cgroup lease when the pre-exec join wrapper fails", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; @@ -387,7 +790,30 @@ describe("AcpAdapterV2", () => { const mockAgentPath = yield* path.fromFileUrl( new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); - const instanceId = ProviderInstanceId.make("acp-test"); + let createCalls = 0; + let cgroupExists = true; + let killCalls = 0; + let removeCalls = 0; + const cgroupController: AcpSessionRuntime.AcpLinuxCgroupController = { + create: () => { + createCalls += 1; + return { + contains: () => false, + exists: () => cgroupExists, + path: "/definitely-missing/t3-acp-cgroup", + relativePath: "/definitely-missing/t3-acp-cgroup", + kill: () => { + killCalls += 1; + }, + populated: () => false, + remove: () => { + removeCalls += 1; + cgroupExists = false; + }, + }; + }, + }; + const instanceId = ProviderInstanceId.make("acp-test-cgroup-join-failure"); const adapter = makeAcpAdapterV2({ crypto: yield* Crypto.Crypto, instanceId, @@ -396,74 +822,5319 @@ describe("AcpAdapterV2", () => { capabilities: AcpProviderCapabilitiesV2, makeRuntime: makeMockRuntime({ childProcessSpawner, + linuxCgroupController: cgroupController, mockAgentPath, - environment: { T3_ACP_EMIT_TOOL_CALLS: "1" }, + ownDescendantProcessGroups: true, + ownDetachedProcessGroup: true, }), }, fileSystem, idAllocator, serverConfig, }); - const threadId = ThreadId.make("thread-acp-cancel-permission"); + const threadId = ThreadId.make("thread-acp-cgroup-join-failure"); const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ - runtimeMode: "approval-required", + runtimeMode: "full-access", interactionMode: "default", cwd: process.cwd(), }); const modelSelection = { instanceId, model: "default" } as const; - const runtime = yield* adapter.openSession({ - threadId, - providerSessionId: ProviderSessionId.make("provider-session-acp-cancel-permission"), - modelSelection, - runtimePolicy, - }); - const providerThread = yield* runtime.ensureThread({ - threadId, - modelSelection, - runtimePolicy, - }); - const now = yield* DateTime.now; - yield* runtime.startTurn( - makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + const sessionScope = yield* Scope.make(); + const opened = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-cgroup-join-failure"), + modelSelection, + runtimePolicy, + }) + .pipe(Effect.provideService(Scope.Scope, sessionScope), Effect.exit); + assert.isTrue(Exit.isFailure(opened)); + yield* Scope.close(sessionScope, Exit.void); + assert.equal(createCalls, 1); + assert.isAtLeast(killCalls, 1); + assert.isAtLeast(removeCalls, 1); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("negotiates and executes optional native session forks through the ACP runtime", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); + const makeRuntime = makeMockRuntime({ childProcessSpawner, mockAgentPath }); - const pendingRequest = Option.getOrThrow( - yield* runtime.events.pipe( + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime, + }, + fileSystem, + idAllocator, + serverConfig, + }); + const sourceThreadId = ThreadId.make("thread-acp-native-fork-source"); + const targetThreadId = ThreadId.make("thread-acp-native-fork-target"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId: sourceThreadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-native-fork"), + modelSelection, + runtimePolicy, + }); + + assert.isTrue(runtime.providerSession.capabilities.threads.canForkThread); + assert.isTrue(runtime.providerSession.capabilities.threads.canReadThreadSnapshot); + + const sourceProviderThread = yield* runtime.ensureThread({ + threadId: sourceThreadId, + modelSelection, + runtimePolicy, + }); + const forkedProviderThread = yield* runtime.forkThread({ + sourceProviderThread, + targetThreadId, + }); + + assert.equal(sourceProviderThread.nativeThreadRef?.nativeId, "mock-session-1"); + assert.equal(forkedProviderThread.nativeThreadRef?.nativeId, "mock-session-1-fork"); + assert.equal(forkedProviderThread.appThreadId, targetThreadId); + assert.equal(forkedProviderThread.forkedFrom?.providerThreadId, sourceProviderThread.id); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("closes an idle ACP session exactly once through the transition permit", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath, protocolEvents }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-idle-finalizer"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const sessionScope = yield* Scope.make(); + yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-idle-finalizer"), + modelSelection, + runtimePolicy, + }) + .pipe(Effect.provideService(Scope.Scope, sessionScope)); + yield* pollProtocolMethods(protocolEvents); + yield* Scope.close(sessionScope, Exit.void); + const finalizerMethods = yield* pollProtocolMethods(protocolEvents); + assert.equal(finalizerMethods.filter((method) => method === "session/close").length, 1); + }).pipe(Effect.provide(testLayer)), + ); + + it.effect("rejects requested options that the active ACP session does not expose", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-unsupported-option"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const error = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-unsupported-option"), + modelSelection: { + instanceId, + model: "default", + options: [{ id: "missing-option", value: "high" }], + }, + runtimePolicy, + }) + .pipe(Effect.flip); + + assert.equal(error._tag, "ProviderAdapterOpenSessionError"); + assert.include(String(error.cause), "does not expose requested configuration option(s)"); + assert.include(String(error.cause), "missing-option"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("reconfigures a loaded ACP session from its own active setup metadata", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath, protocolEvents }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const firstThreadId = ThreadId.make("thread-acp-active-setup:first"); + const secondThreadId = ThreadId.make("thread-acp-active-setup:second"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const initialSelection = { instanceId, model: "default" } satisfies ModelSelection; + const alternateSelection = { + instanceId, + model: "grok-mock-alt", + } satisfies ModelSelection; + const originalSelection = { instanceId, model: "grok-build" } satisfies ModelSelection; + const runtime = yield* adapter.openSession({ + threadId: firstThreadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-active-setup"), + modelSelection: initialSelection, + runtimePolicy, + }); + const firstProviderThread = yield* runtime.ensureThread({ + threadId: firstThreadId, + modelSelection: initialSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId: firstThreadId, + providerThread: firstProviderThread, + instanceId, + runtimePolicy, + modelSelection: alternateSelection, + now, + }), + ); + yield* runtime.events.pipe( + Stream.filter((event) => event.type === "turn.terminal"), + Stream.runHead, + ); + + const secondProviderThread: OrchestrationV2ProviderThread = { + ...firstProviderThread, + id: ProviderThreadId.make("provider-thread-acp-active-setup:second"), + appThreadId: secondThreadId, + nativeThreadRef: { + driver: ACP_TEST_DRIVER, + nativeId: "mock-session-2", + strength: "strong", + }, + status: "idle", + }; + yield* runtime.resumeThread({ + providerThread: secondProviderThread, + modelSelection: alternateSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId: secondThreadId, + providerThread: secondProviderThread, + instanceId, + runtimePolicy, + modelSelection: originalSelection, + now, + ordinal: 2, + }), + ); + yield* runtime.events.pipe( + Stream.filter((event) => event.type === "turn.terminal"), + Stream.runHead, + ); + + const setModelRequests = Array.from(yield* Queue.takeAll(protocolEvents)).filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/set_model", + ); + assert.lengthOf(setModelRequests, 2); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("terminalizes an empty successful foreground Bash tool when the turn completes", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: (runtimeOrdinal) => + runtimeOrdinal === 1 ? { T3_ACP_EMIT_EMPTY_SUCCESSFUL_BASH_THEN_HANG: "1" } : {}, + ownDetachedProcessGroup: true, + protocolEvents, + }), + normalizeToolCall: normalizeXAiAcpToolCallState, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-empty-successful-bash"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-empty-successful-bash"), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + + const statuses: string[] = []; + let runningStartedAt: DateTime.Utc | null = null; + let completedStartedAt: DateTime.Utc | null = null; + let completedAt: DateTime.Utc | null = null; + let completedInput: string | null = null; + let completedOutput: string | null | undefined; + let completedExitCode: number | null | undefined; + let runningProjectedExitCode: number | undefined = undefined; + let terminal = false; + while (!terminal) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.nativeItemRef?.nativeId === "tool-call-empty-success-1" + ) { + statuses.push(event.turnItem.status); + if (event.turnItem.status === "running") { + runningStartedAt ??= event.turnItem.startedAt; + if (event.turnItem.type === "command_execution") { + runningProjectedExitCode = event.turnItem.exitCode; + } + } + if (event.turnItem.status === "completed") { + completedStartedAt = event.turnItem.startedAt; + completedAt = event.turnItem.completedAt; + if (event.turnItem.type === "command_execution") { + completedInput = event.turnItem.input; + completedOutput = event.turnItem.output; + completedExitCode = event.turnItem.exitCode; + } + } + } + if (event.type === "turn.terminal") terminal = true; + } + + assert.deepEqual(statuses, ["running", "running", "completed"]); + assert.deepEqual(completedStartedAt, runningStartedAt); + assert.isNotNull(completedAt); + assert.equal(completedInput, "true"); + assert.equal(completedOutput, undefined); + assert.equal( + runningProjectedExitCode, + undefined, + "mid-stream exit_code must not project until the tool is terminal", + ); + assert.equal(completedExitCode, 0); + + yield* Queue.takeAll(protocolEvents); + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), + ) + .pipe(Effect.forkScoped); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + yield* runtime.interruptTurn({ + providerThread, + providerTurnId: secondProviderTurnId, + requestRuntimeRestart: true, + }); + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 3 }), + ); + const loadAfterRestart = yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => event.direction === "outgoing" && rawProtocolMethod(event) === "session/load", + ), + Stream.runHead, + ); + assert.isTrue(Option.isSome(loadAfterRestart)); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("cancels pending permission requests while interrupting an ACP turn", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const releaseCancel = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_TOOL_CALLS: "1" }, + wrapCancel: (cancel) => Deferred.await(releaseCancel).pipe(Effect.andThen(cancel)), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-cancel-permission"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-cancel-permission"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + + const pendingRequest = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + ), + Stream.runHead, + ), + ); + if ( + pendingRequest.type !== "runtime_request.updated" || + pendingRequest.runtimeRequest.providerTurnId === null + ) { + return yield* Effect.die("Expected a pending ACP permission request with a provider turn"); + } + + const interruptFiber = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId: pendingRequest.runtimeRequest.providerTurnId, + }) + .pipe(Effect.forkScoped); + + const cancelledRequest = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "runtime_request.updated" && + event.runtimeRequest.id === pendingRequest.runtimeRequest.id && + event.runtimeRequest.status === "cancelled", + ), + Stream.runHead, + ), + ); + assert.equal(cancelledRequest.type, "runtime_request.updated"); + yield* Deferred.succeed(releaseCancel, undefined); + yield* Fiber.join(interruptFiber); + const terminal = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter((event) => event.type === "turn.terminal"), + Stream.runHead, + ), + ); + assert.equal(terminal.type, "turn.terminal"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("keeps hard teardown excluded until a permission response is enqueued", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const responseEnqueued = yield* Deferred.make(); + const releaseResponseAcknowledgement = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_TOOL_CALLS: "1" }, + wrapOutgoingResponse: (onOutgoingResponse) => (requestId) => + Deferred.succeed(responseEnqueued, undefined).pipe( + Effect.andThen(Deferred.await(releaseResponseAcknowledgement)), + Effect.andThen(onOutgoingResponse(requestId)), + ), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-response-wins-permission"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-response-wins-permission"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + const pending = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + ), + Stream.runHead, + ), + ); + if ( + pending.type !== "runtime_request.updated" || + pending.runtimeRequest.providerTurnId === null + ) { + return yield* Effect.die("Expected a pending permission request"); + } + const responseFiber = yield* runtime + .respondToRuntimeRequest({ requestId: pending.runtimeRequest.id, decision: "accept" }) + .pipe(Effect.forkScoped); + yield* Deferred.await(responseEnqueued); + const interruptFiber = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId: pending.runtimeRequest.providerTurnId, + requestRuntimeRestart: true, + }) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.isUndefined(responseFiber.pollUnsafe()); + assert.isUndefined(interruptFiber.pollUnsafe()); + + yield* Deferred.succeed(releaseResponseAcknowledgement, undefined); + yield* Fiber.join(responseFiber); + yield* Fiber.join(interruptFiber); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("correlates reordered elicitation schemas through the completed stdout write", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const responseWritten = yield* Deferred.make(); + const releaseResponseAcknowledgement = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make("acp-test-reordered-elicitation"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_ELICITATION: "1" }, + wrapIncomingRequest: (onIncomingRequest) => (requestId, method, payload) => { + if (method !== "session/elicitation") { + return onIncomingRequest(requestId, method, payload); + } + const record = payload as Record; + return onIncomingRequest(requestId, method, { + ...record, + requestedSchema: { + properties: { + approved: { title: "Approved", type: "boolean" }, + }, + type: "object", + }, + }); + }, + wrapOutgoingResponse: (onOutgoingResponse) => (requestId) => + Deferred.succeed(responseWritten, undefined).pipe( + Effect.andThen(Deferred.await(releaseResponseAcknowledgement)), + Effect.andThen(onOutgoingResponse(requestId)), + ), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-reordered-elicitation"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-reordered-elicitation"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + const pending = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + ), + Stream.runHead, + ), + ); + if (pending.type !== "runtime_request.updated") { + return yield* Effect.die("Expected a pending elicitation request"); + } + const responseFiber = yield* runtime + .respondToRuntimeRequest({ + requestId: pending.runtimeRequest.id, + answers: { approved: ["true"] }, + }) + .pipe(Effect.forkScoped); + + yield* Deferred.await(responseWritten); + assert.isUndefined(responseFiber.pollUnsafe()); + yield* Deferred.succeed(releaseResponseAcknowledgement, undefined); + yield* Fiber.join(responseFiber); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("fails a held native response acknowledgement before normal session close", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const responseWritten = yield* Deferred.make(); + const releaseResponseAcknowledgement = yield* Deferred.make(); + const responseLifecycle: Array = []; + const instanceId = ProviderInstanceId.make("acp-test-normal-close-held-response"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_TOOL_CALLS: "1" }, + wrapOutgoingResponse: (onOutgoingResponse) => (requestId) => + Deferred.succeed(responseWritten, undefined).pipe( + Effect.andThen(Deferred.await(releaseResponseAcknowledgement)), + Effect.andThen(onOutgoingResponse(requestId)), + ), + }), + }, + fileSystem, + idAllocator, + serverConfig, + testHooks: { + onNativeResponseLifecycle: (event) => + Effect.sync(() => { + responseLifecycle.push(event.type); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-normal-close-held-response"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const sessionScope = yield* Scope.make(); + const runtime = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-normal-close-held-response", + ), + modelSelection, + runtimePolicy, + }) + .pipe(Effect.provideService(Scope.Scope, sessionScope)); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + const pending = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + ), + Stream.runHead, + ), + ); + if (pending.type !== "runtime_request.updated") { + return yield* Effect.die("Expected a pending permission request"); + } + const responseFiber = yield* runtime + .respondToRuntimeRequest({ requestId: pending.runtimeRequest.id, decision: "accept" }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(responseWritten); + + const closeFiber = yield* Scope.close(sessionScope, Exit.void).pipe(Effect.forkScoped); + while (!responseLifecycle.includes("failed")) { + yield* Effect.yieldNow; + } + const responseExit = yield* Fiber.join(responseFiber); + if (Exit.isSuccess(responseExit)) { + assert.fail("normal close must fail a response whose transport acknowledgement is held"); + } + assert.include(Cause.pretty(responseExit.cause), "ACP session transport closed"); + assert.include(responseLifecycle, "removed"); + assert.include(responseLifecycle, "failed"); + yield* Deferred.succeed(releaseResponseAcknowledgement, undefined); + yield* Fiber.join(closeFiber); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("rejects delayed native response registration when normal close wins the permit", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const registrationStarted = yield* Deferred.make(); + const releaseRegistration = yield* Deferred.make(); + const transportClosed = yield* Deferred.make(); + const releaseTransportClose = yield* Deferred.make(); + const responseLifecycle: Array = []; + const instanceId = ProviderInstanceId.make("acp-test-close-wins-registration"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_TOOL_CALLS: "1" }, + }), + }, + fileSystem, + idAllocator, + serverConfig, + testHooks: { + afterNativeResponseTransportClosed: () => + Deferred.succeed(transportClosed, undefined).pipe( + Effect.andThen(Deferred.await(releaseTransportClose)), + ), + beforeNativeResponseAdmissionCheck: () => + Deferred.succeed(registrationStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseRegistration)), + ), + onNativeResponseLifecycle: (event) => + Effect.sync(() => { + responseLifecycle.push(event.type); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-close-wins-registration"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const sessionScope = yield* Scope.make(); + const runtime = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-close-wins-registration"), + modelSelection, + runtimePolicy, + }) + .pipe(Effect.provideService(Scope.Scope, sessionScope)); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + const pending = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + ), + Stream.runHead, + ), + ); + if (pending.type !== "runtime_request.updated") { + return yield* Effect.die("Expected a pending permission request"); + } + const responseFiber = yield* runtime + .respondToRuntimeRequest({ requestId: pending.runtimeRequest.id, decision: "accept" }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(registrationStarted); + + const closeFiber = yield* Scope.close(sessionScope, Exit.void).pipe(Effect.forkScoped); + yield* Deferred.await(transportClosed); + yield* Deferred.succeed(releaseRegistration, undefined); + while (!responseLifecycle.includes("admission_rejected")) { + yield* Effect.yieldNow; + } + const responseExit = yield* Fiber.join(responseFiber); + if (Exit.isSuccess(responseExit)) { + assert.fail("normal close must reject a response delayed before transport registration"); + } + assert.include(Cause.pretty(responseExit.cause), "ACP session transport closed"); + assert.equal(responseLifecycle.filter((event) => event === "registered").length, 1); + assert.equal(responseLifecycle.filter((event) => event === "removed").length, 1); + assert.equal(responseLifecycle.filter((event) => event === "admission_rejected").length, 1); + yield* Deferred.succeed(releaseTransportClose, undefined); + yield* Fiber.join(closeFiber); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("bounds a missing pending permission response acknowledgement", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const responseEnqueued = yield* Deferred.make(); + const releaseNativeHook = yield* Deferred.make(); + const responseLifecycle: Array = []; + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test-pending-response-timeout"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { + T3_ACP_EMIT_TOOL_CALLS: "1", + T3_ACP_HANG_AFTER_PERMISSION: "1", + }, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + protocolEvents, + windowsProcessTreeTerminator: (pid) => + Deferred.succeed(releaseNativeHook, undefined).pipe( + Effect.andThen( + Effect.sync(() => { + process.kill(pid, "SIGTERM"); + }), + ), + ), + wrapOutgoingResponse: (onOutgoingResponse) => (requestId) => + Deferred.succeed(responseEnqueued, undefined).pipe( + Effect.andThen(Deferred.await(releaseNativeHook)), + Effect.andThen(onOutgoingResponse(requestId)), + ), + }), + }, + fileSystem, + idAllocator, + serverConfig, + testHooks: { + onNativeResponseLifecycle: (event) => + Effect.sync(() => { + responseLifecycle.push(event.type); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-pending-response-timeout"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const sessionScope = yield* Scope.make(); + const runtime = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-pending-response-timeout", + ), + modelSelection, + runtimePolicy, + }) + .pipe(Effect.provideService(Scope.Scope, sessionScope)); + yield* pollProtocolMethods(protocolEvents); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + const pending = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + ), + Stream.runHead, + ), + ); + if ( + pending.type !== "runtime_request.updated" || + pending.runtimeRequest.providerTurnId === null + ) { + return yield* Effect.die("Expected a pending permission request"); + } + const responseExit = yield* runtime + .respondToRuntimeRequest({ requestId: pending.runtimeRequest.id, decision: "accept" }) + .pipe(Effect.exit); + if (Exit.isSuccess(responseExit)) { + assert.fail("missing native response acknowledgement must fail the pending response"); + } + assert.include(Cause.pretty(responseExit.cause), "Native response acknowledgement timed out"); + assert.isTrue(yield* Deferred.isDone(responseEnqueued)); + assert.isFalse(yield* Deferred.isDone(releaseNativeHook)); + assert.isBelow(responseLifecycle.indexOf("removed"), responseLifecycle.indexOf("failed")); + assert.includeMembers(responseLifecycle, [ + "registered", + "removed", + "failed", + "timer_exited", + "timer_started", + "watcher_exited", + "watcher_started", + ]); + + yield* runtime.interruptTurn({ + providerThread, + providerTurnId: pending.runtimeRequest.providerTurnId, + requestRuntimeRestart: true, + }); + assert.isTrue(yield* Deferred.isDone(releaseNativeHook)); + yield* Effect.sleep("50 millis"); + assert.include(responseLifecycle, "late_noop"); + yield* Scope.close(sessionScope, Exit.void); + assert.notInclude(yield* pollProtocolMethods(protocolEvents), "session/close"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("defers caller cancellation until a pending response acknowledgement is bounded", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const responseEnqueued = yield* Deferred.make(); + const releaseNativeHook = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make("acp-test-pending-response-cancel"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { + T3_ACP_EMIT_TOOL_CALLS: "1", + T3_ACP_HANG_AFTER_PERMISSION: "1", + }, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: (pid) => + Deferred.succeed(releaseNativeHook, undefined).pipe( + Effect.andThen( + Effect.sync(() => { + process.kill(pid, "SIGTERM"); + }), + ), + ), + wrapOutgoingResponse: (onOutgoingResponse) => (requestId) => + Deferred.succeed(responseEnqueued, undefined).pipe( + Effect.andThen(Deferred.await(releaseNativeHook)), + Effect.andThen(onOutgoingResponse(requestId)), + ), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-pending-response-cancel"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-pending-response-cancel"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + const pending = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + ), + Stream.runHead, + ), + ); + if ( + pending.type !== "runtime_request.updated" || + pending.runtimeRequest.providerTurnId === null + ) { + return yield* Effect.die("Expected a pending permission request"); + } + const responseFiber = yield* runtime + .respondToRuntimeRequest({ requestId: pending.runtimeRequest.id, decision: "accept" }) + .pipe(Effect.forkScoped); + yield* Deferred.await(responseEnqueued); + const cancellationFiber = yield* Fiber.interrupt(responseFiber).pipe(Effect.forkScoped); + const interruptFiber = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId: pending.runtimeRequest.providerTurnId, + requestRuntimeRestart: true, + }) + .pipe(Effect.forkScoped); + yield* Effect.sleep("100 millis"); + assert.isUndefined(cancellationFiber.pollUnsafe()); + assert.isUndefined(interruptFiber.pollUnsafe()); + assert.isFalse(yield* Deferred.isDone(releaseNativeHook)); + + yield* Fiber.join(cancellationFiber); + yield* Fiber.join(interruptFiber); + assert.isTrue(yield* Deferred.isDone(releaseNativeHook)); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("waits for immediate allow and deny permission responses before hard teardown", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + + for (const [name, sandboxPolicy] of [ + ["allow", undefined], + ["deny", { type: "readOnly" } as const], + ] as const) { + yield* Effect.gen(function* () { + const responseEnqueued = yield* Deferred.make(); + const releaseResponseAcknowledgement = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make(`acp-test-${name}`); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_TOOL_CALLS: "1" }, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: (pid) => + Effect.sync(() => { + process.kill(pid, "SIGTERM"); + }), + wrapOutgoingResponse: (onOutgoingResponse) => (requestId) => + Deferred.succeed(responseEnqueued, undefined).pipe( + Effect.andThen(Deferred.await(releaseResponseAcknowledgement)), + Effect.andThen(onOutgoingResponse(requestId)), + ), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make(`thread-acp-immediate-permission-${name}`); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + approvalPolicy: "never", + cwd: process.cwd(), + ...(sandboxPolicy === undefined ? {} : { sandboxPolicy }), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + `provider-session-acp-immediate-permission-${name}`, + ), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const turnFiber = yield* runtime + .startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ) + .pipe(Effect.forkDetach); + yield* Deferred.await(responseEnqueued); + const interruptFiber = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }), + requestRuntimeRestart: true, + }) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.isUndefined(interruptFiber.pollUnsafe()); + + yield* Deferred.succeed(releaseResponseAcknowledgement, undefined); + yield* Fiber.join(interruptFiber); + yield* Fiber.interrupt(turnFiber).pipe(Effect.forkDetach); + }).pipe(Effect.scoped); + } + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("waits for immediate URL elicitation responses before hard teardown", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const responseEnqueued = yield* Deferred.make(); + const releaseResponseAcknowledgement = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make("acp-test-url-elicitation"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_URL_ELICITATION: "1" }, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: (pid) => + Effect.sync(() => { + process.kill(pid, "SIGTERM"); + }), + wrapOutgoingResponse: (onOutgoingResponse) => (requestId) => + Deferred.succeed(responseEnqueued, undefined).pipe( + Effect.andThen(Deferred.await(releaseResponseAcknowledgement)), + Effect.andThen(onOutgoingResponse(requestId)), + ), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-immediate-url-elicitation"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-immediate-url-elicitation"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const turnFiber = yield* runtime + .startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ) + .pipe(Effect.forkDetach); + yield* Deferred.await(responseEnqueued); + const interruptFiber = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }), + requestRuntimeRestart: true, + }) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.isUndefined(interruptFiber.pollUnsafe()); + + yield* Deferred.succeed(releaseResponseAcknowledgement, undefined); + yield* Fiber.join(interruptFiber); + yield* Fiber.interrupt(turnFiber).pipe(Effect.forkDetach); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("bounds a missing immediate response acknowledgement before hard teardown", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const responseEnqueued = yield* Deferred.make(); + const releaseNativeHook = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make("acp-test-missing-response-ack"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_TOOL_CALLS: "1" }, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: (pid) => + Deferred.succeed(releaseNativeHook, undefined).pipe( + Effect.andThen( + Effect.sync(() => { + process.kill(pid, "SIGTERM"); + }), + ), + ), + wrapOutgoingResponse: (onOutgoingResponse) => (requestId) => + Deferred.succeed(responseEnqueued, undefined).pipe( + Effect.andThen(Deferred.await(releaseNativeHook)), + Effect.andThen(onOutgoingResponse(requestId)), + ), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-missing-response-ack"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + approvalPolicy: "never", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-missing-response-ack"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const turnFiber = yield* runtime + .startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ) + .pipe(Effect.forkDetach); + yield* Deferred.await(responseEnqueued); + const startedAt = yield* Clock.currentTimeMillis; + yield* runtime.interruptTurn({ + providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }), + requestRuntimeRestart: true, + }); + assert.isAtLeast((yield* Clock.currentTimeMillis) - startedAt, 1_500); + yield* Fiber.interrupt(turnFiber).pipe(Effect.forkDetach); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("rejects an elicitation response when hard teardown wins admission", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const teardownStarted = yield* Deferred.make(); + const releaseTeardown = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_ELICITATION: "1" }, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: (pid) => + Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseTeardown)), + Effect.andThen( + Effect.sync(() => { + process.kill(pid, "SIGTERM"); + }), + ), + ), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-teardown-wins-elicitation"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-teardown-wins-elicitation"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + const pending = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + ), + Stream.runHead, + ), + ); + if ( + pending.type !== "runtime_request.updated" || + pending.runtimeRequest.providerTurnId === null + ) { + return yield* Effect.die("Expected a pending elicitation request"); + } + const interruptFiber = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId: pending.runtimeRequest.providerTurnId, + requestRuntimeRestart: true, + }) + .pipe(Effect.forkScoped); + yield* Deferred.await(teardownStarted); + const responseFiber = yield* runtime + .respondToRuntimeRequest({ + requestId: pending.runtimeRequest.id, + answers: { approved: ["true"] }, + }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Effect.yieldNow; + assert.isUndefined(responseFiber.pollUnsafe()); + + yield* Deferred.succeed(releaseTeardown, undefined); + yield* Fiber.join(interruptFiber); + const responseExit = yield* Fiber.join(responseFiber); + if (Exit.isSuccess(responseExit)) { + assert.fail("teardown winning admission must reject the elicitation response"); + } + assert.include(Cause.pretty(responseExit.cause), "No pending ACP runtime request"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("does not release an ACP turn when cancellation is not acknowledged", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const instanceId = ProviderInstanceId.make("acp-test"); + const protocolEvents = yield* Queue.bounded(256); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_PROMPT_DELAY_MS: "5000" }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-cancel-timeout"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-cancel-timeout"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + const firstTurn = makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now, + }); + yield* runtime.startTurn(firstTurn); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/cancel", + ), + Stream.runHead, + ); + yield* TestClock.adjust("10 seconds"); + const interruptError = yield* Fiber.join(interruptFiber); + assert.equal(interruptError._tag, "ProviderAdapterInterruptError"); + + const secondTurnError = yield* runtime + .startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now, + ordinal: 2, + }), + ) + .pipe(Effect.flip); + assert.equal(secondTurnError._tag, "ProviderAdapterTurnStartError"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("treats a second hard Stop as success when the turn is already gone", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + interruptPromptOnCancel: false, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + ownDetachedProcessGroup: true, + environment: { T3_ACP_HANG_PROMPT_FOREVER: "1" }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-double-stop"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-double-stop"), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn(makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now })) + .pipe(Effect.forkScoped); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + yield* runtime.interruptTurn({ + providerThread, + providerTurnId, + requestRuntimeRestart: true, + }); + // Second durable interrupt after activeTurn is cleared must not fail. + const second = yield* Effect.exit( + runtime.interruptTurn({ + providerThread, + providerTurnId, + requestRuntimeRestart: true, + }), + ); + assert.isTrue(Exit.isSuccess(second), "duplicate hard Stop must be idempotent"); + let terminal: string | null = null; + while (terminal === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === providerTurnId) { + terminal = event.status; + } + } + assert.equal(terminal, "interrupted"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("finalizes a settled turn held open for background work when interrupted", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId === "tool-call-generic-1" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : undefined, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-interrupt-background-hold"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-interrupt-background"), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + // The still-running subagent defers finalize after session/prompt returns. + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let terminalStatus: string | null = null; + while (terminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === providerTurnId) { + terminalStatus = event.status; + } + } + assert.equal(terminalStatus, "interrupted"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "carries a live subagent lineage across an interrupt so the next turn can complete it", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : // Hydration-only shape (empty prompt, null title): without a + // carried-over lineage this update is dropped and the item + // stays running forever. + { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-subagent-carryover"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-subagent-carryover"), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let subagentTurnItemId: string | null = null; + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { + subagentTurnItemId = event.turnItem.id; + } + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.notEqual(subagentTurnItemId, null); + + subagentPhase = "complete"; + const secondNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ); + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let carriedItemStatus: string | null = null; + let secondTerminalStatus: string | null = null; + while (secondTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.id === subagentTurnItemId + ) { + carriedItemStatus = event.turnItem.status; + } + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminalStatus = event.status; + } + } + assert.equal(carriedItemStatus, "completed"); + assert.equal(secondTerminalStatus, "completed"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "preserveRuntimeOnSettledInterrupt keeps the process alive and carries subagents through a settled steering interrupt", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + let cancelCalled = false; + let runtimeOrdinalSeen = 0; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + // Hard interrupt flags (stricter than production Grok, which no + // longer sets restartRuntimeOnEveryInterrupt): every interrupt + // would hard-kill the process group without the settled-soft gate + // under test. + restartRuntimeAfterInterrupt: true, + restartRuntimeOnEveryInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + // No ownDetachedProcessGroup: if the interrupt wrongly takes the + // hard path, terminateProcessGroup is missing and the interrupt + // fails loudly with a poisoned session. + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); + return { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }; + }, + protocolEvents, + wrapCancel: (cancel) => + Effect.sync(() => { + cancelCalled = true; + }).pipe(Effect.andThen(cancel)), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-settled-soft-steer"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-settled-soft-steer"), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + // The still-running subagent defers finalize after session/prompt returns, + // so the interrupt below hits a settled turn held open for background work. + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + assert.isFalse( + cancelCalled, + "settled soft steer must not send session/cancel (the real Grok CLI kills background subagents on cancel)", + ); + + let subagentTurnItemId: string | null = null; + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { + subagentTurnItemId = event.turnItem.id; + } + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.notEqual(subagentTurnItemId, null); + + subagentPhase = "complete"; + const secondNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ); + // Same runtime process (mock-session-1): a respawn would start + // mock-session-2 and drop the carryover on the session mismatch. + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let carriedItemStatus: string | null = null; + let secondTerminalStatus: string | null = null; + while (secondTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.id === subagentTurnItemId + ) { + carriedItemStatus = event.turnItem.status; + } + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminalStatus = event.status; + } + } + assert.equal(carriedItemStatus, "completed"); + assert.equal(secondTerminalStatus, "completed"); + assert.equal( + runtimeOrdinalSeen, + 1, + "settled soft steer must not respawn the ACP runtime process", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("preserveRuntimeOnSettledInterrupt does not soften a mid-prompt steering interrupt", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + // Local hard-flavor gate: production Grok no longer sets + // restartRuntimeOnEveryInterrupt, but when a flavor does, the + // settled-soft gate must not leak onto an unsettled prompt. + restartRuntimeOnEveryInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + // No ownDetachedProcessGroup: the expected hard path fails loudly + // on the missing terminateProcessGroup, proving the settled-soft + // gate did not apply to an unsettled prompt. + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_HANG_PROMPT_FOREVER: "1" }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-unsettled-steer-stays-hard"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-unsettled-steer-stays-hard", + ), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptExit = yield* runtime + .interruptTurn({ providerThread, providerTurnId }) + .pipe(Effect.exit); + if (Exit.isSuccess(interruptExit)) { + assert.fail("mid-prompt steering interrupt must still take the hard teardown path"); + } + assert.include(Cause.pretty(interruptExit.cause), "session is poisoned"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live( + "soft mid-prompt interrupt cancels in place, reuses the runtime, and tracks cancel-backgrounded work", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + let cancelCalled = false; + let runtimeOrdinalSeen = 0; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + // Production Grok interrupt flags: hard teardown only with + // requestRuntimeRestart (user Stop). Without + // restartRuntimeOnEveryInterrupt a mid-prompt steering interrupt + // stays soft: session/cancel, same process, session reuse. + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + registerExtensions: ({ runtime: extensionRuntime, applyBackgroundTaskMutation }) => + registerXAiBackgroundTaskTracking(extensionRuntime, applyBackgroundTaskMutation), + // No ownDetachedProcessGroup: if the interrupt wrongly takes the + // hard path, terminateProcessGroup is missing and the interrupt + // fails loudly with a poisoned session. + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); + return { + T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG_FIRST_PROMPT: "1", + T3_ACP_EMIT_TASK_BACKGROUNDED_AFTER_CANCEL: "1", + }; + }, + protocolEvents, + wrapCancel: (cancel) => + Effect.sync(() => { + cancelCalled = true; + }).pipe(Effect.andThen(cancel)), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-soft-mid-prompt-steer"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-soft-mid-prompt-steer"), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + // Wait for the running command tool so the interrupt lands mid-prompt. + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes("tool-call-running-1"), + ), + Stream.runHead, + ); + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + // No requestRuntimeRestart: a steering interrupt, not a user Stop. + yield* runtime.interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }); + assert.isTrue( + cancelCalled, + "soft mid-prompt interrupt must send session/cancel to detach the running work", + ); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + // The cancel handler emitted _x.ai/task_backgrounded for the detached + // command; the tracked task must report as pending background work. + let backgroundTracked = false; + for (let attempt = 0; attempt < 80 && !backgroundTracked; attempt += 1) { + backgroundTracked = yield* hasPendingBackgroundWork; + if (!backgroundTracked) { + yield* Effect.sleep("25 millis"); + } + } + assert.isTrue( + backgroundTracked, + "cancel-backgrounded task must be tracked as running background work", + ); + + // The second prompt reuses the same process and session. + const secondNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ); + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let secondTerminalStatus: string | null = null; + while (secondTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminalStatus = event.status; + } + } + assert.equal(secondTerminalStatus, "completed"); + assert.equal( + runtimeOrdinalSeen, + 1, + "soft mid-prompt interrupt must not respawn the ACP runtime process", + ); + + // _x.ai/task_completed lands ~1.2s after the cancel and clears the + // tracked task without opening a synthetic continuation run. + let backgroundPending = true; + for (let attempt = 0; attempt < 50 && backgroundPending; attempt += 1) { + backgroundPending = yield* hasPendingBackgroundWork; + if (backgroundPending) { + yield* Effect.sleep("100 millis"); + } + } + assert.isFalse( + backgroundPending, + "tracked background task must clear after _x.ai/task_completed", + ); + assert.lengthOf( + continuationRequests, + 0, + "a cancel-backgrounded task completion must not wake a synthetic continuation run", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("direct Stop quarantine drops late background task mutations from the stopped run", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const capturedMutation: { + current: + | ((mutation: { + readonly sessionId: string; + readonly taskId: string; + readonly status: "running" | "completed" | "failed"; + }) => Effect.Effect) + | null; + } = { current: null }; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + registerExtensions: (context) => + Effect.sync(() => { + capturedMutation.current = context.applyBackgroundTaskMutation; + }), + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_HANG_PROMPT_FOREVER: "1" }, + ownDetachedProcessGroup: true, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { offer: () => Effect.void }, + }); + const threadId = ThreadId.make("thread-acp-stop-quarantine-late-task"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-stop-quarantine-late-task"), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const applyMutation = capturedMutation.current; + if (applyMutation === null) { + return yield* Effect.die("registerExtensions must capture applyBackgroundTaskMutation"); + } + // Plumbing sanity: pre-Stop mutations on the root session track and + // clear pending background work through the extension callback. + yield* applyMutation({ + sessionId: "mock-session-1", + taskId: "task-pre-stop", + status: "running", + }); + assert.isTrue( + yield* hasPendingBackgroundWork, + "a running background task mutation on the root session must track before Stop", + ); + yield* applyMutation({ + sessionId: "mock-session-1", + taskId: "task-pre-stop", + status: "completed", + }); + assert.isFalse( + yield* hasPendingBackgroundWork, + "a completed background task mutation must clear pending background work", + ); + + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + yield* runtime.interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true }); + let terminalStatus: string | null = null; + while (terminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === providerTurnId) { + terminalStatus = event.status; + } + } + assert.equal(terminalStatus, "interrupted"); + + // Residual lifecycle from the stopped run: activeSessionId still points + // at the stopped session until the next turn respawns the runtime, so + // only the direct Stop quarantine stands between this mutation and the + // wake machinery. + yield* applyMutation({ + sessionId: "mock-session-1", + taskId: "task-late-after-stop", + status: "running", + }); + assert.isFalse( + yield* hasPendingBackgroundWork, + "direct Stop quarantine must drop residual background task mutations from the stopped run", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("production Grok interrupt flags still hard-kill and respawn on user Stop", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + let runtimeOrdinalSeen = 0; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + // The full production Grok interrupt flag set after the non-Stop + // softening: no restartRuntimeOnEveryInterrupt. User Stop + // (requestRuntimeRestart) must still take the hard teardown and + // respawn path, not the soft cancel path. + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + registerExtensions: ({ runtime: extensionRuntime, applyBackgroundTaskMutation }) => + registerXAiBackgroundTaskTracking(extensionRuntime, applyBackgroundTaskMutation), + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + // Only hang the interrupted first turn. The replacement process must + // complete normally so startTurn / session-load assertions can finish. + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); + return runtimeOrdinal === 1 ? { T3_ACP_HANG_PROMPT_FOREVER: "1" } : {}; + }, + ownDetachedProcessGroup: true, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { offer: () => Effect.void }, + }); + const threadId = ThreadId.make("thread-acp-production-stop-hard-kill"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-production-stop-hard-kill"), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + yield* runtime.interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true }); + let terminalStatus: string | null = null; + while (terminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === providerTurnId) { + terminalStatus = event.status; + } + } + assert.equal(terminalStatus, "interrupted"); + + // Effect 4 Queue.takeAll waits for at least one element when empty. After + // the session/prompt stream drain the protocol queue is often empty, so + // takeAll would hang forever. Use clear (non-blocking drain) instead so + // the session/load wait cannot match residual pre-restart traffic. + yield* Queue.clear(protocolEvents); + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), + ); + const loadAfterRestart = yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => event.direction === "outgoing" && rawProtocolMethod(event) === "session/load", + ), + Stream.runHead, + ); + assert.isTrue( + Option.isSome(loadAfterRestart), + "user Stop must respawn the runtime and reload the session on the next turn", + ); + assert.equal( + runtimeOrdinalSeen, + 2, + "user Stop with production Grok flags must replace the ACP runtime process", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + // it.live: ownDetachedProcessGroup teardown uses wall-clock sleeps; under + // it.effect the interrupt timeout on context.completed still needs real time + // after the soft steer clears the turn. + it.live( + "Stop after settled soft steer contains the orphan runtime and respawns without subagent carryover", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + let cancelCalled = false; + let runtimeOrdinalSeen = 0; + const childSessionId = "mock-child-session-1"; + const streamedSubagentText = "streamed subagent carryover text"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + // Production Grok interrupt flags: soft settle keeps the process; + // only requestRuntimeRestart (user Stop) hard-kills. + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); + return { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }; + }, + ownDetachedProcessGroup: true, + protocolEvents, + wrapCancel: (cancel) => + Effect.sync(() => { + cancelCalled = true; + }).pipe(Effect.andThen(cancel)), + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-stop-after-soft-steer-orphan"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-stop-after-soft-steer-orphan", + ), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + // Stream assistant text onto the carryover subagent while the deferred + // turn is still active so assistantText races ahead of task.result. + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: streamedSubagentText }, + }, + }); + let subagentTurnItemId: string | null = null; + let streamedTextSeen = false; + for (let attempt = 0; attempt < 64; attempt += 1) { + const maybeEvent = yield* Queue.take(events).pipe(Effect.timeoutOption("50 millis")); + if (Option.isNone(maybeEvent)) break; + const event = maybeEvent.value; + if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { + subagentTurnItemId = event.turnItem.id; + } + if ( + event.type === "message.updated" && + event.message.text.includes(streamedSubagentText) + ) { + streamedTextSeen = true; + break; + } + } + assert.isTrue( + streamedTextSeen, + "pre-steer child session chunk must project as subagent assistant text", + ); + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + // Soft steer first: clears the turn, leaves the process alive. + yield* runtime.interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }); + assert.isFalse( + cancelCalled, + "settled soft steer must not send session/cancel before the later Stop", + ); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { + subagentTurnItemId = event.turnItem.id; + } + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.notEqual(subagentTurnItemId, null); + assert.equal( + runtimeOrdinalSeen, + 1, + "soft steer must leave the original ACP runtime process alive", + ); + + // User Stop against the already-cleared turn: contain the orphan runtime. + const stopExit = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId: firstProviderTurnId, + requestRuntimeRestart: true, + }) + .pipe(Effect.exit); + if (Exit.isFailure(stopExit)) { + assert.fail( + `cleared-turn Stop must contain the orphan runtime, not fail: ${Cause.pretty(stopExit.cause)}`, + ); + } + + // Orphan Stop must terminalize carried-over subagents (soft steer left them + // running on purpose; quarantine alone would leave them stuck "running"). + let subagentStopStatus: string | null = null; + let subagentStopResult: string | null | undefined; + let subagentStopProviderThreadId: string | null | undefined; + let subagentUpdatedResult: string | null | undefined; + for (let attempt = 0; attempt < 64; attempt += 1) { + const maybeEvent = yield* Queue.take(events).pipe(Effect.timeoutOption("50 millis")); + if (Option.isNone(maybeEvent)) break; + const event = maybeEvent.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.id === subagentTurnItemId + ) { + subagentStopStatus = event.turnItem.status; + subagentStopResult = event.turnItem.result; + subagentStopProviderThreadId = event.turnItem.providerThreadId; + if (subagentStopStatus === "interrupted") break; + } + if ( + event.type === "subagent.updated" && + event.subagent.status === "interrupted" && + event.subagent.result === streamedSubagentText + ) { + subagentUpdatedResult = event.subagent.result; + } + } + assert.equal( + subagentStopStatus, + "interrupted", + "orphan Stop must emit interrupted terminal for turn-1 carryover subagent", + ); + assert.equal( + subagentStopResult, + streamedSubagentText, + "orphan Stop must merge streamed assistantText into the interrupted result", + ); + assert.equal( + subagentStopProviderThreadId, + providerThread.id, + "orphan Stop parent-level events must use the spawn-time parent provider thread id", + ); + assert.equal( + subagentUpdatedResult, + streamedSubagentText, + "orphan Stop subagent.updated must also carry the streamed result", + ); + + yield* Queue.clear(protocolEvents); + const secondNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ); + const loadAfterRestart = yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && + (rawProtocolMethod(event) === "session/load" || + rawProtocolMethod(event) === "session/new"), + ), + Stream.runHead, + ); + assert.isTrue( + Option.isSome(loadAfterRestart), + "orphan containment must force a runtime respawn before the next turn", + ); + assert.equal( + runtimeOrdinalSeen, + 2, + "Stop after soft steer must replace the orphan ACP runtime process", + ); + + // nativeTurnId is `${sessionId}:turn:${ordinal}`. The mock always uses + // mock-session-1; ordinal 2 yields turn:2 on the replacement process. + // Carryover was quarantined by Stop, so turn-1's subagent must not re-attach. + subagentPhase = "complete"; + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let carriedItemStatus: string | null = null; + let secondTerminalStatus: string | null = null; + while (secondTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.id === subagentTurnItemId + ) { + carriedItemStatus = event.turnItem.status; + } + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminalStatus = event.status; + } + } + assert.isNull( + carriedItemStatus, + "Stop quarantine must drop turn-1 subagent carryover on the respawned runtime", + ); + assert.equal(secondTerminalStatus, "completed"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "settled soft interrupt skips cancel when prompt wire is settled before completion callback", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + let cancelCalled = false; + const promptPhases: Array = []; + // Gate adapter-visible prompt return so we can open the race window after + // the native stopReason is on the wire. Release lets Effect.tap mark + // promptWireSettled before the completion callback requests the permit; + // interrupt then ORs wire-done with promptSettled under the permit. + const promptWireReturned = yield* Deferred.make(); + const releasePromptCompletion = yield* Deferred.make(); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + // Production Grok flags: without settled-soft, a steer soft-cancels. + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapCancel: (cancel) => + Effect.sync(() => { + cancelCalled = true; + }).pipe(Effect.andThen(cancel)), + wrapRuntime: (runtime) => ({ + ...runtime, + prompt: (payload) => + Effect.gen(function* () { + promptPhases.push("prompt-start"); + const result = yield* runtime.prompt(payload); + promptPhases.push("wire-returned"); + yield* Deferred.succeed(promptWireReturned, undefined); + yield* Deferred.await(releasePromptCompletion); + promptPhases.push("completion-released"); + return result; + }), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-settled-soft-admission-race"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-settled-soft-admission-race", + ), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Deferred.await(promptWireReturned); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + assert.deepEqual(promptPhases, ["prompt-start", "wire-returned"]); + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + // Fork interrupt while the adapter-side return is still gated. Release so + // promptWireSettled completes (Effect.tap) before/while the completion + // callback contends for runtimeCallbackPermit. Holding the gate forever + // would also block the wire signal (same Effect resolution), so release + // is required; the assertion is cancel skipped after that race window. + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* Deferred.succeed(releasePromptCompletion, undefined); + for (let attempt = 0; attempt < 20; attempt += 1) { + if (promptPhases.includes("completion-released")) break; + yield* Effect.yieldNow; + } + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + assert.includeMembers(promptPhases, [ + "prompt-start", + "wire-returned", + "completion-released", + ]); + assert.isFalse( + cancelCalled, + "settled soft steer must skip session/cancel once the prompt wire has settled", + ); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.equal(subagentPhase, "spawn"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "does not pin hasPendingBackgroundWork when a late TaskOutput re-reports an in-turn-handled task", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + extractBackgroundTaskId: (toolCall) => + toolCall.toolCallId === "tool-call-monitor-1" ? "task-monitor-1" : undefined, + extractBackgroundTaskCompletion: (toolCall) => + toolCall.toolCallId === "tool-call-fetch-1" + ? [ + { + taskId: "task-monitor-1", + status: toolCall.status === "completed" ? "completed" : "running", + appendOutput: toolCall.status === "completed" ? "MONITOR_LISTING_TOKEN" : "", + }, + ] + : [], + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { + T3_ACP_EMIT_IN_TURN_TASKOUTPUT_THEN_LATE_DUPLICATE: "1", + }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-already-handled-wake-pin"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-already-handled-wake-pin", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + + let terminalStatus: string | null = null; + while (terminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === providerTurnId) { + terminalStatus = event.status; + } + } + assert.equal(terminalStatus, "completed"); + + // Wait for the late post-finalize duplicate TaskOutput frame. + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes("MONITOR_LISTING_TOKEN_LATE"), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + assert.lengthOf( + continuationRequests, + 0, + "already-handled late TaskOutput must not open a continuation run", + ); + assert.isFalse( + yield* hasPendingBackgroundWork, + "wake buffer must not stay non-empty and pin idle release after an already-handled re-report", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "holds a settled turn until the injected monitor report streams instead of finalizing into it", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const triggerDir = yield* fileSystem.makeTempDirectoryScoped(); + const triggerPath = path.join(triggerDir, "report-trigger"); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + extractBackgroundTaskId: (toolCall) => + toolCall.toolCallId === "tool-call-monitor-1" ? "task-monitor-1" : undefined, + extractBackgroundToolMutation: (text) => + text.includes('Monitor "task-monitor-1" ended') + ? [{ taskId: "task-monitor-1", status: "completed", appendOutput: "" }] + : [], + extractBackgroundTaskCompletion: (toolCall) => + toolCall.toolCallId === "tool-call-fetch-1" + ? [ + { + taskId: "task-monitor-1", + status: toolCall.status === "completed" ? "completed" : "running", + appendOutput: toolCall.status === "completed" ? "MONITOR_LISTING_TOKEN" : "", + }, + ] + : [], + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { + T3_ACP_EMIT_POST_SETTLE_MONITOR_FLOW: "1", + T3_ACP_INJECTED_REPORT_TRIGGER_PATH: triggerPath, + }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-injected-report-hold"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-injected-report"), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + + // Wait (real time) until the post-settle end notice and TaskOutput + // hydration are ingested: the hydrated monitor card carries the + // fetched listing. + let reportSeen = false; + const trackReport = (event: ProviderAdapterV2Event): void => { + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "assistant_message" && + event.turnItem.text.includes("MONITOR_REPORT_TOKEN") + ) { + reportSeen = true; + } + }; + let hydrated = false; + while (!hydrated) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.status === "completed" && + (event.turnItem.output ?? "").includes("MONITOR_LISTING_TOKEN") + ) { + hydrated = true; + } + } + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + // Pre-fix the 2s deferred-finalize debounce fires here and the report + // streamed by the injected turn afterwards is dropped on the floor. + yield* TestClock.adjust("3 seconds"); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* Effect.yieldNow; + let drained = yield* Queue.poll(events); + while (Option.isSome(drained)) { + trackReport(drained.value); + assert.notEqual( + drained.value.type, + "turn.terminal", + "deferred finalize must hold while the injected-turn report is owed", + ); + drained = yield* Queue.poll(events); + } + + // Release the report, then the normal debounce finalizes the turn. + yield* fileSystem.writeFileString(triggerPath, "go"); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes("MONITOR_REPORT_TOKEN"), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* TestClock.adjust("3 seconds"); + + let terminalStatus: string | null = null; + while (terminalStatus === null) { + const event = yield* Queue.take(events); + trackReport(event); + if (event.type === "turn.terminal" && event.providerTurnId === providerTurnId) { + terminalStatus = event.status; + } + } + assert.equal(terminalStatus, "completed"); + assert.isTrue( + reportSeen, + "the injected-turn report must project before the turn finalizes", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("restarts the ACP child process before the next prompt after interrupt", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-restart-after-interrupt"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-restart-after-interrupt"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: `${providerThread.nativeThreadRef?.nativeId}:turn:1`, + }); + yield* runtime.interruptTurn({ + providerThread, + providerTurnId, + requestRuntimeRestart: true, + }); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/cancel", + ), + Stream.runHead, + ); + + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), + ); + const loadAfterRestart = yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => event.direction === "outgoing" && rawProtocolMethod(event) === "session/load", + ), + Stream.runHead, + ); + assert.isTrue( + Option.isSome(loadAfterRestart), + "post-interrupt startTurn should respawn the runtime and replay session/load", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("Windows teardown is one-shot explicitly with independent finalizer cleanup", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const path = yield* Path.Path; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const runtimeScope = yield* Scope.make(); + const taskkillCommands: Array<{ + readonly command: string; + readonly args: ReadonlyArray; + }> = []; + const taskkillSpawner = makeTaskkillSpawner({ exitCode: 0, commands: taskkillCommands }); + const spawner = ChildProcessSpawner.make((command) => { + const value = command as unknown as { readonly command: string }; + return value.command === "taskkill" + ? taskkillSpawner.spawn(command) + : childProcessSpawner.spawn(command); + }); + const context = yield* Layer.build( + AcpSessionRuntime.layer({ + spawn: { + command: process.execPath, + args: [mockAgentPath], + cwd: process.cwd(), + env: { T3_ACP_SESSION_LIFECYCLE: "1" }, + }, + cwd: process.cwd(), + clientInfo: { name: "t3-acp-test", version: "0.0.0" }, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + }).pipe(Layer.provide(Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner))), + ).pipe(Effect.provideService(Scope.Scope, runtimeScope)); + const runtime = yield* Effect.service(AcpSessionRuntime.AcpSessionRuntime).pipe( + Effect.provide(context), + ); + assert.isDefined(runtime.terminateProcessGroup); + yield* Effect.all([runtime.terminateProcessGroup!, runtime.terminateProcessGroup!], { + concurrency: "unbounded", + }); + assert.equal(taskkillCommands.length, 1); + yield* Scope.close(runtimeScope, Exit.void); + assert.equal(taskkillCommands.length, 1); + }).pipe(Effect.provide(testLayer)), + ); + + it("accepts only taskkill exit code zero as successful tree termination", () => { + assert.isTrue(AcpSessionRuntime.windowsTaskkillResultIsSuccess(0, "")); + assert.isFalse( + AcpSessionRuntime.windowsTaskkillResultIsSuccess( + 128, + "FEHLER: Der Prozess wurde nicht gefunden.", + ), + ); + assert.isFalse(AcpSessionRuntime.windowsTaskkillResultIsSuccess(128, "")); + assert.isFalse(AcpSessionRuntime.windowsTaskkillResultIsSuccess(1, "localized failure")); + assert.isFalse(AcpSessionRuntime.windowsTaskkillResultIsSuccess(255, "")); + }); + + it.effect("runs the default taskkill path and preserves every failure mode", () => + Effect.gen(function* () { + const commands: Array<{ readonly command: string; readonly args: ReadonlyArray }> = + []; + yield* AcpSessionRuntime.terminateWindowsProcessTreeWithTaskkill( + makeTaskkillSpawner({ exitCode: 0, commands }), + 4321, + ); + assert.deepEqual(commands, [{ command: "taskkill", args: ["/PID", "4321", "/T", "/F"] }]); + + for (const fixture of [ + { exitCode: 128, output: 'ERROR: The process "4321" not found.' }, + { exitCode: 128, output: "FEHLER: Prozess nicht gefunden." }, + { exitCode: 128, output: "" }, + { exitCode: 128, output: "ERROR: Access is denied." }, + { exitCode: 1, output: "generic failure" }, + ]) { + const failed = yield* AcpSessionRuntime.terminateWindowsProcessTreeWithTaskkill( + makeTaskkillSpawner(fixture), + 4321, + ).pipe(Effect.exit); + if (Exit.isSuccess(failed)) assert.fail(`taskkill ${fixture.exitCode} must fail`); + const error = Cause.squash(failed.cause); + assert.instanceOf(error, AcpSessionRuntime.AcpProcessGroupTerminationError); + const termination = error as AcpSessionRuntime.AcpProcessGroupTerminationError; + assert.equal( + termination.detail, + `taskkill exited ${fixture.exitCode} for ACP process tree 4321`, + ); + assert.equal(termination.pid, 4321); + assert.equal(termination.exitCode, fixture.exitCode); + if (fixture.output.length > 0) { + assert.equal(termination.cause, fixture.output); + } + } + + for (const fixture of [ + { spawnFailure: true }, + { outputFailure: true }, + { exitFailure: true }, + ]) { + const failed = yield* AcpSessionRuntime.terminateWindowsProcessTreeWithTaskkill( + makeTaskkillSpawner(fixture), + 4321, + ).pipe(Effect.exit); + if (Exit.isSuccess(failed)) assert.fail("taskkill infrastructure failure must fail"); + const error = Cause.squash(failed.cause); + assert.instanceOf(error, AcpSessionRuntime.AcpProcessGroupTerminationError); + assert.equal( + (error as AcpSessionRuntime.AcpProcessGroupTerminationError).detail, + "Failed to run taskkill for ACP process tree 4321", + ); + } + }), + ); + + it.effect("Windows process-tree teardown surfaces taskkill failure", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const path = yield* Path.Path; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const context = yield* Layer.build( + AcpSessionRuntime.layer({ + spawn: { command: process.execPath, args: [mockAgentPath], cwd: process.cwd() }, + cwd: process.cwd(), + clientInfo: { name: "t3-acp-test", version: "0.0.0" }, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: () => + Effect.fail( + new AcpSessionRuntime.AcpProcessGroupTerminationError({ + detail: "mock taskkill failure", + }), + ), + }).pipe( + Layer.provide( + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, childProcessSpawner), + ), + ), + ); + const runtime = yield* Effect.service(AcpSessionRuntime.AcpSessionRuntime).pipe( + Effect.provide(context), + ); + const error = yield* Effect.flip(runtime.terminateProcessGroup!); + assert.equal(error.detail, "mock taskkill failure"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("poisons the session when hard teardown defects and blocks replacement work", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const commandPidPath = yield* fileSystem.makeTempFileScoped({ + prefix: "t3-acp-failed-teardown-command-", + }); + const residualCallbackDir = yield* fileSystem.makeTempDirectoryScoped(); + const residualCallbackResponseLogPath = path.join(residualCallbackDir, "responses.log"); + const residualCallbackTriggerPath = path.join(residualCallbackDir, "trigger"); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const teardownStarted = yield* Deferred.make(); + const releaseTeardown = yield* Deferred.make(); + let terminatorCallCount = 0; + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: () => + Effect.gen(function* () { + terminatorCallCount += 1; + yield* Deferred.succeed(teardownStarted, undefined); + yield* Deferred.await(releaseTeardown); + return yield* Effect.die("mock taskkill defect"); + }), + environment: { + T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG: "1", + T3_ACP_RESIDUAL_CALLBACK_RESPONSE_LOG_PATH: residualCallbackResponseLogPath, + T3_ACP_RESIDUAL_CALLBACK_TRIGGER_PATH: residualCallbackTriggerPath, + T3_ACP_RUNNING_COMMAND_PID_PATH: commandPidPath, + }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-failed-hard-teardown"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const sessionScope = yield* Scope.make(); + const runtime = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-failed-hard-teardown"), + modelSelection, + runtimePolicy, + }) + .pipe(Effect.provideService(Scope.Scope, sessionScope)); + const adapterEvents = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(adapterEvents, event)), + Effect.forkIn(sessionScope), + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + while ((yield* fileSystem.readFileString(commandPidPath)).trim().length === 0) { + yield* Effect.yieldNow; + } + const [commandRootPid, commandSleepPid] = (yield* fileSystem.readFileString(commandPidPath)) + .trim() + .split(/\s+/) + .map(Number); + assert.isTrue( + Option.isSome(yield* waitForProcesses([commandRootPid!, commandSleepPid!])), + "declared failed-teardown Bash and sleep PIDs must both become live", + ); + yield* pollProtocolMethods(protocolEvents); + + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(teardownStarted); + const startFiber = yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), + ) + .pipe(Effect.exit, Effect.forkScoped); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + assert.isUndefined(startFiber.pollUnsafe()); + const methodsDuringTeardown = yield* pollProtocolMethods(protocolEvents); + yield* Deferred.succeed(releaseTeardown, undefined); + assert.notInclude(methodsDuringTeardown, "session/load"); + assert.notInclude(methodsDuringTeardown, "session/prompt"); + + const interruptExit = yield* Fiber.join(interruptFiber); + if (Exit.isSuccess(interruptExit)) assert.fail("hard teardown failure must fail interrupt"); + assert.include(Cause.pretty(interruptExit.cause), "session is poisoned"); + assert.isTrue( + Option.isSome(yield* waitForProcesses([commandRootPid!, commandSleepPid!])), + "failed teardown must leave both declared Bash and sleep PIDs live", + ); + + while (Option.isSome(yield* Queue.poll(adapterEvents))) { + // Discard the interrupted turn's expected projection before residual traffic. + } + yield* fileSystem.writeFileString(residualCallbackTriggerPath, "go"); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"method":"session/elicitation"'), + ), + Stream.runHead, + ); + yield* Effect.sleep("100 millis"); + assert.isTrue( + Option.isNone(yield* Queue.poll(adapterEvents)), + "residual callbacks must not mutate or emit adapter projection after poison", + ); + assert.isFalse( + yield* fileSystem.exists(residualCallbackResponseLogPath), + "permission and elicitation callbacks must remain unresolved after poison", + ); + assert.lengthOf( + continuationRequests, + 0, + "residual callbacks must not offer a continuation after poison", + ); + + const startExit = yield* Fiber.join(startFiber); + if (Exit.isSuccess(startExit)) assert.fail("poisoned session must reject startTurn"); + assert.include(Cause.pretty(startExit.cause), "session is poisoned"); + const resumeExit = yield* runtime + .resumeThread({ providerThread, modelSelection, runtimePolicy }) + .pipe(Effect.exit); + if (Exit.isSuccess(resumeExit)) assert.fail("poisoned session must reject resumeThread"); + assert.include(Cause.pretty(resumeExit.cause), "session is poisoned"); + const snapshotExit = yield* runtime.readThreadSnapshot({ providerThread }).pipe(Effect.exit); + if (Exit.isSuccess(snapshotExit)) { + assert.fail("poisoned session must reject readThreadSnapshot"); + } + assert.include(Cause.pretty(snapshotExit.cause), "session is poisoned"); + const forkExit = yield* runtime + .forkThread({ + sourceProviderThread: providerThread, + targetThreadId: ThreadId.make("thread-acp-poisoned-fork"), + }) + .pipe(Effect.exit); + if (Exit.isSuccess(forkExit)) assert.fail("poisoned session must reject forkThread"); + assert.include(Cause.pretty(forkExit.cause), "session is poisoned"); + yield* pollProtocolMethods(protocolEvents); + const retryInterruptExit = yield* runtime + .interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true }) + .pipe(Effect.exit); + if (Exit.isSuccess(retryInterruptExit)) { + assert.fail("poisoned session must reject a repeated hard interrupt"); + } + const firstInterruptError = Cause.squash(interruptExit.cause) as Error; + const retryInterruptError = Cause.squash(retryInterruptExit.cause) as Error; + assert.strictEqual(retryInterruptError.cause, firstInterruptError.cause); + assert.equal(terminatorCallCount, 1); + const methodsAfterPoison = yield* pollProtocolMethods(protocolEvents); + assert.notInclude(methodsAfterPoison, "initialize"); + assert.notInclude(methodsAfterPoison, "session/cancel"); + assert.notInclude(methodsAfterPoison, "session/fork"); + assert.notInclude(methodsAfterPoison, "session/load"); + assert.notInclude(methodsAfterPoison, "session/prompt"); + assert.isTrue(processExists(commandRootPid!)); + assert.isTrue(processExists(commandSleepPid!)); + yield* Scope.close(sessionScope, Exit.void); + assert.isTrue( + Option.isSome(yield* waitForProcessesToExit([commandRootPid!, commandSleepPid!])), + "finalizer cleanup must reap both declared Bash and sleep PIDs", + ); + assert.isFalse(processExists(commandRootPid!)); + assert.isFalse(processExists(commandSleepPid!)); + const finalizerMethods = yield* pollProtocolMethods(protocolEvents); + assert.notInclude(finalizerMethods, "session/close"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("durably poisons start and resume when required hard teardown is unavailable", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_HANG_PROMPT_FOREVER: "1" }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-missing-hard-teardown"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-missing-hard-teardown"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptExit = yield* runtime + .interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true }) + .pipe(Effect.exit); + if (Exit.isSuccess(interruptExit)) assert.fail("missing hard teardown must fail interrupt"); + assert.include(Cause.pretty(interruptExit.cause), "session is poisoned"); + + const startExit = yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), + ) + .pipe(Effect.exit); + if (Exit.isSuccess(startExit)) assert.fail("poisoned session must reject startTurn"); + assert.include(Cause.pretty(startExit.cause), "session is poisoned"); + const resumeExit = yield* runtime + .resumeThread({ providerThread, modelSelection, runtimePolicy }) + .pipe(Effect.exit); + if (Exit.isSuccess(resumeExit)) assert.fail("poisoned session must reject resumeThread"); + assert.include(Cause.pretty(resumeExit.cause), "session is poisoned"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("holds concurrent startTurn behind successful hard teardown and reloads once", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const teardownStarted = yield* Deferred.make(); + const releaseTeardown = yield* Deferred.make(); + let runtimeOrdinalSeen = 0; + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + interruptPromptOnCancel: true, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: () => + Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseTeardown)), + ), + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = runtimeOrdinal; + return runtimeOrdinal === 1 ? { T3_ACP_HANG_PROMPT_FOREVER: "1" } : {}; + }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-concurrent-hard-teardown"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-concurrent-hard-teardown"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* Effect.yieldNow; + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + yield* pollProtocolMethods(protocolEvents); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true }) + .pipe(Effect.forkScoped); + yield* Deferred.await(teardownStarted); + const startFiber = yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), + ) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + assert.isUndefined(startFiber.pollUnsafe()); + assert.equal(runtimeOrdinalSeen, 1); + const methodsDuringTeardown = yield* pollProtocolMethods(protocolEvents); + assert.notInclude(methodsDuringTeardown, "session/load"); + assert.notInclude(methodsDuringTeardown, "session/prompt"); + + yield* Deferred.succeed(releaseTeardown, undefined); + yield* Fiber.join(interruptFiber); + yield* Fiber.join(startFiber); + assert.equal(runtimeOrdinalSeen, 2); + const replacementMethods = yield* pollProtocolMethods(protocolEvents); + assert.equal(replacementMethods.filter((method) => method === "session/load").length, 1); + if (!replacementMethods.includes("session/prompt")) { + yield* Stream.fromQueue(protocolEvents).pipe( Stream.filter( (event) => - event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", ), Stream.runHead, + ); + } + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("quarantines old-runtime callbacks after successful hard teardown", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const responseLifecycle: Array = []; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + type HandlerRecord = { + sessionUpdate?: Parameters[0]; + permission?: Parameters[0]; + elicitation?: Parameters[0]; + requestUserInput?: AcpAdapterV2ExtensionContext["requestUserInput"]; + }; + const handlerRecords: HandlerRecord[] = []; + const runtimeInputs: AcpAdapterV2RuntimeInput[] = []; + const oldPromptCompletion = yield* Deferred.make(); + const transportDrained = yield* Deferred.make(); + const releaseTransportDrain = yield* Deferred.make(); + let extensionOrdinal = 0; + let runtimeOrdinalSeen = 0; + const instanceId = ProviderInstanceId.make("acp-test"); + const makeRuntime = makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: () => Effect.void, + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = runtimeOrdinal; + return { T3_ACP_HANG_PROMPT_FOREVER: "1" }; + }, + protocolEvents, + wrapRuntime: (runtime, runtimeOrdinal) => { + const record: HandlerRecord = {}; + handlerRecords[runtimeOrdinal - 1] = record; + return { + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + record.sessionUpdate = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + handleRequestPermission: (handler) => + Effect.sync(() => { + record.permission = handler; + }).pipe(Effect.andThen(runtime.handleRequestPermission(handler))), + handleElicitation: (handler) => + Effect.sync(() => { + record.elicitation = handler; + }).pipe(Effect.andThen(runtime.handleElicitation(handler))), + ...(runtimeOrdinal === 1 + ? { + prompt: (payload) => + Effect.gen(function* () { + yield* runtime.prompt(payload).pipe(Effect.ignore, Effect.forkDetach); + return yield* Deferred.await(oldPromptCompletion); + }), + } + : {}), + }; + }, + }); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + registerExtensions: ({ requestUserInput }) => + Effect.sync(() => { + handlerRecords[extensionOrdinal++]!.requestUserInput = requestUserInput; + }), + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: (runtimeInput) => + Effect.sync(() => { + runtimeInputs.push(runtimeInput); + }).pipe(Effect.andThen(makeRuntime(runtimeInput))), + }, + fileSystem, + idAllocator, + serverConfig, + testHooks: { + afterHardTeardownTransportDrained: () => + Deferred.succeed(transportDrained, undefined).pipe( + Effect.andThen(Deferred.await(releaseTransportDrain)), + ), + onNativeResponseLifecycle: (event) => + Effect.sync(() => { + responseLifecycle.push(event.type); + }), + }, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-successful-teardown-callback-quarantine"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-successful-teardown-callback-quarantine", + ), + modelSelection, + runtimePolicy, + }); + const adapterEvents = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(adapterEvents, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const permissionRequest = { + sessionId: "mock-session-1", + toolCall: { + toolCallId: "stale-generation-1-permission", + title: "Stale generation 1 permission", + }, + options: [{ optionId: "allow", name: "Allow", kind: "allow_once" as const }], + }; + const interruptFiber = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId, + requestRuntimeRestart: true, + }) + .pipe(Effect.forkScoped); + yield* Deferred.await(transportDrained); + const heldInboundFiber = yield* runtimeInputs[0]!.onIncomingRequest!( + "post-drain-stale-permission-id", + "session/request_permission", + permissionRequest, + ).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.isUndefined(heldInboundFiber.pollUnsafe()); + yield* Deferred.succeed(releaseTransportDrain, undefined); + yield* Fiber.join(heldInboundFiber); + yield* Fiber.join(interruptFiber); + assert.notInclude(responseLifecycle, "registered"); + assert.notInclude(responseLifecycle, "watcher_started"); + assert.isFalse( + (yield* Queue.takeAll(adapterEvents)).some( + (event) => event.type === "runtime_request.updated", + ), + "post-drain inbound callback must not emit a runtime request", + ); + assert.equal(runtimeOrdinalSeen, 1); + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + assert.equal(runtimeOrdinalSeen, 2); + const oldHandlers = handlerRecords[0]!; + const replacementHandlers = handlerRecords[1]!; + assert.isDefined(oldHandlers.sessionUpdate); + assert.isDefined(oldHandlers.permission); + assert.isDefined(oldHandlers.elicitation); + assert.isDefined(oldHandlers.requestUserInput); + assert.isDefined(replacementHandlers.sessionUpdate); + assert.isDefined(replacementHandlers.permission); + assert.isDefined(replacementHandlers.elicitation); + assert.isDefined(replacementHandlers.requestUserInput); + assert.lengthOf(runtimeInputs, 2); + while (Option.isSome(yield* Queue.poll(adapterEvents))) { + // Discard generation 1 terminal and generation 2 startup projection. + } + + yield* oldHandlers.sessionUpdate!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "stale generation 1 assistant" }, + }, + }); + yield* oldHandlers.sessionUpdate!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "stale-generation-1-tool", + title: "Stale generation 1 tool", + kind: "other", + status: "completed", + rawOutput: { output: "stale continuation evidence" }, + }, + }); + const oldPermissionFiber = yield* oldHandlers.permission!(permissionRequest).pipe( + Effect.exit, + Effect.forkScoped, + ); + const oldElicitationFiber = yield* oldHandlers.elicitation!({ + sessionId: "mock-session-1", + message: "Stale generation 1 elicitation", + mode: "form", + requestedSchema: { + type: "object", + properties: { approved: { type: "boolean", title: "Approved" } }, + }, + }).pipe(Effect.exit, Effect.forkScoped); + const oldXAiUserInputFiber = yield* oldHandlers.requestUserInput!({ + nativeItemId: "stale-generation-1-xai-item", + nativeMethod: "_x.ai/ask_user_question", + nativeRequestId: "stale-generation-1-xai-request", + nativeSessionId: "mock-session-1", + questions: [ + { + id: "approved", + header: "Approve", + question: "Approve stale generation 1?", + options: [{ label: "yes", description: "Approve" }], + }, + ], + }).pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.succeed(oldPromptCompletion, { stopReason: "end_turn" }); + yield* Effect.sleep("100 millis"); + assert.isTrue(Option.isNone(yield* Queue.poll(adapterEvents))); + assert.isUndefined(oldPermissionFiber.pollUnsafe()); + assert.isUndefined(oldElicitationFiber.pollUnsafe()); + assert.isUndefined(oldXAiUserInputFiber.pollUnsafe()); + assert.lengthOf(continuationRequests, 0); + + yield* replacementHandlers.sessionUpdate!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "live generation 2 assistant" }, + }, + }); + let replacementMessageSeen = false; + while (!replacementMessageSeen) { + const event = yield* Queue.take(adapterEvents); + replacementMessageSeen = + event.type === "message.updated" && event.message.text.includes("live generation 2"); + } + const missingPermissionTransport = yield* replacementHandlers.permission!( + permissionRequest, + ).pipe(Effect.exit); + if (Exit.isSuccess(missingPermissionTransport)) { + assert.fail("a core permission request without transport correlation must fail closed"); + } + assert.include(Cause.pretty(missingPermissionTransport.cause), "Could not correlate"); + yield* runtimeInputs[1]!.onIncomingRequest!( + "live-generation-2-permission-id", + "session/request_permission", + permissionRequest, + ); + const replacementPermission = yield* replacementHandlers.permission!(permissionRequest); + assert.equal(replacementPermission.outcome.outcome, "selected"); + yield* runtimeInputs[1]!.onOutgoingResponse!("live-generation-2-permission-id"); + const urlElicitation = { + elicitationId: "replacement-url-id", + message: "Open replacement URL", + mode: "url" as const, + sessionId: "mock-session-1", + url: "https://example.com/replacement", + }; + const missingElicitationTransport = yield* replacementHandlers.elicitation!( + urlElicitation, + ).pipe(Effect.exit); + if (Exit.isSuccess(missingElicitationTransport)) { + assert.fail("a core elicitation request without transport correlation must fail closed"); + } + assert.include(Cause.pretty(missingElicitationTransport.cause), "Could not correlate"); + yield* runtimeInputs[1]!.onIncomingRequest!( + "live-generation-2-url-id", + "session/elicitation", + urlElicitation, + ); + yield* replacementHandlers.elicitation!(urlElicitation); + yield* runtimeInputs[1]!.onOutgoingResponse!("live-generation-2-url-id"); + + const collidingRequestPayload = { + sessionId: "mock-session-1", + toolCallId: "shared-request-id", + }; + const missingXAiTransport = yield* replacementHandlers.requestUserInput!({ + nativeItemId: "missing-generation-2-xai-item", + nativeMethod: "_x.ai/ask_user_question", + nativeRequestId: "shared-request-id", + nativeSessionId: "mock-session-1", + questions: [ + { + id: "approved", + header: "Approve", + question: "Approve missing transport?", + options: [{ label: "yes", description: "Approve" }], + }, + ], + }).pipe(Effect.exit); + if (Exit.isSuccess(missingXAiTransport)) { + assert.fail("an xAI user input request without transport correlation must fail closed"); + } + assert.include(Cause.pretty(missingXAiTransport.cause), "Could not correlate"); + yield* runtimeInputs[0]!.onIncomingRequest!( + "stale-generation-1-transport-id", + "x.ai/ask_user_question", + collidingRequestPayload, + ); + yield* runtimeInputs[1]!.onIncomingRequest!( + "live-generation-2-wrong-method-id", + "x.ai/ask_user_question", + collidingRequestPayload, + ); + yield* runtimeInputs[1]!.onIncomingRequest!( + "live-generation-2-transport-id", + "_x.ai/ask_user_question", + collidingRequestPayload, + ); + const replacementUserInputFiber = yield* replacementHandlers.requestUserInput!({ + nativeItemId: "live-generation-2-xai-item", + nativeMethod: "_x.ai/ask_user_question", + nativeRequestId: "shared-request-id", + nativeSessionId: "mock-session-1", + questions: [ + { + id: "approved", + header: "Approve", + question: "Approve live generation 2?", + options: [{ label: "yes", description: "Approve" }], + }, + ], + }).pipe(Effect.forkScoped); + let replacementRequest: ProviderAdapterV2Event | undefined; + while (replacementRequest === undefined) { + const event = yield* Queue.take(adapterEvents); + if (event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending") { + replacementRequest = event; + } + } + if (replacementRequest.type !== "runtime_request.updated") { + return yield* Effect.die("Expected generation 2 xAI user input request"); + } + const responseFiber = yield* runtime + .respondToRuntimeRequest({ + requestId: replacementRequest.runtimeRequest.id, + answers: { approved: ["yes"] }, + }) + .pipe(Effect.forkScoped); + const replacementUserInput = yield* Fiber.join(replacementUserInputFiber); + assert.deepEqual(replacementUserInput.answers, { approved: ["yes"] }); + yield* replacementUserInput.acknowledgeNativeResponse; + yield* runtimeInputs[0]!.onOutgoingResponse!("stale-generation-1-transport-id"); + yield* runtimeInputs[1]!.onOutgoingResponse!("live-generation-2-wrong-method-id"); + yield* Effect.yieldNow; + assert.isUndefined(responseFiber.pollUnsafe()); + yield* runtimeInputs[1]!.onOutgoingResponse!("live-generation-2-transport-id"); + yield* Fiber.join(responseFiber); + + yield* runtime.interruptTurn({ + providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }), + requestRuntimeRestart: true, + }); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("keeps stale deferred cleanup inert while replacement requests remain live", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: () => Effect.void, + environment: { T3_ACP_EMIT_TOOL_CALLS: "1" }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-stale-deferred-cleanup"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-stale-deferred-cleanup"), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ); + const oldPending = yield* Queue.take(events).pipe( + Effect.repeat({ + until: (event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + }), + ); + if ( + oldPending.type !== "runtime_request.updated" || + oldPending.runtimeRequest.providerTurnId === null + ) { + return yield* Effect.die("Expected the old runtime permission request"); + } + yield* runtime.interruptTurn({ + providerThread, + providerTurnId: oldPending.runtimeRequest.providerTurnId, + requestRuntimeRestart: true, + }); + const staleResponse = yield* runtime + .respondToRuntimeRequest({ + requestId: oldPending.runtimeRequest.id, + decision: "accept", + }) + .pipe(Effect.exit); + if (Exit.isSuccess(staleResponse)) { + assert.fail("teardown must synchronously remove the old pending request"); + } + while (Option.isSome(yield* Queue.poll(events))) { + // Discard generation 1 cancellation and terminal projection. + } + + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), + ); + let replacementPending: ProviderAdapterV2Event | undefined; + while (replacementPending === undefined) { + const event = yield* Queue.take(events); + if (event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending") { + replacementPending = event; + } + } + if (replacementPending.type !== "runtime_request.updated") { + return yield* Effect.die("Expected the replacement runtime permission request"); + } + assert.notEqual(replacementPending.runtimeRequest.id, oldPending.runtimeRequest.id); + yield* runtime.respondToRuntimeRequest({ + requestId: replacementPending.runtimeRequest.id, + decision: "accept", + }); + let replacementTerminal = false; + while (!replacementTerminal) { + const event = yield* Queue.take(events); + if ( + event.type === "runtime_request.updated" && + event.runtimeRequest.id === oldPending.runtimeRequest.id + ) { + assert.fail("stale deferred cleanup must not emit into generation 2"); + } + replacementTerminal = event.type === "turn.terminal"; + } + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("resolves owner cancellation and concurrent resume waiters after hard teardown", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const teardownStarted = yield* Deferred.make(); + const releaseTeardown = yield* Deferred.make(); + let runtimeOrdinalSeen = 0; + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + interruptPromptOnCancel: true, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: () => + Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseTeardown)), + ), + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = runtimeOrdinal; + return runtimeOrdinal === 1 ? { T3_ACP_HANG_PROMPT_FOREVER: "1" } : {}; + }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-concurrent-resume-teardown"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-concurrent-resume-teardown", ), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* pollProtocolMethods(protocolEvents); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true }) + .pipe(Effect.forkScoped); + yield* Deferred.await(teardownStarted); + const resumeFiber = yield* runtime + .resumeThread({ providerThread, modelSelection, runtimePolicy }) + .pipe(Effect.forkScoped); + const secondResumeFiber = yield* runtime + .resumeThread({ providerThread, modelSelection, runtimePolicy }) + .pipe(Effect.forkScoped); + const snapshotProviderThread = { + ...providerThread, + nativeThreadRef: { + ...providerThread.nativeThreadRef!, + nativeId: "mock-session-snapshot", + }, + }; + const snapshotFiber = yield* runtime + .readThreadSnapshot({ providerThread: snapshotProviderThread }) + .pipe(Effect.forkScoped); + const forkFiber = yield* runtime + .forkThread({ + sourceProviderThread: providerThread, + targetThreadId: ThreadId.make("thread-acp-concurrent-fork-after-teardown"), + }) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + assert.isUndefined(resumeFiber.pollUnsafe()); + assert.isUndefined(secondResumeFiber.pollUnsafe()); + assert.isUndefined(snapshotFiber.pollUnsafe()); + assert.isUndefined(forkFiber.pollUnsafe()); + assert.equal(runtimeOrdinalSeen, 1); + const methodsDuringTeardown = yield* pollProtocolMethods(protocolEvents); + assert.notInclude(methodsDuringTeardown, "session/load"); + assert.notInclude(methodsDuringTeardown, "session/fork"); + const cancelInterruptOwner = yield* Fiber.interrupt(interruptFiber).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + assert.isUndefined(cancelInterruptOwner.pollUnsafe()); + yield* Deferred.succeed(releaseTeardown, undefined); + yield* Fiber.join(cancelInterruptOwner); + yield* Fiber.join(resumeFiber); + yield* Fiber.join(secondResumeFiber); + yield* Fiber.join(snapshotFiber); + yield* Fiber.join(forkFiber); + assert.equal(runtimeOrdinalSeen, 2); + const methodsAfterRestart = yield* pollProtocolMethods(protocolEvents); + assert.equal(methodsAfterRestart.filter((method) => method === "session/load").length, 2); + assert.equal(methodsAfterRestart.filter((method) => method === "session/fork").length, 1); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("direct Stop skips uninterruptible ACP cancel and recovers after native teardown", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); - if ( - pendingRequest.type !== "runtime_request.updated" || - pendingRequest.runtimeRequest.providerTurnId === null - ) { - return yield* Effect.die("Expected a pending ACP permission request with a provider turn"); + const protocolEvents = yield* Queue.bounded(256); + const commandPidPath = yield* fileSystem.makeTempFileScoped({ + prefix: "t3-acp-direct-stop-command-", + }); + yield* Effect.addFinalizer(() => + Effect.sync(() => cleanupPublishedDetachedFixture(commandPidPath)), + ); + let cancelCalled = false; + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + extractBackgroundTaskId: (toolCall) => + toolCall.toolCallId === "tool-call-running-1" ? "task-running-1" : undefined, + extractBackgroundTaskCompletion: (toolCall) => + toolCall.toolCallId === "tool-call-output-1" + ? [{ taskId: "task-running-1", status: "running", appendOutput: "" }] + : [], + interruptPromptOnCancel: true, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + ownDescendantProcessGroups: true, + ownDetachedProcessGroup: true, + environment: (runtimeOrdinal) => + runtimeOrdinal === 1 + ? { + T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG: "1", + T3_ACP_EMIT_LATE_UPDATE_AFTER_CANCEL: "1", + T3_ACP_RUNNING_COMMAND_PID_PATH: commandPidPath, + T3_ACP_RUNNING_COMMAND_IGNORE_TERM: "1", + T3_ACP_RUNNING_COMMAND_SEPARATE_SESSION: "1", + } + : {}, + protocolEvents, + wrapCancel: () => + Effect.sync(() => { + cancelCalled = true; + }).pipe(Effect.andThen(Effect.never)), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { offer: () => Effect.void }, + }); + const threadId = ThreadId.make("thread-acp-direct-stop-running-command"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-direct-stop-running-command", + ), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + + let runningMonitorSeen = false; + let runningTaskOutputSeen = false; + while (!runningMonitorSeen || !runningTaskOutputSeen) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + (event.turnItem.status === "running" || event.turnItem.status === "pending") + ) { + if (event.turnItem.nativeItemRef?.nativeId === "tool-call-running-1") { + runningMonitorSeen = true; + } + if (event.turnItem.nativeItemRef?.nativeId === "tool-call-output-1") { + runningTaskOutputSeen = true; + } + } } + const [commandLauncherPid, commandRootPid, commandSleepPid] = Option.getOrThrow( + yield* waitForPublishedProcessIds(fileSystem, commandPidPath, 3), + ); + assert.isTrue( + Option.isSome( + yield* waitForProcesses([commandLauncherPid!, commandRootPid!, commandSleepPid!]), + ), + "declared direct Stop launcher, Bash, and sleep PIDs must become live", + ); - yield* runtime.interruptTurn({ - providerThread, - providerTurnId: pendingRequest.runtimeRequest.providerTurnId, + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", }); + yield* Queue.takeAll(events); + const interruptFiber = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId: firstProviderTurnId, + requestRuntimeRestart: true, + }) + .pipe(Effect.forkScoped); + const interruptCompleted = yield* Fiber.join(interruptFiber).pipe( + Effect.timeoutOption("3 seconds"), + ); + assert.isTrue(Option.isSome(interruptCompleted), "hung ACP cancel must not block teardown"); + assert.isFalse(cancelCalled, "hard process-group teardown must skip ACP cancel"); + yield* Effect.sleep("250 millis"); + assert.isFalse(processExists(commandLauncherPid!)); + assert.isFalse(processExists(commandRootPid!)); + assert.isFalse(processExists(commandSleepPid!)); - const cancelledRequest = Option.getOrThrow( + let terminalStatus: string | null = null; + let openToolTerminalStatus: string | null = null; + let openToolInterruptedExitCode: number | undefined = 42; + let lateAfterCancelSeen = false; + let runningMonitorAfterInterrupt = false; + while (terminalStatus === null || openToolTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.nativeItemRef?.nativeId === "tool-call-running-1" && + event.turnItem.status === "running" + ) { + runningMonitorAfterInterrupt = true; + if (event.turnItem.type === "command_execution") { + assert.equal( + event.turnItem.exitCode, + undefined, + "mid-stream exit_code 0 must not project while the command is still running", + ); + } + } + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + (event.turnItem.status === "failed" || + event.turnItem.status === "cancelled" || + event.turnItem.status === "interrupted" || + event.turnItem.status === "completed") + ) { + openToolTerminalStatus = event.turnItem.status; + openToolInterruptedExitCode = event.turnItem.exitCode; + } + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + terminalStatus = event.status; + } + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes("late after cancel") + ) { + lateAfterCancelSeen = true; + } + } + assert.equal(terminalStatus, "interrupted"); + assert.equal(openToolTerminalStatus, "interrupted"); + assert.equal( + openToolInterruptedExitCode, + undefined, + "interrupted commands must not retain a mid-stream exit_code 0", + ); + assert.isFalse(runningMonitorAfterInterrupt); + assert.isFalse(yield* runtime.hasPendingBackgroundWork!); + assert.isFalse( + lateAfterCancelSeen, + "late post-Stop assistant text must not attach to the stopped run", + ); + + // Give residual cancel-path updates a chance to mis-project if quarantine fails. + yield* Effect.sleep("200 millis"); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + let residual = yield* Queue.poll(events); + while (Option.isSome(residual)) { + const event = residual.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.nativeItemRef?.nativeId === "tool-call-running-1" + ) { + assert.notEqual(event.turnItem.status, "running"); + } + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes("late after cancel") + ) { + lateAfterCancelSeen = true; + } + residual = yield* Queue.poll(events); + } + assert.isFalse(lateAfterCancelSeen, "quarantine must drop residual stopped-run events"); + + const secondNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ); + const loadAfterRestart = yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => event.direction === "outgoing" && rawProtocolMethod(event) === "session/load", + ), + Stream.runHead, + ); + assert.isTrue( + Option.isSome(loadAfterRestart), + "Direct Stop follow-up must respawn the ACP runtime", + ); + + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let secondTerminal: string | null = null; + let stoppedRunTextOnFollowUp = false; + while (secondTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes("late after cancel") + ) { + stoppedRunTextOnFollowUp = true; + } + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminal = event.status; + } + } + assert.equal(secondTerminal, "completed"); + assert.isFalse( + stoppedRunTextOnFollowUp, + "stopped-run residual text must not attach to the follow-up run", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "direct Stop on a deferred subagent hold terminalizes the subagent and does not carry it forward", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + restartRuntimeAfterInterrupt: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-direct-stop-subagent-hold"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-direct-stop-subagent-hold", + ), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( Stream.filter( (event) => - event.type === "runtime_request.updated" && - event.runtimeRequest.id === pendingRequest.runtimeRequest.id && - event.runtimeRequest.status === "cancelled", + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), ), Stream.runHead, - ), - ); - assert.equal(cancelledRequest.type, "runtime_request.updated"); - }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId: firstProviderTurnId, + requestRuntimeRestart: true, + }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let subagentStatus: string | null = null; + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { + subagentStatus = event.turnItem.status; + } + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.equal(subagentStatus, "interrupted"); + + subagentPhase = "complete"; + const secondNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ); + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let carriedCompleted = false; + let secondTerminalStatus: string | null = null; + while (secondTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + carriedCompleted = true; + } + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminalStatus = event.status; + } + } + assert.equal(secondTerminalStatus, "completed"); + assert.isFalse( + carriedCompleted, + "Direct Stop must not carry a stopped subagent into the follow-up run", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), ); - it.effect("does not release an ACP turn when cancellation is not acknowledged", () => + it.live("restart_active terminates native work and reloads a clean runtime", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; @@ -473,26 +6144,49 @@ describe("AcpAdapterV2", () => { const mockAgentPath = yield* path.fromFileUrl( new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); - const instanceId = ProviderInstanceId.make("acp-test"); const protocolEvents = yield* Queue.bounded(256); + const commandPidPath = yield* fileSystem.makeTempFileScoped({ + prefix: "t3-acp-restart-active-command-", + }); + yield* Effect.addFinalizer(() => + Effect.sync(() => cleanupPublishedDetachedFixture(commandPidPath)), + ); + const instanceId = ProviderInstanceId.make("acp-test"); const adapter = makeAcpAdapterV2({ crypto: yield* Crypto.Crypto, instanceId, flavor: { driver: ACP_TEST_DRIVER, capabilities: AcpProviderCapabilitiesV2, + interruptPromptOnCancel: true, + restartRuntimeAfterInterrupt: true, + restartRuntimeOnEveryInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath, - environment: { T3_ACP_PROMPT_DELAY_MS: "5000" }, + ownDescendantProcessGroups: true, + ownDetachedProcessGroup: true, + processGroupTerminationGrace: 0, + environment: (runtimeOrdinal) => + runtimeOrdinal === 1 + ? { + T3_ACP_EXIT_ON_CANCEL: "1", + T3_ACP_EMIT_LATE_UPDATE_AFTER_CANCEL: "1", + T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG: "1", + T3_ACP_RUNNING_COMMAND_PID_PATH: commandPidPath, + T3_ACP_RUNNING_COMMAND_SEPARATE_SESSION: "1", + } + : {}, protocolEvents, + wrapCancel: (cancel) => cancel.pipe(Effect.andThen(Effect.sleep("250 millis"))), }), }, fileSystem, idAllocator, serverConfig, }); - const threadId = ThreadId.make("thread-acp-cancel-timeout"); + const threadId = ThreadId.make("thread-acp-restart-active-in-process"); const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ runtimeMode: "full-access", interactionMode: "default", @@ -501,50 +6195,76 @@ describe("AcpAdapterV2", () => { const modelSelection = { instanceId, model: "default" } as const; const runtime = yield* adapter.openSession({ threadId, - providerSessionId: ProviderSessionId.make("provider-session-acp-cancel-timeout"), + providerSessionId: ProviderSessionId.make("provider-session-acp-restart-active-in-process"), modelSelection, runtimePolicy, }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); const providerThread = yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy, }); const now = yield* DateTime.now; - const firstTurn = makeTurnInput({ - threadId, - providerThread, - instanceId, - runtimePolicy, - now, - }); - yield* runtime.startTurn(firstTurn); - yield* Stream.fromQueue(protocolEvents).pipe( - Stream.filter( - (event) => - event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + + let runningToolSeen = false; + while (!runningToolSeen) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + (event.turnItem.status === "running" || event.turnItem.status === "pending") + ) { + runningToolSeen = true; + } + } + const [commandLauncherPid, commandRootPid, commandSleepPid] = Option.getOrThrow( + yield* waitForPublishedProcessIds(fileSystem, commandPidPath, 3), + ); + assert.isTrue( + Option.isSome( + yield* waitForProcesses([commandLauncherPid!, commandRootPid!, commandSleepPid!]), ), - Stream.runHead, + "declared restart_active launcher, Bash, and sleep PIDs must become live", ); - const providerTurnId = idAllocator.derive.providerTurn({ + + const firstProviderTurnId = idAllocator.derive.providerTurn({ driver: ACP_TEST_DRIVER, nativeTurnId: "mock-session-1:turn:1", }); + // restart_active path: interrupt without requestRuntimeRestart. const interruptFiber = yield* runtime - .interruptTurn({ providerThread, providerTurnId }) - .pipe(Effect.flip, Effect.forkScoped); - yield* Stream.fromQueue(protocolEvents).pipe( - Stream.filter( - (event) => - event.direction === "outgoing" && rawProtocolMethod(event) === "session/cancel", - ), - Stream.runHead, - ); - yield* TestClock.adjust("10 seconds"); - const interruptError = yield* Fiber.join(interruptFiber); - assert.equal(interruptError._tag, "ProviderAdapterInterruptError"); + .interruptTurn({ + providerThread, + providerTurnId: firstProviderTurnId, + }) + .pipe(Effect.forkScoped); + yield* Fiber.join(interruptFiber); + yield* Effect.sleep("250 millis"); + assert.isFalse(processExists(commandLauncherPid!)); + assert.isFalse(processExists(commandRootPid!)); + assert.isFalse(processExists(commandSleepPid!)); - const secondTurnError = yield* runtime + let firstTerminal: string | null = null; + while (firstTerminal === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminal = event.status; + } + } + assert.equal(firstTerminal, "interrupted"); + + yield* Queue.takeAll(protocolEvents); + yield* runtime .startTurn( makeTurnInput({ threadId, @@ -555,8 +6275,331 @@ describe("AcpAdapterV2", () => { ordinal: 2, }), ) - .pipe(Effect.flip); - assert.equal(secondTurnError._tag, "ProviderAdapterTurnStartError"); + .pipe(Effect.forkScoped); + let loadSeen = false; + let promptSeen = false; + while (!loadSeen || !promptSeen) { + const event = yield* Queue.take(protocolEvents); + if (event.direction !== "outgoing") continue; + const method = rawProtocolMethod(event); + loadSeen ||= method === "session/load"; + promptSeen ||= method === "session/prompt"; + } + assert.isTrue(loadSeen, "restart_active must replay session/load on a new ACP process"); + assert.isTrue(promptSeen, "replacement prompt must start after reload"); + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let staleEventSeen = false; + let secondTerminal: string | null = null; + while (secondTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.nativeItemRef?.nativeId === "tool-call-running-1" + ) { + staleEventSeen = true; + } + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes("late after cancel") + ) { + staleEventSeen = true; + } + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminal = event.status; + } + } + assert.equal(secondTerminal, "completed"); + assert.isFalse(staleEventSeen, "interrupted runtime events must not attach to attempt 2"); }).pipe(Effect.provide(testLayer), Effect.scoped), ); }); + +describe("acpPostSettleWakeEvidence", () => { + const sessionId = "session-wake"; + + it("accepts assistant text and tool updates as wake evidence", () => { + assert.isTrue( + acpPostSettleWakeEvidence({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "Subagent finished. SUBAGENT_DONE" }, + }, + }), + ); + assert.isTrue( + acpPostSettleWakeEvidence({ + sessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: "call-1", + title: "get_command_or_subagent_output", + status: "pending", + kind: "other", + content: [], + locations: [], + rawInput: {}, + }, + }), + ); + }); + + it("rejects monitor end chatter and background mutations", () => { + assert.isFalse( + acpPostSettleWakeEvidence({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { + type: "text", + text: 'Monitor "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" ended', + }, + }, + }), + ); + assert.isFalse( + acpPostSettleWakeEvidence( + { + sessionId, + update: { + sessionUpdate: "user_message_chunk", + content: { type: "text", text: "task-1 completed" }, + }, + }, + { + extractBackgroundToolMutation: () => [ + { + taskId: "task-1", + status: "completed", + appendOutput: "", + }, + ], + }, + ), + ); + }); +}); + +describe("acpPostSettleContinuationOfferEvidence", () => { + const sessionId = "session-wake-offer"; + + it("offers on assistant text and terminal tool status", () => { + assert.isTrue( + acpPostSettleContinuationOfferEvidence({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "Background shell finished." }, + }, + }), + ); + assert.isTrue( + acpPostSettleContinuationOfferEvidence({ + sessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "call-1", + title: "run_terminal_command", + status: "completed", + kind: "other", + content: [], + locations: [], + rawInput: {}, + }, + }), + ); + assert.isTrue( + acpPostSettleContinuationOfferEvidence({ + sessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "call-2", + title: "run_terminal_command", + status: "failed", + kind: "other", + content: [], + locations: [], + rawInput: {}, + }, + }), + ); + }); + + it("does not offer on thought-only chunks", () => { + assert.isFalse( + acpPostSettleContinuationOfferEvidence({ + sessionId, + update: { + sessionUpdate: "agent_thought_chunk", + content: { type: "text", text: "Still reasoning about the monitor output…" }, + }, + }), + ); + // Thoughts may still be wake evidence for buffering once a real offer opens. + assert.isTrue( + acpPostSettleWakeEvidence({ + sessionId, + update: { + sessionUpdate: "agent_thought_chunk", + content: { type: "text", text: "Still reasoning about the monitor output…" }, + }, + }), + ); + }); + + it("buffers in-progress tool updates without offering", () => { + assert.isTrue( + acpPostSettleWakeEvidence({ + sessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "call-1", + title: "run_terminal_command", + status: "in_progress", + kind: "other", + content: [], + locations: [], + rawInput: {}, + }, + }), + ); + assert.isFalse( + acpPostSettleContinuationOfferEvidence({ + sessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "call-1", + title: "run_terminal_command", + status: "in_progress", + kind: "other", + content: [], + locations: [], + rawInput: {}, + }, + }), + ); + assert.isFalse( + acpPostSettleContinuationOfferEvidence({ + sessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: "call-1", + title: "run_terminal_command", + status: "pending", + kind: "other", + content: [], + locations: [], + rawInput: {}, + }, + }), + ); + }); + + it("does not offer filtered monitor chatter", () => { + assert.isFalse( + acpPostSettleContinuationOfferEvidence({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { + type: "text", + text: 'Monitor "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" ended', + }, + }, + }), + ); + }); + + it("does not offer on a normalized monitor start ACK despite raw completed status", () => { + const monitorStartAck = { + sessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "call-monitor", + title: "Tool", + status: "completed", + kind: "other", + content: [], + locations: [], + rawInput: { variant: "Monitor", description: "stream test" }, + rawOutput: { + type: "Monitor", + taskId: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + timeoutMs: 60000, + }, + }, + } as const; + // Raw frame looks terminal; the Grok flavor knows it is a running monitor. + assert.isTrue(acpPostSettleContinuationOfferEvidence(monitorStartAck)); + assert.isFalse( + acpPostSettleContinuationOfferEvidence(monitorStartAck, { + normalizeToolCall: normalizeXAiAcpToolCallState, + }), + ); + }); +}); + +describe("acpPostSettleWakeShouldBuffer", () => { + const sessionId = "session-wake-buffer"; + + it("drops agent progress chatter while background work is running", () => { + for (const sessionUpdate of ["agent_message_chunk", "agent_thought_chunk"] as const) { + assert.isFalse( + acpPostSettleWakeShouldBuffer( + { + sessionId, + update: { + sessionUpdate, + content: { type: "text", text: "Still running." }, + }, + }, + true, + ), + ); + } + }); + + it("retains tool state while running and agent output after completion", () => { + const agentMessage = { + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "Monitor finished successfully." }, + }, + } as const; + const toolUpdate = { + sessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "call-monitor", + title: "monitor", + status: "in_progress", + kind: "other", + content: [], + locations: [], + rawInput: {}, + }, + } as const; + + assert.isTrue(acpPostSettleWakeShouldBuffer(toolUpdate, true)); + assert.isTrue(acpPostSettleWakeShouldBuffer(agentMessage, false)); + }); +}); + +describe("acpPostSettleMonitorPromptShouldSuppress", () => { + it("suppresses running monitor prompts but not terminal notices", () => { + assert.isTrue( + acpPostSettleMonitorPromptShouldSuppress({ taskId: "task-active", status: "running" }), + ); + assert.isFalse( + acpPostSettleMonitorPromptShouldSuppress({ taskId: "task-ended", status: "completed" }), + ); + assert.isFalse( + acpPostSettleMonitorPromptShouldSuppress({ taskId: "task-failed", status: "failed" }), + ); + }); +}); diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts index 5a6517d9652..af70062ebfb 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts @@ -18,6 +18,7 @@ import { type ProviderInstanceId, type ProviderDriverKind, type ProviderRequestKind, + type ProviderThreadId, type ProviderUserInputAnswers, type RuntimeRequestId, type ThreadId, @@ -28,11 +29,14 @@ import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; -import type * as Scope from "effect/Scope"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as EffectAcpErrors from "effect-acp/errors"; import type * as EffectAcpSchema from "effect-acp/schema"; @@ -53,6 +57,7 @@ import type { } from "../../provider/acp/AcpSessionRuntime.ts"; import * as AcpSessionRuntime from "../../provider/acp/AcpSessionRuntime.ts"; import { IdAllocatorV2, type IdAllocatorV2Shape } from "../IdAllocator.ts"; +import { type ProviderContinuationRequest } from "../ProviderContinuationRequests.ts"; import { makeProviderFailure } from "../ProviderFailure.ts"; import { acpSelectionTransition } from "../ProviderSelectionTransition.ts"; import { @@ -88,11 +93,15 @@ export const ACP_PROTOCOL = "acp.ndjson-jsonrpc" as const; export interface AcpAdapterV2RuntimeInput { readonly cwd: string; readonly mcpServers: ReadonlyArray; - readonly interruptPromptOnCancel: false; + readonly interruptPromptOnCancel?: boolean; readonly clientCapabilities: EffectAcpSchema.InitializeRequest["clientCapabilities"]; readonly clientInfo: AcpSessionRuntimeOptions["clientInfo"]; readonly requestLogger?: NonNullable; readonly protocolLogging: NonNullable; + readonly onIncomingRequest?: AcpSessionRuntimeOptions["onIncomingRequest"]; + readonly onTermination: NonNullable; + readonly onOutgoingResponseFailure?: AcpSessionRuntimeOptions["onOutgoingResponseFailure"]; + readonly onOutgoingResponse?: AcpSessionRuntimeOptions["onOutgoingResponse"]; } export type AcpAdapterV2NativeLogging = Pick< @@ -102,15 +111,79 @@ export type AcpAdapterV2NativeLogging = Pick< export interface AcpAdapterV2UserInputRequest { readonly nativeItemId: string; + readonly nativeMethod?: string; readonly nativeRequestId: string; + readonly nativeSessionId?: string; readonly questions: ReadonlyArray; } export interface AcpAdapterV2ExtensionContext { readonly runtime: AcpSessionRuntime.AcpSessionRuntime["Service"]; - readonly requestUserInput: ( - input: AcpAdapterV2UserInputRequest, - ) => Effect.Effect; + /** + * Session-scoped background-task lifecycle reported via extension + * notifications (e.g. Grok `_x.ai/task_backgrounded` after a soft cancel + * detaches foreground work). Mutations for non-root sessions are ignored. + */ + readonly applyBackgroundTaskMutation: (mutation: { + readonly sessionId: string; + readonly taskId: string; + readonly status: "running" | "completed" | "failed"; + }) => Effect.Effect; + readonly requestUserInput: (input: AcpAdapterV2UserInputRequest) => Effect.Effect< + { + readonly acknowledgeNativeResponse: Effect.Effect; + readonly answers: ProviderUserInputAnswers | null; + }, + EffectAcpErrors.AcpError + >; +} + +export interface AcpRootTurnIdleSnapshot { + readonly finalized: boolean; + readonly interrupted: boolean; + readonly assistantStreamOpen: boolean; + readonly reasoningStreamOpen: boolean; + readonly hasRunningTool: boolean; + readonly hasPendingRuntimeRequest: boolean; + readonly hasToolHistory: boolean; + readonly hasRunningSubagent: boolean; + readonly hasOutput: boolean; +} + +/** + * Debounce used if a flavor re-enables speculative idle settlement. + * Kept for tests and future root-matched recovery; Grok no longer idle-settles. + */ +export const acpRootTurnSettleDebounceMs = 2_000; + +/** Let trailing root session chunks land before terminalizing a settled turn. */ +export const acpRootTurnCompletionDrainMs = 100; + +/** + * True when root-session streaming is quiescent enough for speculative settle. + * + * Always false today: settling on "assistant text then quiet" over-settles Grok + * preamble-before-tools turns, and settling after tools drops later tool waves + * while `session/prompt` is still open. Terminalize from the prompt RPC (or a + * future root-matched completion signal), not from local silence. + */ +export function acpRootTurnIsIdle(snapshot: AcpRootTurnIdleSnapshot): boolean { + if (snapshot.finalized || snapshot.interrupted) return false; + if (snapshot.assistantStreamOpen || snapshot.reasoningStreamOpen) return false; + if (snapshot.hasRunningTool || snapshot.hasPendingRuntimeRequest) return false; + if (snapshot.hasRunningSubagent) return false; + if (!snapshot.hasOutput) return false; + // Structural gates above stay for unit tests / future re-enable. Speculative + // idle completion is intentionally disabled. + return false; +} + +/** True when idle settle should be (re-)scheduled after pending runtime work clears. */ +export function acpRootTurnShouldRearmRecoveryTimers(context: { + readonly finalized: boolean; + readonly interrupted: boolean; +}): boolean { + return !context.finalized && !context.interrupted; } export interface AcpAdapterV2Flavor { @@ -130,7 +203,117 @@ export interface AcpAdapterV2Flavor { readonly extractSubagentUpdate?: ( toolCall: AcpToolCallState, ) => AcpAdapterV2SubagentUpdate | undefined; + /** + * Optional Grok-style rewrite before tool projection (e.g. keep monitor start + * ACKs in the running state until stream end). + */ + readonly normalizeToolCall?: (toolCall: AcpToolCallState) => AcpToolCallState; + /** + * Optional mapping from a long-lived background tool start ACK to a task id + * (e.g. monitor task uuid) so later synthetic text events can update it. + */ + readonly extractBackgroundTaskId?: (toolCall: AcpToolCallState) => string | undefined; + /** + * Optional parse of root-session synthetic text (monitor-event lines, monitor + * ended reminders). Returns every task mutation in the chunk so coalesced + * progress / end notices are not dropped. + */ + readonly extractBackgroundToolMutation?: (text: string) => ReadonlyArray<{ + readonly taskId: string; + readonly status: "running" | "completed" | "failed"; + readonly appendOutput: string; + }>; + /** + * Optional parse of root-session synthetic text announcing a background + * subagent's end ("Background subagent "" ... completed successfully"). + * The agent may never hydrate via get_command_or_subagent_output, so this + * notice can be the only terminal signal for the subagent row. + */ + readonly extractSubagentEndNotice?: (text: string) => + | { + readonly childSessionId: string; + readonly status: "completed" | "failed"; + } + | undefined; + /** + * Optional hydration when a later tool (e.g. get_command TaskOutput) completes + * previously registered background task id(s). + */ + readonly extractBackgroundTaskCompletion?: (toolCall: AcpToolCallState) => ReadonlyArray<{ + readonly taskId: string; + readonly status: "running" | "completed" | "failed"; + readonly appendOutput: string; + }>; + /** + * Persistent monitors (e.g. Grok `persistent: true`) should not hold root-turn + * deferred finalize open forever. Still tracked for post-settle wake. + */ + readonly isPersistentBackgroundTool?: (toolCall: AcpToolCallState) => boolean; + /** + * When true, keep the active turn open after session/prompt returns while + * background tools/subagents are still running so later monitor/wake traffic + * can project (Grok monitors finish after the root prompt settles). + */ + readonly deferFinalizeForBackgroundWork?: boolean; readonly assertComplete?: Effect.Effect; + /** + * When true, schedule speculative local settlement after root session + * quiet. Disabled for Grok: short idle windows over-settle preamble-before- + * tools turns and `session/cancel` from that path freezes projection while + * the agent keeps working. Prefer `session/prompt` return (or a future + * root-matched terminal signal). + */ + readonly settleRootTurnWhenIdle?: boolean; + /** Interrupt the local prompt fiber before `session/cancel` (Grok wedged prompts). */ + readonly interruptPromptOnCancel?: boolean; + /** + * Kill and respawn the ACP child process before the next `session/prompt` after a + * user interrupt. Grok can keep `task_already_running` state until the process exits. + */ + readonly restartRuntimeAfterInterrupt?: boolean; + /** + * When true, every interrupt restarts the runtime, not just those carrying + * `requestRuntimeRestart` (user Stop). Leave unset to keep non-Stop + * interrupts (steering, restart_active) soft: `session/cancel` plus session + * reuse in the same process. + */ + readonly restartRuntimeOnEveryInterrupt?: boolean; + readonly terminateRuntimeProcessGroupOnInterrupt?: boolean; + /** + * When true, an interrupt without `requestRuntimeRestart` (steering restart) + * on a turn whose native prompt already settled skips the hard process-group + * kill, the ACP cancel, and the runtime respawn entirely: the turn + * terminalizes locally while background subagents keep running in the same + * process and carry over into the replacement turn. Verified against the + * real Grok CLI (tmp/grok-acp-experiments E1): a new session/prompt is + * accepted concurrently while a fire-and-forget subagent is still running, + * with no task_already_running. User Stop (`requestRuntimeRestart: true`) + * keeps the hard teardown; mid-prompt non-Stop interrupts go soft + * (`session/cancel` plus same-session re-prompt) unless + * `restartRuntimeOnEveryInterrupt` is set. + */ + readonly preserveRuntimeOnSettledInterrupt?: boolean; + /** + * When true (with continuationRequests), post-settle root session/update traffic + * buffers and requests a provider continuation run instead of being dropped or + * only appended to loaded history. + */ + readonly enablePostSettleContinuation?: boolean; + /** + * When true, send image attachment content blocks even if the ACP agent + * advertises `promptCapabilities.image: false`. Grok CLI currently accepts + * and vision-processes image blocks while still reporting the capability as + * false; without this override, screenshot turns fail before `session/prompt`. + */ + readonly supportsImagePrompts?: boolean; +} + +/** Whether image attachment blocks may be included in session/prompt. */ +export function acpSupportsImagePrompts(input: { + readonly flavorSupportsImagePrompts?: boolean | undefined; + readonly negotiatedImage?: boolean | undefined; +}): boolean { + return input.flavorSupportsImagePrompts === true || input.negotiatedImage === true; } export interface AcpAdapterV2SubagentUpdate { @@ -138,9 +321,14 @@ export interface AcpAdapterV2SubagentUpdate { readonly prompt: string; readonly title: string | null; readonly model: string | null; - readonly status: "running" | "completed" | "failed"; + readonly status: "running" | "completed" | "failed" | "interrupted" | "cancelled"; readonly childSessionId: string | null; readonly result: string | null; + /** + * When false, still project a normal tool turn item after the subagent update + * (hydration tools like get_command_or_subagent_output). Defaults to true. + */ + readonly suppressNormalTool?: boolean; } export interface AcpAdapterV2Options { @@ -151,6 +339,36 @@ export interface AcpAdapterV2Options { readonly idAllocator: IdAllocatorV2Shape; readonly serverConfig: ServerConfig["Service"]; readonly nativeLogging?: (threadId: ThreadId) => AcpAdapterV2NativeLogging; + /** + * Shared with ProviderContinuationService so post-settle wake traffic can start + * a continuation run. Optional: adapters that omit it keep pre-continuation drop + * / history-only behavior for null-activeTurn updates. + */ + readonly continuationRequests?: { + readonly offer: (request: ProviderContinuationRequest) => Effect.Effect; + }; + readonly testHooks?: { + readonly afterNativeResponseTransportClosed?: () => Effect.Effect; + readonly afterHardTeardownTransportDrained?: () => Effect.Effect; + readonly beforeNativeResponseAdmissionCheck?: ( + generation: number, + requestId: string, + ) => Effect.Effect; + readonly onNativeResponseLifecycle?: (event: { + readonly generation?: number; + readonly requestId?: string; + readonly type: + | "admission_rejected" + | "failed" + | "late_noop" + | "registered" + | "removed" + | "timer_exited" + | "timer_started" + | "watcher_exited" + | "watcher_started"; + }) => Effect.Effect; + }; } export const AcpProviderCapabilitiesV2 = { @@ -355,14 +573,114 @@ function unknownRecord(value: unknown): Record | undefined { : undefined; } +export function acpCanonicalJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map(acpCanonicalJson).join(",")}]`; + } + const record = unknownRecord(value); + if (record !== undefined) { + return `{${Object.keys(record) + .toSorted() + .map((key) => `${JSON.stringify(key)}:${acpCanonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} + +export function acpNativeUserInputRequestMatches( + request: Pick< + AcpAdapterV2UserInputRequest, + "nativeMethod" | "nativeRequestId" | "nativeSessionId" + >, + transport: { readonly method: string; readonly payload: unknown }, +): boolean { + if ( + request.nativeMethod === undefined || + request.nativeMethod.trim().length === 0 || + request.nativeRequestId.trim().length === 0 || + request.nativeSessionId === undefined || + request.nativeSessionId.trim().length === 0 + ) { + return false; + } + if ( + transport.method !== "x.ai/ask_user_question" && + transport.method !== "_x.ai/ask_user_question" + ) { + return false; + } + if (transport.method !== request.nativeMethod) { + return false; + } + const payloadRecord = unknownRecord(transport.payload); + const paramsRecord = unknownRecord(payloadRecord?.params) ?? payloadRecord; + return ( + paramsRecord?.toolCallId !== undefined && + String(paramsRecord.toolCallId).trim().length > 0 && + String(paramsRecord.toolCallId) === request.nativeRequestId && + paramsRecord.sessionId !== undefined && + String(paramsRecord.sessionId).trim().length > 0 && + String(paramsRecord.sessionId) === request.nativeSessionId + ); +} + +export function acpClaimNativeTransportRequest< + T extends { + readonly generation: number; + readonly requestId: string; + readonly sequence: number; + }, +>( + requests: ReadonlyArray, + generation: number, + predicate: (request: T) => boolean, +): readonly [string | undefined, Array] { + let claimedIndex = -1; + let claimedSequence = Number.POSITIVE_INFINITY; + for (let index = 0; index < requests.length; index += 1) { + const request = requests[index]!; + if ( + request.generation === generation && + request.sequence < claimedSequence && + predicate(request) + ) { + claimedIndex = index; + claimedSequence = request.sequence; + } + } + if (claimedIndex < 0) return [undefined, [...requests]]; + return [ + requests[claimedIndex]!.requestId, + [...requests.slice(0, claimedIndex), ...requests.slice(claimedIndex + 1)], + ]; +} + function nonEmptyText(value: unknown, fallback: string): string { return typeof value === "string" && value.trim().length > 0 ? value.trim() : fallback; } +function decodeByteText(value: unknown): string | undefined { + if (!Array.isArray(value) || value.length === 0) return undefined; + if (!value.every((entry) => typeof entry === "number" && Number.isInteger(entry))) { + return undefined; + } + try { + // Preserve leading/trailing whitespace like the string path in textFromUnknown. + const text = new TextDecoder().decode(Uint8Array.from(value as number[])); + return text.length > 0 ? text : undefined; + } catch { + return undefined; + } +} + function textFromUnknown(value: unknown): string | undefined { if (typeof value === "string") { return value; } + const fromBytes = decodeByteText(value); + if (fromBytes !== undefined) { + return fromBytes; + } if (Array.isArray(value)) { const parts = value.flatMap((entry) => { const text = textFromUnknown(entry); @@ -374,12 +692,33 @@ function textFromUnknown(value: unknown): string | undefined { if (record === undefined) { return undefined; } - for (const key of ["stdout", "stderr", "output", "content", "text", "message"]) { - const text = textFromUnknown(record[key]); + // Prefer prompt-facing Grok fields before nested envelopes. + for (const key of [ + "output_for_prompt", + "stdout", + "stderr", + "output", + "content", + "text", + "message", + ]) { + const direct = record[key]; + if (typeof direct === "string" && direct.length > 0) { + return direct; + } + const decoded = decodeByteText(direct); + if (decoded !== undefined) { + return decoded; + } + const text = textFromUnknown(direct); if (text !== undefined && text.length > 0) { return text; } } + const result = unknownRecord(record.Result) ?? unknownRecord(record.result); + if (result !== undefined) { + return textFromUnknown(result); + } return undefined; } @@ -394,6 +733,21 @@ function commandExitCode(value: unknown): number | undefined { return undefined; } +/** + * Project an exit code only when the tool has a terminal native status. + * Mid-stream Grok Bash re-reports carry exit_code 0 while still in progress; + * interrupted tools must not retain that stale success code. + */ +export function acpProjectedCommandExitCode( + status: "pending" | "running" | "completed" | "failed" | "interrupted", + rawOutput: unknown, +): number | undefined { + if (status !== "completed" && status !== "failed") { + return undefined; + } + return commandExitCode(rawOutput); +} + function pathFromToolCall(toolCall: AcpToolCallState): string | undefined { const locations = toolCall.data.locations; if (Array.isArray(locations)) { @@ -446,15 +800,14 @@ function toolStatus( } } -function nodeStatus(status: ReturnType): OrchestrationV2ExecutionNode["status"] { +type ProjectedToolStatus = ReturnType | "interrupted"; + +function nodeStatus(status: ProjectedToolStatus): OrchestrationV2ExecutionNode["status"] { return status === "pending" ? "running" : status; } -function completedAtForStatus( - status: ReturnType, - now: DateTime.Utc, -): DateTime.Utc | null { - return status === "completed" || status === "failed" ? now : null; +function completedAtForStatus(status: ProjectedToolStatus, now: DateTime.Utc): DateTime.Utc | null { + return status === "completed" || status === "failed" || status === "interrupted" ? now : null; } function selectPermissionOptionId( @@ -561,12 +914,175 @@ interface ActiveAcpTurn { readonly subagents: Map; readonly subagentsBySessionId: Map; readonly pendingSubagentNotifications: Map>; + /** Background monitor/task id → toolCallId for synthetic root text updates. */ + readonly toolCallIdsByBackgroundTaskId: Map; + /** + * Persistent monitors registered this turn. Excluded from deferred-finalize + * holds so the root turn can settle while they keep streaming post-settle. + */ + readonly persistentBackgroundTaskIds: Set; + /** + * Monitor end events often only say "use get_command…"; keep the turn open + * until TaskOutput hydration arrives (or the safety timeout elapses). + */ + readonly awaitingBackgroundHydration: Set; + /** + * A monitor end notice landed after the prompt settled: the CLI runs an + * injected turn whose report never gets a turn_completed marker, so the + * report chunk races the deferred-finalize debounce (thread a8e8b0a9 run 5 + * dropped the listing this way). Hold finalize until the report streams or + * the safety timeout elapses. + */ + readonly pendingInjectedReport: Set; plan: { readonly id: OrchestrationV2PlanArtifact["id"]; readonly startedAt: DateTime.Utc; } | null; interrupted: boolean; finalized: boolean; + settleScheduleGeneration: number; + /** session/prompt already returned; finalize deferred for background work. */ + promptSettled: boolean; + promptSettledStatus: "completed" | "interrupted" | "failed" | "cancelled" | null; + /** + * Completed the moment `runtime.prompt` resolves on the wire, before the + * completion callback requests `runtimeCallbackPermit`. Failure does not + * complete this; settled-soft classification ORs it with `promptSettled`. + */ + readonly promptWireSettled: Deferred.Deferred; + backgroundFinalizeGeneration: number; +} + +type AcpRuntimeTeardownState = + | { readonly _tag: "Idle" } + | { + readonly _tag: "InProgress"; + readonly completed: Deferred.Deferred; + } + | { readonly _tag: "Failed"; readonly error: ProviderAdapterProtocolError }; + +export function acpRootTurnHasIngestedOutput(context: { + readonly assistant: ActiveTextStream; + readonly reasoning: ActiveTextStream; + readonly tools: ReadonlyMap; + readonly plan: unknown; +}): boolean { + return ( + context.assistant.nextSegment > 0 || + context.reasoning.nextSegment > 0 || + context.tools.size > 0 || + context.plan !== null + ); +} + +/** True when a root session/update carries ingestible turn output, not keepalive noise. */ +export function acpRootSessionUpdateIngestsOutput( + notification: EffectAcpSchema.SessionNotification, +): boolean { + const update = notification.update; + switch (update.sessionUpdate) { + case "agent_message_chunk": + case "agent_thought_chunk": + return update.content.type === "text" && update.content.text.length > 0; + case "tool_call": + case "tool_call_update": + case "plan": + return parseSessionUpdateEvent(notification).events.some( + (event) => event._tag === "ToolCallUpdated" || event._tag === "PlanUpdated", + ); + default: + return false; + } +} + +/** + * Post-settle traffic that should be *buffered* for a continuation attach. + * Excludes monitor end/event chatter that must not become ghost history. + * Broader than {@link acpPostSettleContinuationOfferEvidence}: incremental + * tool progress may still need replay once a real completion offers a run. + */ +export function acpPostSettleWakeEvidence( + notification: EffectAcpSchema.SessionNotification, + flavor: Pick = {}, +): boolean { + if (!acpRootSessionUpdateIngestsOutput(notification)) return false; + const update = notification.update; + if ( + (update.sessionUpdate === "user_message_chunk" || + update.sessionUpdate === "agent_message_chunk") && + update.content.type === "text" + ) { + const text = update.content.text; + if ((flavor.extractBackgroundToolMutation?.(text) ?? []).length > 0) return false; + if (/ = {}, +): boolean { + if (!acpPostSettleWakeEvidence(notification, flavor)) { + return false; + } + const update = notification.update; + // Assistant text only. Thought/reasoning bursts alone must not open synthetic + // "Background task completed." runs (duplicate-run spam after monitors). + if (update.sessionUpdate === "agent_message_chunk") { + return update.content.type === "text" && update.content.text.length > 0; + } + if (update.sessionUpdate === "tool_call" || update.sessionUpdate === "tool_call_update") { + return parseSessionUpdateEvent(notification).events.some((event) => { + if (event._tag !== "ToolCallUpdated") return false; + // Normalize first: a Grok monitor start ACK arrives with raw status + // "completed" but is a still-running background task, not completion. + const toolCall = flavor.normalizeToolCall?.(event.toolCall) ?? event.toolCall; + return toolCall.status === "completed" || toolCall.status === "failed"; + }); + } + return false; +} + +export function acpPostSettleWakeShouldBuffer( + notification: EffectAcpSchema.SessionNotification, + backgroundWorkRunning: boolean, +): boolean { + if (!backgroundWorkRunning) return true; + const update = notification.update; + return ( + update.sessionUpdate !== "agent_message_chunk" && update.sessionUpdate !== "agent_thought_chunk" + ); +} + +export function acpPostSettleMonitorPromptShouldSuppress( + mutation: + | { + readonly taskId: string; + readonly status: "running" | "completed" | "failed"; + } + | undefined, +): boolean { + return mutation?.status === "running"; +} + +export function acpCompletedTurnShouldTerminalizeTool( + tool: AcpToolCallState, + flavor: Pick, +): boolean { + const status = toolStatus(tool.status); + if (status !== "pending" && status !== "running") return false; + if (flavor.extractBackgroundTaskId?.(tool) !== undefined) return false; + return flavor.extractSubagentUpdate?.(tool) === undefined; } interface ActiveAcpSubagent { @@ -575,12 +1091,29 @@ interface ActiveAcpSubagent { readonly childRootNodeId: OrchestrationV2ExecutionNode["id"]; readonly turnItemId: OrchestrationV2TurnItem["id"]; readonly turnItemOrdinal: number; + /** Turn that spawned the subagent; carryover updates keep this lineage. */ + readonly providerTurnId: OrchestrationV2ProviderTurn["id"]; + readonly parentProviderThreadId: ProviderThreadId; childSessionId: string | null; assistantText: string; nextChildOrdinal: number; } +function acpTurnHasPendingRuntimeRequest( + providerTurnId: OrchestrationV2ProviderTurn["id"], + pending: ReadonlyMap, +): boolean { + return [...pending.values()].some( + (request) => + request.runtimeRequest.providerTurnId === providerTurnId && + request.runtimeRequest.status === "pending", + ); +} + type PendingRuntimeRequest = { + readonly generation: number; + readonly nativeResponseAcknowledgement: Deferred.Deferred; + readonly transportRequestId: string; readonly requestId: RuntimeRequestId; readonly runtimeRequest: OrchestrationV2RuntimeRequest; readonly node: OrchestrationV2ExecutionNode; @@ -606,6 +1139,9 @@ interface SnapshotMessageState { export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV2Shape { const { flavor, fileSystem, idAllocator, serverConfig } = options; const driver = flavor.driver; + const continuationRequests = options.continuationRequests; + const postSettleContinuationEnabled = + flavor.enablePostSettleContinuation === true && continuationRequests !== undefined; return ProviderAdapterV2.of({ instanceId: options.instanceId, @@ -620,7 +1156,33 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV const activeSessionId = yield* Ref.make(null); const activeSessionSetup = yield* Ref.make(null); const activeSelection = yield* Ref.make(null); + const runtimeRestartRequired = yield* Ref.make(false); + const runtimeTeardownState = yield* Ref.make({ _tag: "Idle" }); + const runtimeCallbackGeneration = yield* Ref.make(0); + const runtimeCallbackPermit = yield* Semaphore.make(1); + const runtimeTransitionPermit = yield* Semaphore.make(1); + const nativeTransportRequests = yield* Ref.make< + Array<{ + readonly generation: number; + readonly method: string; + readonly payload: unknown; + readonly requestId: string; + readonly sequence: number; + }> + >([]); + const nextNativeTransportSequence = yield* Ref.make(0); + const nativeResponseAcknowledgements = yield* Ref.make( + new Map< + string, + { + readonly acknowledgement: Deferred.Deferred; + readonly generation: number; + } + >(), + ); const pendingRuntimeRequests = yield* Ref.make(new Map()); + const emitNativeResponseLifecycle = + options.testHooks?.onNativeResponseLifecycle ?? (() => Effect.void); const nextElicitationOrdinal = yield* Ref.make(0); const itemOrdinals = yield* Ref.make(new Map()); const nextItemOrdinalsByTurn = yield* Ref.make(new Map()); @@ -632,32 +1194,450 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV loadingIndex: 0, }); + const awaitRuntimeTeardown = Effect.fnUntraced(function* () { + const state = yield* Ref.get(runtimeTeardownState); + if (state._tag === "InProgress") { + yield* Deferred.await(state.completed); + } else if (state._tag === "Failed") { + return yield* state.error; + } + }); + const runRuntimeCallbackAtGeneration = ( + generation: number, + effect: Effect.Effect, + ) => + runtimeCallbackPermit.withPermit( + Effect.gen(function* () { + if ((yield* Ref.get(runtimeTeardownState))._tag !== "Idle") { + return Option.none(); + } + if ((yield* Ref.get(runtimeCallbackGeneration)) !== generation) { + return Option.none(); + } + return Option.some(yield* effect); + }), + ); + const claimNativeTransportRequest = ( + generation: number, + predicate: (request: { readonly method: string; readonly payload: unknown }) => boolean, + ) => + Ref.modify( + nativeTransportRequests, + ( + requests, + ): readonly [ + string | undefined, + Array<{ + readonly generation: number; + readonly method: string; + readonly payload: unknown; + readonly requestId: string; + readonly sequence: number; + }>, + ] => acpClaimNativeTransportRequest(requests, generation, predicate), + ); + const registerNativeResponseAcknowledgement = ( + generation: number, + transportRequestId: string, + acknowledgement: Deferred.Deferred, + ) => + Effect.gen(function* () { + yield* Ref.update(nativeResponseAcknowledgements, (current) => { + const updated = new Map(current); + updated.set(transportRequestId, { acknowledgement, generation }); + return updated; + }); + yield* emitNativeResponseLifecycle({ + type: "registered", + generation, + requestId: transportRequestId, + }); + if (!(yield* Deferred.isDone(acknowledgement))) return; + yield* Ref.update(nativeResponseAcknowledgements, (current) => { + if (current.get(transportRequestId)?.acknowledgement !== acknowledgement) { + return current; + } + const updated = new Map(current); + updated.delete(transportRequestId); + return updated; + }); + }); + const acknowledgeNativeResponse = (generation: number, transportRequestId: string) => + ( + options.testHooks?.beforeNativeResponseAdmissionCheck?.( + generation, + transportRequestId, + ) ?? Effect.void + ).pipe( + Effect.andThen(runRuntimeCallbackAtGeneration(generation, Effect.void)), + Effect.flatMap((registered) => + Option.isSome(registered) + ? Effect.void + : emitNativeResponseLifecycle({ + type: "admission_rejected", + generation, + requestId: transportRequestId, + }).pipe( + Effect.andThen( + new EffectAcpErrors.AcpTransportError({ + detail: "The ACP runtime closed before its response could be admitted", + cause: "Native response registration rejected during teardown", + }), + ), + ), + ), + ); + const awaitNativeResponseAcknowledgements = Effect.fnUntraced(function* ( + acknowledgements: ReadonlyArray< + readonly [string | undefined, Deferred.Deferred] + >, + ) { + if (acknowledgements.length === 0) return true; + const completed = yield* Deferred.make<"settled" | "timeout">(); + yield* emitNativeResponseLifecycle({ type: "watcher_started" }); + const acknowledgementFiber = yield* Effect.forEach( + acknowledgements, + ([, acknowledgement]) => Deferred.await(acknowledgement).pipe(Effect.exit), + { concurrency: "unbounded", discard: true }, + ).pipe( + Effect.andThen(Deferred.succeed(completed, "settled")), + Effect.interruptible, + Effect.ensuring(emitNativeResponseLifecycle({ type: "watcher_exited" })), + Effect.forkDetach, + ); + yield* emitNativeResponseLifecycle({ type: "timer_started" }); + const timerFiber = yield* Effect.sleep("2 seconds").pipe( + Effect.andThen(Deferred.succeed(completed, "timeout")), + Effect.interruptible, + Effect.ensuring(emitNativeResponseLifecycle({ type: "timer_exited" })), + Effect.forkDetach, + ); + const outcome = yield* Deferred.await(completed); + yield* Fiber.interrupt(acknowledgementFiber); + yield* Fiber.interrupt(timerFiber); + if (outcome === "settled") return true; + + yield* Ref.update(nativeResponseAcknowledgements, (current) => { + const updated = new Map(current); + for (const [requestId, acknowledgement] of acknowledgements) { + if ( + requestId !== undefined && + updated.get(requestId)?.acknowledgement === acknowledgement + ) { + updated.delete(requestId); + } + } + return updated; + }); + yield* Effect.forEach( + acknowledgements, + ([requestId]) => + emitNativeResponseLifecycle({ + type: "removed", + ...(requestId === undefined ? {} : { requestId }), + }), + { concurrency: "unbounded", discard: true }, + ); + const timeoutError = new EffectAcpErrors.AcpTransportError({ + detail: "Timed out waiting for an admitted ACP response to reach the transport queue", + cause: "Native response acknowledgement timed out", + }); + yield* Effect.forEach( + acknowledgements, + ([requestId, acknowledgement]) => + Deferred.fail(acknowledgement, timeoutError).pipe( + Effect.andThen( + emitNativeResponseLifecycle({ + type: "failed", + ...(requestId === undefined ? {} : { requestId }), + }), + ), + ), + { concurrency: "unbounded", discard: true }, + ); + return false; + }); + const awaitAdmittedNativeResponses = Effect.gen(function* () { + yield* awaitNativeResponseAcknowledgements( + [...(yield* Ref.get(nativeResponseAcknowledgements)).entries()].map( + ([requestId, entry]) => [requestId, entry.acknowledgement] as const, + ), + ); + }); + const quarantineNativeTransportAtGeneration = Effect.fnUntraced(function* ( + generation: number, + ) { + yield* Ref.update(nativeTransportRequests, (requests) => + requests.filter((request) => request.generation !== generation), + ); + const quarantined = yield* Ref.modify(nativeResponseAcknowledgements, (current) => { + const updated = new Map(current); + const acknowledgements: Array> = []; + for (const [requestId, entry] of updated) { + if (entry.generation !== generation) continue; + updated.delete(requestId); + acknowledgements.push(entry.acknowledgement); + } + return [acknowledgements, updated] as const; + }); + const error = new EffectAcpErrors.AcpTransportError({ + detail: "The ACP runtime was replaced before its response reached the transport queue", + cause: "ACP runtime transport was quarantined during teardown", + }); + yield* Effect.forEach( + quarantined, + (acknowledgement) => Deferred.fail(acknowledgement, error), + { concurrency: "unbounded", discard: true }, + ); + }); + const closeNativeTransport = runtimeCallbackPermit.withPermit( + Effect.gen(function* () { + yield* Ref.update(runtimeCallbackGeneration, (generation) => generation + 1); + yield* Ref.set(nativeTransportRequests, []); + const acknowledgements = yield* Ref.getAndSet( + nativeResponseAcknowledgements, + new Map(), + ); + const error = new EffectAcpErrors.AcpTransportError({ + detail: "The ACP session closed before its admitted response reached the transport", + cause: "ACP session transport closed", + }); + yield* Effect.forEach( + acknowledgements, + ([requestId, entry]) => + Deferred.fail(entry.acknowledgement, error).pipe( + Effect.andThen( + emitNativeResponseLifecycle({ + type: "removed", + generation: entry.generation, + requestId, + }), + ), + Effect.andThen( + emitNativeResponseLifecycle({ + type: "failed", + generation: entry.generation, + requestId, + }), + ), + ), + { concurrency: "unbounded", discard: true }, + ); + return acknowledgements.size > 0; + }), + ); + // Post-settle wake support (Grok async subagent/monitor follow-up). After + // the root turn finalizes, later root session/update traffic buffers here + // until a provider continuation run attaches and drains it. + const lastTurnRoute = yield* Ref.make<{ + readonly threadId: ThreadId; + readonly providerThreadId: ProviderThreadId; + } | null>(null); + const wakeBuffer = yield* Ref.make>([]); + const continuationRequested = yield* Ref.make(false); + const continuationGeneration = yield* Ref.make(0); + const continuationPermit = yield* Semaphore.make(1); + const continuationClosed = yield* Ref.make(false); + // Direct Stop (requestRuntimeRestart) quarantines residual events from the + // stopped run so they cannot wake or attach to a later prompt/run. + const stoppedRunQuarantine = yield* Ref.make(false); + // A steering restart (or any interrupt) can finalize a turn while its + // spawned subagents are still running natively. Carry the live + // lineages into the next turn on the same session so their terminal + // signals can still flip the original turn items instead of leaving + // them running forever. + const carryoverSubagents = yield* Ref.make<{ + readonly sessionId: string; + readonly subagents: ReadonlyArray; + } | null>(null); + const handledBackgroundTaskIdsInActiveTurn = yield* Ref.make>( + new Set(), + ); + // A monitor-event can arrive after its task and the user-facing provider + // turn already completed. Grok starts another internal prompt for that + // stale notification; suppress its agent output until a genuine terminal + // mutation or the next app turn so it cannot create a redundant app + // continuation. Tool frames continue through normal hydration. + const suppressPostSettleMonitorPrompt = yield* Ref.make(false); + // Background tasks (Grok monitors) known to still run at session level. + // Turn contexts are too short-lived to carry this: a continuation run + // finalizes between monitor events, and the next commentary burst must + // not reopen a run while the monitor is still streaming. + const runningBackgroundTaskIds = yield* Ref.make>(new Set()); + // Task ids with a GENUINE end signal (monitor-ended reminder or + // TaskOutput completion). Normalized tool statuses are not genuine: + // Grok Bash re-reports carry exit_code 0 mid-stream. A straggler + // monitor-event can land after the real end (the CLI keeps streaming + // while the agent already consumed the output via + // get_command_or_subagent_output); without the tombstone it would + // resurrect the running set and pin offers/idle-release forever. + // A tool-level failed get_command tombstones too: failing open to a + // single continuation beats failing closed to a dead thread. + const endedBackgroundTaskIds = yield* Ref.make>(new Set()); + const endedBackgroundTaskIdLimit = 128; + + const setBackgroundTaskRunning = (taskId: string, running: boolean) => + Effect.gen(function* () { + if (running && (yield* Ref.get(endedBackgroundTaskIds)).has(taskId)) { + return; + } + yield* Ref.update(runningBackgroundTaskIds, (current) => { + if (current.has(taskId) === running) return current; + const next = new Set(current); + if (running) { + next.add(taskId); + } else { + next.delete(taskId); + } + return next; + }); + }); + + const markBackgroundTaskEnded = (taskId: string) => + Ref.update(endedBackgroundTaskIds, (current) => { + if (current.has(taskId)) return current; + const next = new Set(current).add(taskId); + for (const oldest of next) { + if (next.size <= endedBackgroundTaskIdLimit) break; + next.delete(oldest); + } + return next; + }).pipe(Effect.andThen(setBackgroundTaskRunning(taskId, false))); + + const applyBackgroundTaskMutationRunning = (mutation: { + readonly taskId: string; + readonly status: "running" | "completed" | "failed"; + }) => + mutation.status === "running" + ? setBackgroundTaskRunning(mutation.taskId, true) + : markBackgroundTaskEnded(mutation.taskId); + + const trackRunningBackgroundTools = ( + notification: EffectAcpSchema.SessionNotification, + ): Effect.Effect => + Effect.gen(function* () { + if (flavor.extractBackgroundTaskId === undefined) return; + for (const event of parseSessionUpdateEvent(notification).events) { + if (event._tag !== "ToolCallUpdated") continue; + const toolCall = flavor.normalizeToolCall?.(event.toolCall) ?? event.toolCall; + const taskId = flavor.extractBackgroundTaskId(toolCall); + if (taskId === undefined) continue; + const status = toolStatus(toolCall.status); + yield* setBackgroundTaskRunning(taskId, status === "pending" || status === "running"); + } + }); + const emitProviderEvent = (event: ProviderAdapterV2Event) => Queue.offer(events, event).pipe(Effect.asVoid); + let scheduleSettleRootTurnWhenIdle = (_context: ActiveAcpTurn) => Effect.void; + let rearmRootTurnRecoveryTimers = (_context: ActiveAcpTurn) => Effect.void; + let scheduleDeferredFinalize: (context: ActiveAcpTurn) => Effect.Effect = () => + Effect.void; const nativeLogging = options.nativeLogging?.(input.threadId); - - const runtime = yield* flavor - .makeRuntime({ - cwd: input.runtimePolicy.cwd ?? process.cwd(), - mcpServers: acpMcpServers(input.threadId), - interruptPromptOnCancel: false, - clientCapabilities: { - fs: { readTextFile: false, writeTextFile: false }, - terminal: false, - elicitation: { form: {} }, - }, - clientInfo: { name: "t3-code", version: "0.0.0" }, - ...(nativeLogging?.requestLogger === undefined - ? {} - : { requestLogger: nativeLogging.requestLogger }), - protocolLogging: nativeLogging?.protocolLogging ?? { - logIncoming: true, - logOutgoing: true, - logger: () => Effect.void, - }, - }) - .pipe(Effect.provideService(Crypto.Crypto, options.crypto)); + const makeRuntimeInput = (runtimeGeneration: number): AcpAdapterV2RuntimeInput => ({ + cwd: input.runtimePolicy.cwd ?? process.cwd(), + mcpServers: acpMcpServers(input.threadId), + interruptPromptOnCancel: flavor.interruptPromptOnCancel ?? false, + clientCapabilities: { + fs: { readTextFile: false, writeTextFile: false }, + terminal: false, + elicitation: { form: {} }, + }, + clientInfo: { name: "t3-code", version: "0.0.0" }, + onIncomingRequest: (requestId, method, payload) => + runRuntimeCallbackAtGeneration( + runtimeGeneration, + Effect.gen(function* () { + const sequence = yield* Ref.getAndUpdate( + nextNativeTransportSequence, + (current) => current + 1, + ); + yield* Ref.update(nativeTransportRequests, (current) => [ + ...current, + { generation: runtimeGeneration, method, payload, requestId, sequence }, + ]); + }), + ).pipe(Effect.asVoid), + onTermination: () => + runRuntimeCallbackAtGeneration( + runtimeGeneration, + Ref.set(runtimeRestartRequired, true), + ).pipe(Effect.asVoid), + onOutgoingResponseFailure: (requestId, error) => + Ref.modify(nativeResponseAcknowledgements, (current) => { + const entry = current.get(requestId); + if (entry === undefined || entry.generation !== runtimeGeneration) { + return [ + emitNativeResponseLifecycle({ + type: "late_noop", + generation: runtimeGeneration, + requestId, + }), + current, + ] as const; + } + const updated = new Map(current); + updated.delete(requestId); + return [ + Deferred.fail(entry.acknowledgement, error).pipe( + Effect.andThen( + emitNativeResponseLifecycle({ + type: "removed", + generation: runtimeGeneration, + requestId, + }), + ), + Effect.asVoid, + ), + updated, + ] as const; + }).pipe(Effect.flatten), + onOutgoingResponse: (requestId) => + Ref.modify(nativeResponseAcknowledgements, (current) => { + const entry = current.get(requestId); + if (entry === undefined || entry.generation !== runtimeGeneration) { + return [ + emitNativeResponseLifecycle({ + type: "late_noop", + generation: runtimeGeneration, + requestId, + }), + current, + ] as const; + } + const updated = new Map(current); + updated.delete(requestId); + return [ + Deferred.succeed(entry.acknowledgement, undefined).pipe( + Effect.andThen( + emitNativeResponseLifecycle({ + type: "removed", + generation: runtimeGeneration, + requestId, + }), + ), + Effect.asVoid, + ), + updated, + ] as const; + }).pipe(Effect.flatten), + ...(nativeLogging?.requestLogger === undefined + ? {} + : { requestLogger: nativeLogging.requestLogger }), + protocolLogging: nativeLogging?.protocolLogging ?? { + logIncoming: true, + logOutgoing: true, + logger: () => Effect.void, + }, + }); + let runtimeScope: Scope.Closeable | undefined; + let runtime!: AcpSessionRuntime.AcpSessionRuntime["Service"]; + yield* Effect.addFinalizer(() => + runtimeScope === undefined + ? Effect.void + : Scope.close(runtimeScope, Exit.void).pipe(Effect.ignore), + ); const resolveItemOrdinal = Effect.fnUntraced(function* ( context: ActiveAcpTurn, @@ -817,6 +1797,9 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV if (stream.current === null) return; yield* emitTextSegment(context, kind, true); stream.current = null; + if (kind === "assistant") { + yield* scheduleSettleRootTurnWhenIdle(context); + } }); const closeTextStreams = Effect.fnUntraced(function* (context: ActiveAcpTurn) { @@ -889,39 +1872,56 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV context: ActiveAcpTurn, update: AcpAdapterV2SubagentUpdate, ) { - const existing = context.subagents.get(update.nativeTaskId); + // Hydration tools (get_command_or_subagent_output) use a new toolCallId + // but reference the child via subagent_id / task id. + const existing = + context.subagents.get(update.nativeTaskId) ?? + (update.childSessionId !== null + ? context.subagentsBySessionId.get(update.childSessionId) + : undefined); + // get_command_or_subagent_output may target monitors/bash tasks. Only + // hydrate when we already have a matching subagent lineage. Spawn ACKs + // (non-empty prompt) may create a new lineage; empty-prompt hydration + // with suppressNormalTool must not invent a phantom subagent. + if (existing === undefined) { + const isHydrationOnly = update.prompt === "" && update.title === null; + if (isHydrationOnly || update.suppressNormalTool === false) { + return; + } + } const now = yield* DateTime.now; + const nativeTaskId = existing?.task.nativeTaskRef?.nativeId ?? update.nativeTaskId; const nativeItemRef = { driver, - nativeId: update.nativeTaskId, + nativeId: nativeTaskId, strength: "strong" as const, }; const nodeId = existing?.task.id ?? idAllocator.derive.nodeFromProviderItem({ driver, - nativeItemId: update.nativeTaskId, + nativeItemId: nativeTaskId, }); const childThreadId = existing?.childThreadId ?? idAllocator.derive.threadFromProviderThread({ driver, - nativeThreadId: `${nativeThreadId(driver, context.input.providerThread)}:task:${update.nativeTaskId}`, + nativeThreadId: `${nativeThreadId(driver, context.input.providerThread)}:task:${nativeTaskId}`, }); const childRootNodeId = existing?.childRootNodeId ?? idAllocator.derive.nodeFromProviderItem({ driver, - nativeItemId: `${update.nativeTaskId}:child-root`, + nativeItemId: `${nativeTaskId}:child-root`, }); const turnItemId = existing?.turnItemId ?? idAllocator.derive.turnItemFromProviderItem({ driver, - nativeItemId: update.nativeTaskId, + nativeItemId: nativeTaskId, }); const turnItemOrdinal = - existing?.turnItemOrdinal ?? (yield* resolveItemOrdinal(context, update.nativeTaskId)); + existing?.turnItemOrdinal ?? (yield* resolveItemOrdinal(context, nativeTaskId)); const taskStatus = update.status; const task: OrchestrationV2Subagent = { ...(existing?.task ?? { @@ -953,12 +1953,14 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV childRootNodeId, turnItemId, turnItemOrdinal, + providerTurnId: context.providerTurnId, + parentProviderThreadId: context.input.providerThread.id, childSessionId: null, assistantText: "", nextChildOrdinal: 101, }; subagent.task = task; - context.subagents.set(update.nativeTaskId, subagent); + context.subagents.set(nativeTaskId, subagent); if (existing === undefined) { yield* emitProviderEvent({ @@ -985,7 +1987,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV creationSource: "provider", }), }); - const promptNativeItemId = `${update.nativeTaskId}:prompt`; + const promptNativeItemId = `${nativeTaskId}:prompt`; const promptArtifacts = makeSubagentConversationArtifacts({ messageId: idAllocator.derive.messageFromProviderItem({ driver, @@ -1070,14 +2072,14 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV node: { id: nodeId, threadId: context.input.threadId, - runId: context.input.runId, - parentNodeId: context.input.rootNodeId, - rootNodeId: context.input.rootNodeId, + runId: subagent.task.runId, + parentNodeId: subagent.task.parentNodeId, + rootNodeId: subagent.task.parentNodeId, kind: "subagent", status: taskStatus, countsForRun: false, providerThreadId: context.input.providerThread.id, - providerTurnId: context.providerTurnId, + providerTurnId: subagent.providerTurnId, nativeItemRef, runtimeRequestId: null, checkpointScopeId: null, @@ -1113,10 +2115,10 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV turnItem: { id: turnItemId, threadId: context.input.threadId, - runId: context.input.runId, + runId: subagent.task.runId, nodeId, providerThreadId: context.input.providerThread.id, - providerTurnId: context.providerTurnId, + providerTurnId: subagent.providerTurnId, nativeItemRef, parentItemId: null, ordinal: turnItemOrdinal, @@ -1137,22 +2139,230 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }); }); - const emitTool = Effect.fnUntraced(function* ( + const toolOutputText = (toolCall: AcpToolCallState): string => { + if (typeof toolCall.data.rawOutput === "string") { + return toolCall.data.rawOutput; + } + if ( + typeof (toolCall.data.rawOutput as { text?: unknown } | undefined)?.text === "string" + ) { + return String((toolCall.data.rawOutput as { text: string }).text); + } + return textFromUnknown(toolCall.data.rawOutput) ?? ""; + }; + + const setToolOutputText = (toolCall: AcpToolCallState, text: string): AcpToolCallState => ({ + ...toolCall, + data: { + ...toolCall.data, + rawOutput: { + type: "Text", + text, + }, + }, + }); + + const appendToolOutputText = ( + toolCall: AcpToolCallState, + appendOutput: string, + ): AcpToolCallState => { + if (appendOutput.length === 0) return toolCall; + return setToolOutputText(toolCall, `${toolOutputText(toolCall)}${appendOutput}`); + }; + + const isMonitorEndNoticeText = (text: string): boolean => + /Monitor\s+["']?[0-9a-f-]{8,}/i.test(text) && /ended/i.test(text); + + const hasDeferredBackgroundWork = (context: ActiveAcpTurn): boolean => { + if (context.awaitingBackgroundHydration.size > 0) return true; + if (context.pendingInjectedReport.size > 0) return true; + for (const [taskId, toolCallId] of context.toolCallIdsByBackgroundTaskId) { + // Persistent monitors stream after root settle; do not pin finalize. + if (context.persistentBackgroundTaskIds.has(taskId)) continue; + const tool = context.tools.get(toolCallId); + if (tool === undefined) continue; + const status = toolStatus(tool.status); + if (status === "pending" || status === "running") return true; + } + for (const subagent of context.subagents.values()) { + if (subagent.task.status === "running" || subagent.task.status === "pending") { + return true; + } + } + return false; + }; + + // scheduleDeferredFinalize is declared above openSession body start and + // assigned after finalizeTurn so forked finalize Effect typing stays clean. + const rearmDeferredFinalize = (context: ActiveAcpTurn) => + Effect.gen(function* () { + if (!flavor.deferFinalizeForBackgroundWork) return; + if (!context.promptSettled || context.finalized) return; + if (hasDeferredBackgroundWork(context)) return; + yield* scheduleDeferredFinalize(context); + }); + + // `let` breaks circular inference from monitor hydration re-entry. + let emitTool: ( + context: ActiveAcpTurn, + incoming: AcpToolCallState, + projectedStatus?: ProjectedToolStatus, + ) => Effect.Effect = () => Effect.void; + + const markAwaitingBackgroundHydration = (context: ActiveAcpTurn, taskId: string) => + Effect.gen(function* () { + if (context.awaitingBackgroundHydration.has(taskId)) return; + context.awaitingBackgroundHydration.add(taskId); + // Safety: do not hold the root turn forever if the agent never hydrates. + yield* Effect.gen(function* () { + yield* Effect.sleep("60000 millis"); + if (context.finalized || !context.awaitingBackgroundHydration.has(taskId)) return; + context.awaitingBackgroundHydration.delete(taskId); + // End notices keep the tool running until hydration; force-complete so + // deferred finalize can proceed when get_command never arrives. + const toolCallId = context.toolCallIdsByBackgroundTaskId.get(taskId); + const tool = toolCallId !== undefined ? context.tools.get(toolCallId) : undefined; + if (tool !== undefined) { + const status = toolStatus(tool.status); + if (status === "pending" || status === "running") { + yield* emitTool(context, { ...tool, status: "completed" }); + } + } + yield* rearmDeferredFinalize(context); + }).pipe(Effect.forkIn(sessionScope), Effect.asVoid); + }); + + const markPendingInjectedReport = (context: ActiveAcpTurn, taskId: string) => + Effect.gen(function* () { + // Only the settled-and-held window has the race; mid-turn reports + // are protected by the prompt RPC still being open. + if (!context.promptSettled) return; + if (context.pendingInjectedReport.has(taskId)) return; + context.pendingInjectedReport.add(taskId); + // Safety: the injected turn may end without a report chunk. + yield* Effect.gen(function* () { + yield* Effect.sleep("25000 millis"); + if (context.finalized || !context.pendingInjectedReport.has(taskId)) return; + context.pendingInjectedReport.delete(taskId); + yield* rearmDeferredFinalize(context); + }).pipe(Effect.forkIn(sessionScope), Effect.asVoid); + }); + + emitTool = Effect.fnUntraced(function* ( context: ActiveAcpTurn, incoming: AcpToolCallState, + projectedStatus?: ProjectedToolStatus, ) { yield* closeTextStreams(context); const previous = context.tools.get(incoming.toolCallId); - const toolCall = mergeToolCallState(previous, incoming); + const merged = mergeToolCallState(previous, incoming); + const toolCall = flavor.normalizeToolCall?.(merged) ?? merged; context.tools.set(toolCall.toolCallId, toolCall); - const subagentUpdate = flavor.extractSubagentUpdate?.(toolCall); - if (subagentUpdate !== undefined) { - yield* emitSubagent(context, subagentUpdate); - return; - } - const status = toolStatus(toolCall.status); - const now = yield* DateTime.now; - const nativeItemId = `${nativeThreadId(driver, context.input.providerThread)}:tool:${toolCall.toolCallId}`; + const backgroundTaskId = flavor.extractBackgroundTaskId?.(toolCall); + if (backgroundTaskId !== undefined) { + context.toolCallIdsByBackgroundTaskId.set(backgroundTaskId, toolCall.toolCallId); + if (flavor.isPersistentBackgroundTool?.(toolCall) === true) { + context.persistentBackgroundTaskIds.add(backgroundTaskId); + } + const backgroundStatus = projectedStatus ?? toolStatus(toolCall.status); + yield* setBackgroundTaskRunning( + backgroundTaskId, + backgroundStatus === "pending" || backgroundStatus === "running", + ); + // Background tool that reaches a terminal status while the root + // prompt is still open (completed, failed, or interrupted) was + // consumed in-turn. Mark handled so late CLI re-reports and residual + // agent chatter do not open synthetic "Background task completed." + // runs. get_command TaskOutput still marks handled below. + // Only before promptSettled: after STARTED the deferred-finalize + // hold can let a monitor finish while the turn is still active; + // marking then would suppress the legitimate post-settle TaskOutput + // continuation (live: grok-post-settle-continuation-poll). + // Terminal statuses only after normalizeToolCall (start ACKs stay + // inProgress/running). + if ( + !context.promptSettled && + backgroundStatus !== "pending" && + backgroundStatus !== "running" + ) { + yield* Ref.update(handledBackgroundTaskIdsInActiveTurn, (current) => + new Set(current).add(backgroundTaskId), + ); + } + } + + // get_command TaskOutput for registered monitor(s): hydrate those tools. + const backgroundCompletions = + projectedStatus === undefined + ? (flavor.extractBackgroundTaskCompletion?.(toolCall) ?? []) + : []; + let hydratedRegisteredMonitor = false; + for (const backgroundCompletion of backgroundCompletions) { + // Genuine end signal when terminal: tombstone so straggler + // monitor-event chatter cannot resurrect the running set after the + // task truly ended. A still-running poll keeps the id running. + yield* applyBackgroundTaskMutationRunning(backgroundCompletion); + if (backgroundCompletion.status !== "running") { + yield* Ref.update(handledBackgroundTaskIdsInActiveTurn, (current) => + new Set(current).add(backgroundCompletion.taskId), + ); + } + // A still-running fetch must keep the hydration hold (and its + // safety timer) alive until output actually lands. + if (backgroundCompletion.status !== "running") { + context.awaitingBackgroundHydration.delete(backgroundCompletion.taskId); + } + const targetToolCallId = context.toolCallIdsByBackgroundTaskId.get( + backgroundCompletion.taskId, + ); + // Known background-task id (monitor registration) — never open a + // phantom subagent for the same get_output poll. + if (targetToolCallId !== undefined) { + hydratedRegisteredMonitor = true; + } + const target = + targetToolCallId !== undefined ? context.tools.get(targetToolCallId) : undefined; + if (target !== undefined && target.toolCallId !== toolCall.toolCallId) { + const nextStatus = + backgroundCompletion.status === "running" + ? ("inProgress" as const) + : backgroundCompletion.status === "failed" + ? ("failed" as const) + : ("completed" as const); + const hydrated = + backgroundCompletion.appendOutput.length > 0 + ? // TaskOutput is the real stdout; replace end-notice boilerplate + // so the timeline shows the listing, not only "Monitor ended…". + isMonitorEndNoticeText(toolOutputText(target)) || + toolOutputText(target).trim().length === 0 + ? setToolOutputText( + { ...target, status: nextStatus }, + backgroundCompletion.appendOutput, + ) + : appendToolOutputText( + { ...target, status: nextStatus }, + backgroundCompletion.appendOutput, + ) + : { ...target, status: nextStatus }; + yield* emitTool(context, hydrated); + } + } + + // Monitor TaskOutput shares the get_command tool shape with subagent + // hydration; do not spawn a phantom subagent for a registered monitor. + const subagentUpdate = hydratedRegisteredMonitor + ? undefined + : flavor.extractSubagentUpdate?.(toolCall); + if (subagentUpdate !== undefined) { + yield* emitSubagent(context, subagentUpdate); + if (subagentUpdate.suppressNormalTool !== false) { + yield* rearmDeferredFinalize(context); + return; + } + } + const status = projectedStatus ?? toolStatus(toolCall.status); + const now = yield* DateTime.now; + const nativeItemId = `${nativeThreadId(driver, context.input.providerThread)}:tool:${toolCall.toolCallId}`; const ordinal = yield* resolveItemOrdinal(context, nativeItemId); const nodeId = idAllocator.derive.nodeFromProviderItem({ driver, nativeItemId }); const turnItemId = idAllocator.derive.turnItemFromProviderItem({ @@ -1209,6 +2419,30 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV const rawInput = toolCall.data.rawInput; const rawOutput = toolCall.data.rawOutput ?? toolCall.data.content; const path = pathFromToolCall(toolCall); + const rawInputRecord = unknownRecord(rawInput); + const inputVariant = + typeof rawInputRecord?.variant === "string" + ? rawInputRecord.variant.trim().toLowerCase() + : ""; + const rawOutputRecord = unknownRecord(rawOutput); + const outputCommand = + typeof rawOutputRecord?.command === "string" && + rawOutputRecord.command.trim().length > 0 + ? rawOutputRecord.command.trim() + : undefined; + const monitorCommand = + (typeof rawInputRecord?.command === "string" && rawInputRecord.command.trim().length > 0 + ? rawInputRecord.command.trim() + : undefined) ?? outputCommand; + const outputIsBashResult = + typeof rawOutputRecord?.type === "string" && + rawOutputRecord.type.trim().toLowerCase() === "bash" && + commandExitCode(rawOutput) !== undefined; + // Grok Monitor tools arrive as generic kind + variant; project like shell + // so stdout is plain text in the timeline (not JSON {type:Text,text:...}). + // Post-settle wake re-reports of a finished monitor carry no rawInput at + // all, only a structured Bash result; project those as commands too. + const projectAsCommandExecution = inputVariant === "monitor" || outputIsBashResult; let turnItem: OrchestrationV2TurnItem; switch (toolCall.kind) { case "read": @@ -1231,19 +2465,19 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }), }; break; - case "execute": + case "execute": { + const exitCode = acpProjectedCommandExitCode(status, rawOutput); turnItem = { ...base, type: "command_execution", - input: toolCall.command ?? toolCall.title ?? "Command", + input: toolCall.command ?? monitorCommand ?? toolCall.title ?? "Command", ...(textFromUnknown(rawOutput) === undefined ? {} : { output: textFromUnknown(rawOutput) }), - ...(commandExitCode(rawOutput) === undefined - ? {} - : { exitCode: commandExitCode(rawOutput) }), + ...(exitCode === undefined ? {} : { exitCode }), }; break; + } case "edit": case "delete": case "move": @@ -1276,15 +2510,33 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }; break; default: - turnItem = { - ...base, - type: "dynamic_tool", - toolName: toolCall.title ?? toolCall.kind ?? null, - input: rawInput ?? {}, - ...(rawOutput === undefined ? {} : { output: rawOutput }), - }; + if (projectAsCommandExecution) { + const exitCode = acpProjectedCommandExitCode(status, rawOutput); + turnItem = { + ...base, + type: "command_execution", + input: + toolCall.command ?? + monitorCommand ?? + toolCall.title ?? + (inputVariant === "monitor" ? "Monitor" : "Command"), + ...(textFromUnknown(rawOutput) === undefined + ? {} + : { output: textFromUnknown(rawOutput) }), + ...(exitCode === undefined ? {} : { exitCode }), + }; + } else { + turnItem = { + ...base, + type: "dynamic_tool", + toolName: toolCall.title ?? toolCall.kind ?? null, + input: rawInput ?? {}, + ...(rawOutput === undefined ? {} : { output: rawOutput }), + }; + } } yield* emitProviderEvent({ type: "turn_item.updated", driver, turnItem }); + yield* rearmDeferredFinalize(context); }); const emitPlan = Effect.fnUntraced(function* ( @@ -1423,22 +2675,287 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }); }); + const offerContinuationRun = Effect.fnUntraced(function* (sessionId: string) { + if (continuationRequests === undefined) { + return; + } + const pending = yield* continuationPermit.withPermit( + Effect.gen(function* () { + if (yield* Ref.get(continuationClosed)) return Option.none(); + if (yield* Ref.get(stoppedRunQuarantine)) return Option.none(); + if (yield* Ref.get(continuationRequested)) return Option.none(); + const route = yield* Ref.get(lastTurnRoute); + if (route === null) return Option.none(); + yield* Ref.set(continuationRequested, true); + const generation = yield* Ref.updateAndGet( + continuationGeneration, + (value) => value + 1, + ); + return Option.some({ route, generation }); + }), + ); + if (Option.isNone(pending)) return; + const { route, generation } = pending.value; + yield* Effect.logInfo("orchestration-v2.acp-wake-turn-detected", { + driver, + providerSessionId: input.providerSessionId, + threadId: route.threadId, + providerThreadId: route.providerThreadId, + }); + yield* continuationRequests.offer({ + threadId: route.threadId, + providerThreadId: route.providerThreadId, + driver, + detail: null, + dispatchIfCurrent: (effect) => + continuationPermit.withPermit( + Effect.gen(function* () { + const clearIfOwner = Effect.gen(function* () { + if ((yield* Ref.get(continuationGeneration)) === generation) { + yield* Ref.set(continuationRequested, false); + } + }); + if (yield* Ref.get(stoppedRunQuarantine)) { + yield* clearIfOwner; + return Option.none(); + } + if ((yield* Ref.get(continuationGeneration)) !== generation) { + // Superseded by a newer offer; do not clear its flag. + return Option.none(); + } + if (!(yield* Ref.get(continuationRequested))) return Option.none(); + // Clear after success or failure so a dropped dispatch does not + // stick the flag and pin hasPendingBackgroundWork forever. + const exit = yield* Effect.exit(effect); + yield* clearIfOwner; + if (Exit.isFailure(exit)) { + return yield* Effect.failCause(exit.cause); + } + return Option.some(exit.value); + }), + ), + }); + }); + + const applyLateBackgroundMutation = Effect.fnUntraced(function* ( + sessionId: string, + mutation: { + readonly taskId: string; + readonly status: "running" | "completed" | "failed"; + }, + ) { + const taskAlreadyEnded = (yield* Ref.get(endedBackgroundTaskIds)).has(mutation.taskId); + yield* applyBackgroundTaskMutationRunning(mutation); + if (taskAlreadyEnded && acpPostSettleMonitorPromptShouldSuppress(mutation)) { + yield* Ref.set(suppressPostSettleMonitorPrompt, true); + } + if (mutation.status !== "running") { + yield* Ref.set(suppressPostSettleMonitorPrompt, false); + yield* Ref.update(handledBackgroundTaskIdsInActiveTurn, (current) => { + if (!current.has(mutation.taskId)) return current; + const next = new Set(current); + next.delete(mutation.taskId); + return next; + }); + if ( + postSettleContinuationEnabled && + (yield* Ref.get(activeSessionId)) === sessionId && + (yield* Ref.get(runningBackgroundTaskIds)).size === 0 && + (yield* Ref.get(wakeBuffer)).length > 0 + ) { + yield* offerContinuationRun(sessionId); + } + } + }); + + const bufferPostSettleWake = Effect.fnUntraced(function* ( + notification: EffectAcpSchema.SessionNotification, + ) { + if (!postSettleContinuationEnabled || continuationRequests === undefined) { + return false; + } + // Direct Stop quarantine: drop residual wake evidence instead of + // buffering it for a later continuation or follow-up run. + if (yield* Ref.get(stoppedRunQuarantine)) { + return true; + } + const rootSessionId = yield* Ref.get(activeSessionId); + if (rootSessionId === null || notification.sessionId !== rootSessionId) { + return false; + } + if (!acpPostSettleWakeEvidence(notification, flavor)) { + return false; + } + const update = notification.update; + let alreadyHandledToolUpdate = false; + if (update.sessionUpdate === "tool_call" || update.sessionUpdate === "tool_call_update") { + yield* trackRunningBackgroundTools(notification); + // When the agent consumes the monitor output itself via + // get_command_or_subagent_output (long timeout), the TaskOutput + // completion is the ONLY end signal: no "Monitor ended" reminder + // follows. Without applying it here the running set never clears, + // every offer stays suppressed, and the wake buffer never drains + // (observed live 2026-07-12, thread 8dbe607f). The completion + // frame itself is offer evidence, so clearing the id before the + // gate below lets it open the single continuation run. + for (const event of parseSessionUpdateEvent(notification).events) { + if (event._tag !== "ToolCallUpdated") continue; + const toolCall = flavor.normalizeToolCall?.(event.toolCall) ?? event.toolCall; + const toolTaskId = flavor.extractBackgroundTaskId?.(toolCall); + if ( + toolTaskId !== undefined && + (yield* Ref.get(handledBackgroundTaskIdsInActiveTurn)).has(toolTaskId) + ) { + alreadyHandledToolUpdate = true; + } + if (flavor.extractBackgroundTaskCompletion !== undefined) { + for (const completion of flavor.extractBackgroundTaskCompletion(toolCall)) { + alreadyHandledToolUpdate = + alreadyHandledToolUpdate || + (yield* Ref.get(handledBackgroundTaskIdsInActiveTurn)).has(completion.taskId); + yield* applyBackgroundTaskMutationRunning(completion); + } + } + } + } + const backgroundWorkRunning = (yield* Ref.get(runningBackgroundTaskIds)).size > 0; + // Grok prompts itself for every monitor event after the root turn + // settles. Its assistant/reasoning replies are progress chatter, not + // separate wake results. Retaining them would replay the entire burst + // into the single continuation once the monitor finishes. Keep tool + // state so the final command card still hydrates, then begin retaining + // agent output again after the genuine end signal clears the running + // set. With multiple tasks, text remains best-effort until every task + // ends; their retained tool cards are the authoritative results. + // + // Skip retaining frames for tasks already hydrated in the root turn + // (`handledBackgroundTaskIdsInActiveTurn`). Those re-reports must not + // pin `hasPendingBackgroundWork` via a wake buffer that never drains + // (the already-handled gate below intentionally skips + // `offerContinuationRun` to avoid synthetic "Background task + // completed." spam). + if ( + !alreadyHandledToolUpdate && + acpPostSettleWakeShouldBuffer(notification, backgroundWorkRunning) + ) { + yield* Ref.update(wakeBuffer, (current) => [...current, notification]); + } + // Buffer progress without offering; only completion-like frames open a run. + if (!acpPostSettleContinuationOfferEvidence(notification, flavor)) { + return true; + } + // While a monitor is still streaming, tool re-reports buffer without + // offering and per-event agent commentary is consumed without being + // retained. Grok re-reports a running monitor as Bash frames that + // already carry exit_code 0 mid-stream, so a "terminal" normalized + // status is not evidence the task ended; each burst would otherwise + // reopen a synthetic "Background task completed." run. Retained tool + // frames drain into the single continuation offered once the monitor + // actually ends (end-notice mutation below, or the first frame after + // it). + if (backgroundWorkRunning) { + return true; + } + if (alreadyHandledToolUpdate) { + // Drop leftover wake noise for in-turn-handled work so idle release + // is not pinned forever. Leave the buffer alone when a continuation + // is already outstanding: its startTurn will drain legitimate frames + // from other tasks. + if (!(yield* Ref.get(continuationRequested))) { + yield* Ref.set(wakeBuffer, []); + } + return true; + } + // Residual Grok agent/thought chatter after in-turn-handled background + // work must not open synthetic continuation runs. Tool-path + // alreadyHandled covers re-reports with a task id; this covers + // agent_message_chunk frames that carry no task id (live: + // grok-in-turn-monitor-no-wake). Running work already returned above. + // Do not clear wakeBuffer here: frames for other still-tracked tasks + // must remain drainable when a real (tool) completion later offers. + const handledInTurnCount = (yield* Ref.get(handledBackgroundTaskIdsInActiveTurn)).size; + if ( + handledInTurnCount > 0 && + (update.sessionUpdate === "agent_message_chunk" || + update.sessionUpdate === "agent_thought_chunk") + ) { + return true; + } + yield* offerContinuationRun(notification.sessionId); + return true; + }); + const handleSessionUpdate = Effect.fnUntraced(function* ( notification: EffectAcpSchema.SessionNotification, ) { const context = yield* Ref.get(activeTurn); const update = notification.update; + // Only while a finalized turn is still the active context. When + // activeTurn is null, post-settle agent frames must reach + // bufferPostSettleWake so continuation can attach (context?.finalized + // !== false incorrectly treated null as finalized and dropped them). + if ( + context !== null && + context.finalized && + (yield* Ref.get(handledBackgroundTaskIdsInActiveTurn)).size > 0 && + (update.sessionUpdate === "agent_message_chunk" || + update.sessionUpdate === "agent_thought_chunk") + ) { + yield* Ref.set(suppressPostSettleMonitorPrompt, true); + return; + } + if ( + context !== null && + (yield* Ref.get(suppressPostSettleMonitorPrompt)) && + (update.sessionUpdate === "agent_message_chunk" || + update.sessionUpdate === "agent_thought_chunk") + ) { + return; + } + if ( + context?.finalized === true && + update.sessionUpdate === "user_message_chunk" && + update.content.type === "text" + ) { + const mutations = flavor.extractBackgroundToolMutation?.(update.content.text) ?? []; + for (const mutation of mutations) { + yield* applyLateBackgroundMutation(notification.sessionId, mutation); + } + return; + } if (context === null) { + // Direct Stop: quarantine residual events from the stopped run so + // they cannot become history, wake buffers, or a later run attach. + if (yield* Ref.get(stoppedRunQuarantine)) { + return; + } + // Prefer continuation buffering over history append so the same + // frames are not double-counted once a continuation run attaches. + if (yield* bufferPostSettleWake(notification)) { + return; + } if ( (update.sessionUpdate === "user_message_chunk" || update.sessionUpdate === "agent_message_chunk") && update.content.type === "text" ) { - yield* appendLoadedHistory( - notification, - update.sessionUpdate === "user_message_chunk" ? "user" : "assistant", - update.content.text, - ); + // Late monitor end/event reminders must not become ghost user/assistant + // history (or OS-facing chatter) after the root turn already finalized. + const text = update.content.text; + const lateBackgroundMutations = flavor.extractBackgroundToolMutation?.(text) ?? []; + for (const lateBackgroundMutation of lateBackgroundMutations) { + yield* applyLateBackgroundMutation(notification.sessionId, lateBackgroundMutation); + } + const lateMonitorChatter = + / 0) { + context.pendingInjectedReport.clear(); + } yield* appendText(context, "assistant", update.content.text); } - return; + break; case "agent_thought_chunk": if (update.content.type === "text") { yield* appendText(context, "reasoning", update.content.text); } - return; + break; + case "user_message_chunk": + if (update.content.type === "text" && flavor.extractBackgroundToolMutation) { + for (const mutation of flavor.extractBackgroundToolMutation(update.content.text)) { + const toolCallId = context.toolCallIdsByBackgroundTaskId.get(mutation.taskId); + const previous = + toolCallId !== undefined ? context.tools.get(toolCallId) : undefined; + if (previous !== undefined) { + let nextStatus = + mutation.status === "running" + ? ("inProgress" as const) + : mutation.status === "failed" + ? ("failed" as const) + : ("completed" as const); + // End notices typically omit full stdout (no get_command mention + // either). Keep the tool running and hold finalize until + // TaskOutput hydrates, or the safety timer force-completes. + if (nextStatus !== "inProgress") { + yield* markAwaitingBackgroundHydration(context, mutation.taskId); + nextStatus = "inProgress"; + } + yield* emitTool( + context, + appendToolOutputText( + { ...previous, status: nextStatus }, + mutation.appendOutput, + ), + ); + } + // After emitTool: the hydration hold keeps the tool row at + // inProgress past an end notice, but the offer gate must see + // the mutation's semantic status. Also tracks monitors that + // never surfaced a tool_call row at all. + yield* applyBackgroundTaskMutationRunning(mutation); + if (mutation.status !== "running") { + yield* markPendingInjectedReport(context, mutation.taskId); + } + } + } + if (update.content.type === "text" && flavor.extractSubagentEndNotice) { + const notice = flavor.extractSubagentEndNotice(update.content.text); + const subagent = + notice !== undefined + ? context.subagentsBySessionId.get(notice.childSessionId) + : undefined; + if ( + notice !== undefined && + subagent !== undefined && + (subagent.task.status === "running" || subagent.task.status === "pending") + ) { + // The agent is free to answer with text only and never call + // get_command_or_subagent_output; without this, the subagent + // row stays running and holds deferred finalize open forever. + yield* emitSubagent(context, { + nativeTaskId: subagent.task.nativeTaskRef?.nativeId ?? notice.childSessionId, + prompt: subagent.task.prompt, + title: subagent.task.title, + model: subagent.task.model, + status: notice.status, + childSessionId: notice.childSessionId, + result: null, + suppressNormalTool: true, + }); + } + } + break; default: { const parsed = parseSessionUpdateEvent(notification); for (const event of parsed.events) { @@ -1486,20 +3078,13 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV } } } + if (acpRootSessionUpdateIngestsOutput(notification)) { + yield* scheduleSettleRootTurnWhenIdle(context); + } + // Keep deferred finalize quiet-window fresh while wake traffic lands. + yield* rearmDeferredFinalize(context); }); - yield* runtime.handleSessionUpdate((notification) => - handleSessionUpdate(notification).pipe( - Effect.mapError( - (cause) => - new EffectAcpErrors.AcpTransportError({ - detail: "Failed to project an ACP session update", - cause, - }), - ), - ), - ); - const activeContext = Effect.gen(function* () { const context = yield* Ref.get(activeTurn); if (context === null) { @@ -1511,9 +3096,11 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV return context; }); - const emitApprovalRequest = Effect.fnUntraced(function* ( + const beginApprovalRequest = Effect.fnUntraced(function* ( context: ActiveAcpTurn, params: EffectAcpSchema.RequestPermissionRequest, + generation: number, + transportRequestId: string, ) { yield* closeTextStreams(context); const parsed = parsePermissionRequest(params); @@ -1524,6 +3111,15 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV nativeRequestId, }); const decision = yield* Deferred.make(); + const nativeResponseAcknowledgement = yield* Deferred.make< + void, + EffectAcpErrors.AcpError + >(); + yield* registerNativeResponseAcknowledgement( + generation, + transportRequestId, + nativeResponseAcknowledgement, + ); const now = yield* DateTime.now; const nodeId = idAllocator.derive.approvalNode({ requestId }); const requestKind = providerRequestKind(parsed.kind); @@ -1587,7 +3183,10 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV const updated = new Map(current); updated.set(String(requestId), { type: "approval", + generation, + nativeResponseAcknowledgement, requestId, + transportRequestId, decision, runtimeRequest, node, @@ -1611,55 +3210,19 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV driver, turnItem, }); - const resolved = yield* Deferred.await(decision).pipe( - Effect.ensuring( - Ref.update(pendingRuntimeRequests, (current) => { - const updated = new Map(current); - updated.delete(String(requestId)); - return updated; - }), - ), - ); - return resolved; + return { + context, + decision, + nativeResponseAcknowledgement, + requestId, + transportRequestId, + } as const; }); - yield* runtime.handleRequestPermission((params) => - Effect.gen(function* () { - const context = yield* activeContext; - const disposition = acpPermissionDisposition(context.input.runtimePolicy, params); - if (disposition === "allow") { - const optionId = selectAutoApprovedPermissionOption(params); - return optionId === undefined - ? ({ outcome: { outcome: "cancelled" } } as const) - : ({ outcome: { outcome: "selected", optionId } } as const); - } - if (disposition === "deny") { - const optionId = selectPermissionOptionId(params, "decline"); - return optionId === undefined - ? ({ outcome: { outcome: "cancelled" } } as const) - : ({ outcome: { outcome: "selected", optionId } } as const); - } - const decision = yield* emitApprovalRequest(context, params); - if (decision === "cancel") { - return { outcome: { outcome: "cancelled" } } as const; - } - const optionId = selectPermissionOptionId(params, decision); - return optionId === undefined - ? ({ outcome: { outcome: "cancelled" } } as const) - : ({ outcome: { outcome: "selected", optionId } } as const); - }).pipe( - Effect.mapError( - (cause) => - new EffectAcpErrors.AcpTransportError({ - detail: "Failed to handle an ACP permission request", - cause, - }), - ), - ), - ); - - const requestUserInputInternal = Effect.fnUntraced(function* ( + const beginUserInputRequest = Effect.fnUntraced(function* ( request: AcpAdapterV2UserInputRequest, + generation: number, + transportRequestId: string, ) { const context = yield* activeContext; yield* closeTextStreams(context); @@ -1669,6 +3232,15 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV nativeRequestId: request.nativeRequestId, }); const answers = yield* Deferred.make(); + const nativeResponseAcknowledgement = yield* Deferred.make< + void, + EffectAcpErrors.AcpError + >(); + yield* registerNativeResponseAcknowledgement( + generation, + transportRequestId, + nativeResponseAcknowledgement, + ); const now = yield* DateTime.now; const nodeId = idAllocator.derive.nodeFromProviderItem({ driver, @@ -1742,7 +3314,10 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV const updated = new Map(current); updated.set(String(requestId), { type: "user_input", + generation, + nativeResponseAcknowledgement, requestId, + transportRequestId, answers, runtimeRequest, node, @@ -1766,19 +3341,62 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV driver, turnItem, }); - return yield* Deferred.await(answers).pipe( - Effect.ensuring( - Ref.update(pendingRuntimeRequests, (current) => { - const updated = new Map(current); - updated.delete(String(requestId)); - return updated; - }), - ), - ); + return { + answers, + context, + nativeResponseAcknowledgement, + requestId, + transportRequestId, + } as const; }); - const requestUserInput = (request: AcpAdapterV2UserInputRequest) => - requestUserInputInternal(request).pipe( + const requestUserInputWithAdmission = ( + generation: number, + request: Effect.Effect, + transportRequestId: string, + ) => + runRuntimeCallbackAtGeneration( + generation, + request.pipe( + Effect.flatMap((value) => + beginUserInputRequest(value, generation, transportRequestId), + ), + ), + ).pipe( + Effect.flatMap((pending) => { + if (Option.isNone(pending)) return Effect.never; + const { answers, context, requestId, transportRequestId } = pending.value; + return Deferred.await(answers).pipe( + Effect.flatMap((result) => + runRuntimeCallbackAtGeneration(generation, Effect.succeed(result)).pipe( + Effect.flatMap((checked) => + Option.isSome(checked) + ? Effect.succeed({ + acknowledgeNativeResponse: acknowledgeNativeResponse( + generation, + transportRequestId, + ), + answers: checked.value, + }) + : Effect.never, + ), + ), + ), + Effect.ensuring( + runRuntimeCallbackAtGeneration( + generation, + Effect.gen(function* () { + yield* Ref.update(pendingRuntimeRequests, (current) => { + const updated = new Map(current); + updated.delete(String(requestId)); + return updated; + }); + yield* rearmRootTurnRecoveryTimers(context); + }), + ).pipe(Effect.asVoid), + ), + ); + }), Effect.mapError( (cause) => new EffectAcpErrors.AcpTransportError({ @@ -1789,7 +3407,10 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ); const cancelPendingRuntimeRequests = Effect.fnUntraced(function* () { - const requests = [...(yield* Ref.get(pendingRuntimeRequests)).values()]; + const requests = yield* Ref.modify(pendingRuntimeRequests, (current) => [ + [...current.values()], + new Map(), + ]); if (requests.length === 0) return; const now = yield* DateTime.now; @@ -1836,61 +3457,426 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ); }); - yield* runtime.handleElicitation((params) => - Effect.gen(function* () { - if (params.mode === "url") { - return { action: { action: "decline" } } as const; + /** + * Direct Stop after a soft steer clears carryover without an active turn. + * Emit the same interrupted terminal events terminalizeOpenRunOwnedItems + * would have, context-free (no ActiveAcpTurn). + */ + const terminalizeCarryoverSubagents = Effect.fnUntraced(function* ( + carryover: { + readonly sessionId: string; + readonly subagents: ReadonlyArray; + } | null, + ) { + if (carryover === null) return; + const now = yield* DateTime.now; + for (const subagent of carryover.subagents) { + if (subagent.task.status !== "running" && subagent.task.status !== "pending") { + continue; } - const questions = Object.entries(params.requestedSchema.properties ?? {}).map( - ([id, property], index): OrchestrationV2UserInputQuestion => { - const record = unknownRecord(property); - const enumValues = Array.isArray(record?.enum) - ? record.enum.filter((value): value is string => typeof value === "string") - : []; - const options = - enumValues.length > 0 - ? enumValues.map((value) => ({ label: value, description: value })) - : record?.type === "boolean" - ? [ - { label: "true", description: "Yes" }, - { label: "false", description: "No" }, - ] - : []; - return { - id, - header: nonEmptyText(record?.title, `Question ${index + 1}`), - question: nonEmptyText(record?.description, params.message), - options, - }; + const nativeTaskId = subagent.task.nativeTaskRef?.nativeId ?? subagent.task.id; + const nativeItemRef = { + driver, + nativeId: nativeTaskId, + strength: "strong" as const, + }; + const parentProviderThreadId = subagent.parentProviderThreadId; + const result = subagent.assistantText || subagent.task.result; + subagent.task = { + ...subagent.task, + status: "interrupted", + result, + completedAt: now, + updatedAt: now, + }; + yield* emitProviderEvent({ + type: "node.updated", + driver, + node: { + id: subagent.task.id, + threadId: subagent.task.threadId, + runId: subagent.task.runId, + parentNodeId: subagent.task.parentNodeId, + rootNodeId: subagent.task.parentNodeId, + kind: "subagent", + status: "interrupted", + countsForRun: false, + providerThreadId: parentProviderThreadId, + providerTurnId: subagent.providerTurnId, + nativeItemRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: subagent.task.startedAt, + completedAt: now, }, - ); - const ordinal = yield* Ref.getAndUpdate( - nextElicitationOrdinal, - (current) => current + 1, - ); - const nativeRequestId = `${params.sessionId}:elicitation:${ordinal}`; - const answers = yield* requestUserInput({ - nativeItemId: nativeRequestId, - nativeRequestId, - questions, }); - return answers === null - ? ({ action: { action: "cancel" } } as const) - : ({ - action: { - action: "accept", - content: elicitationContent( - answers, - new Set(Object.keys(params.requestedSchema.properties ?? {})), + yield* emitProviderEvent({ + type: "node.updated", + driver, + node: { + id: subagent.childRootNodeId, + threadId: subagent.childThreadId, + runId: null, + parentNodeId: null, + rootNodeId: subagent.childRootNodeId, + kind: "root_turn", + status: "interrupted", + countsForRun: false, + providerThreadId: subagent.task.providerThreadId, + providerTurnId: null, + nativeItemRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: subagent.task.startedAt, + completedAt: now, + }, + }); + yield* emitProviderEvent({ + type: "subagent.updated", + driver, + subagent: subagent.task, + }); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver, + turnItem: { + id: subagent.turnItemId, + threadId: subagent.task.threadId, + runId: subagent.task.runId, + nodeId: subagent.task.id, + providerThreadId: parentProviderThreadId, + providerTurnId: subagent.providerTurnId, + nativeItemRef, + parentItemId: null, + ordinal: subagent.turnItemOrdinal, + status: "interrupted", + title: subagent.task.title, + startedAt: subagent.task.startedAt, + completedAt: now, + updatedAt: now, + type: "subagent", + subagentId: subagent.task.id, + origin: "provider_native", + driver, + providerInstanceId: subagent.task.providerInstanceId, + childThreadId: subagent.childThreadId, + prompt: subagent.task.prompt, + result, + }, + }); + } + }); + + const wireAcpRuntimeHandlers = Effect.fnUntraced(function* () { + const handlerGeneration = yield* Ref.get(runtimeCallbackGeneration); + const requestUserInput = (request: AcpAdapterV2UserInputRequest) => + Effect.gen(function* () { + const transportRequestId = yield* claimNativeTransportRequest( + handlerGeneration, + (transport) => acpNativeUserInputRequestMatches(request, transport), + ); + const correlated = yield* runRuntimeCallbackAtGeneration( + handlerGeneration, + transportRequestId === undefined + ? new EffectAcpErrors.AcpTransportError({ + detail: + "Could not correlate the ACP user input request with its transport ID", + cause: "Could not correlate xAI user input transport request", + }) + : Effect.succeed(transportRequestId), + ); + if (Option.isNone(correlated)) return yield* Effect.never; + return yield* requestUserInputWithAdmission( + handlerGeneration, + Effect.succeed(request), + correlated.value, + ); + }); + yield* runtime.handleSessionUpdate((notification) => + runRuntimeCallbackAtGeneration( + handlerGeneration, + handleSessionUpdate(notification), + ).pipe( + Effect.asVoid, + Effect.mapError( + (cause) => + new EffectAcpErrors.AcpTransportError({ + detail: "Failed to project an ACP session update", + cause, + }), + ), + ), + ); + yield* runtime.handleRequestPermission((params) => + Effect.gen(function* () { + const transportRequestId = yield* claimNativeTransportRequest( + handlerGeneration, + ({ method, payload }) => + method === "session/request_permission" && + unknownRecord(payload)?.sessionId === params.sessionId && + unknownRecord(unknownRecord(payload)?.toolCall)?.toolCallId === + params.toolCall.toolCallId, + ); + const correlated = yield* runRuntimeCallbackAtGeneration( + handlerGeneration, + transportRequestId === undefined + ? new EffectAcpErrors.AcpTransportError({ + detail: + "Could not correlate the ACP permission request with its transport ID", + cause: "Could not correlate session/request_permission transport request", + }) + : Effect.succeed(transportRequestId), + ); + if (Option.isNone(correlated)) return yield* Effect.never; + const correlatedTransportRequestId = correlated.value; + const admitted = yield* runRuntimeCallbackAtGeneration( + handlerGeneration, + Effect.gen(function* () { + const context = yield* activeContext; + const disposition = acpPermissionDisposition(context.input.runtimePolicy, params); + if (disposition === "allow") { + const optionId = selectAutoApprovedPermissionOption(params); + return { + _tag: "Immediate" as const, + response: + optionId === undefined + ? ({ outcome: { outcome: "cancelled" } } as const) + : ({ outcome: { outcome: "selected", optionId } } as const), + }; + } + if (disposition === "deny") { + const optionId = selectPermissionOptionId(params, "decline"); + return { + _tag: "Immediate" as const, + response: + optionId === undefined + ? ({ outcome: { outcome: "cancelled" } } as const) + : ({ outcome: { outcome: "selected", optionId } } as const), + }; + } + return { + _tag: "Pending" as const, + pending: yield* beginApprovalRequest( + context, + params, + handlerGeneration, + correlatedTransportRequestId, ), - }, - } as const); - }), - ); + }; + }), + ); + if (Option.isNone(admitted)) { + return yield* Effect.never; + } + if (admitted.value._tag === "Immediate") { + const response = admitted.value.response; + const checked = yield* runRuntimeCallbackAtGeneration( + handlerGeneration, + Effect.gen(function* () { + const nativeResponseAcknowledgement = yield* Deferred.make< + void, + EffectAcpErrors.AcpError + >(); + yield* registerNativeResponseAcknowledgement( + handlerGeneration, + correlatedTransportRequestId, + nativeResponseAcknowledgement, + ); + return response; + }), + ); + return Option.isSome(checked) ? checked.value : yield* Effect.never; + } + const { + context, + decision: pendingDecision, + requestId, + transportRequestId: pendingTransportRequestId, + } = admitted.value.pending; + const decision = yield* Deferred.await(pendingDecision).pipe( + Effect.ensuring( + runRuntimeCallbackAtGeneration( + handlerGeneration, + Effect.gen(function* () { + yield* Ref.update(pendingRuntimeRequests, (current) => { + const updated = new Map(current); + updated.delete(String(requestId)); + return updated; + }); + yield* rearmRootTurnRecoveryTimers(context); + }), + ).pipe(Effect.asVoid), + ), + ); + const response = (() => { + if (decision === "cancel") { + return { outcome: { outcome: "cancelled" } } as const; + } + const optionId = selectPermissionOptionId(params, decision); + return optionId === undefined + ? ({ outcome: { outcome: "cancelled" } } as const) + : ({ outcome: { outcome: "selected", optionId } } as const); + })(); + const checked = yield* runRuntimeCallbackAtGeneration( + handlerGeneration, + Effect.succeed(response), + ); + if (Option.isNone(checked)) return yield* Effect.never; + yield* acknowledgeNativeResponse(handlerGeneration, pendingTransportRequestId); + return checked.value; + }).pipe( + Effect.mapError( + (cause) => + new EffectAcpErrors.AcpTransportError({ + detail: "Failed to handle an ACP permission request", + cause, + }), + ), + ), + ); + yield* runtime.handleElicitation((params) => + Effect.gen(function* () { + const transportRequestId = yield* claimNativeTransportRequest( + handlerGeneration, + ({ method, payload }) => { + const record = unknownRecord(payload); + return ( + method === "session/elicitation" && + record?.sessionId === params.sessionId && + record.message === params.message && + record.mode === params.mode && + (params.mode === "url" + ? record.elicitationId === params.elicitationId && record.url === params.url + : acpCanonicalJson(record.requestedSchema) === + acpCanonicalJson(params.requestedSchema)) + ); + }, + ); + const correlated = yield* runRuntimeCallbackAtGeneration( + handlerGeneration, + transportRequestId === undefined + ? new EffectAcpErrors.AcpTransportError({ + detail: + "Could not correlate the ACP elicitation request with its transport ID", + cause: "Could not correlate session/elicitation transport request", + }) + : Effect.succeed(transportRequestId), + ); + if (Option.isNone(correlated)) return yield* Effect.never; + const correlatedTransportRequestId = correlated.value; + if (params.mode === "url") { + const admitted = yield* runRuntimeCallbackAtGeneration( + handlerGeneration, + Effect.gen(function* () { + const nativeResponseAcknowledgement = yield* Deferred.make< + void, + EffectAcpErrors.AcpError + >(); + yield* registerNativeResponseAcknowledgement( + handlerGeneration, + correlatedTransportRequestId, + nativeResponseAcknowledgement, + ); + return { action: { action: "decline" } } as const; + }), + ); + if (Option.isNone(admitted)) return yield* Effect.never; + return admitted.value; + } + const questions = Object.entries(params.requestedSchema.properties ?? {}).map( + ([id, property], index): OrchestrationV2UserInputQuestion => { + const record = unknownRecord(property); + const enumValues = Array.isArray(record?.enum) + ? record.enum.filter((value): value is string => typeof value === "string") + : []; + const options = + enumValues.length > 0 + ? enumValues.map((value) => ({ label: value, description: value })) + : record?.type === "boolean" + ? [ + { label: "true", description: "Yes" }, + { label: "false", description: "No" }, + ] + : []; + return { + id, + header: nonEmptyText(record?.title, `Question ${index + 1}`), + question: nonEmptyText(record?.description, params.message), + options, + }; + }, + ); + const userInput = yield* requestUserInputWithAdmission( + handlerGeneration, + Effect.gen(function* () { + const ordinal = yield* Ref.getAndUpdate( + nextElicitationOrdinal, + (current) => current + 1, + ); + const nativeRequestId = `${params.sessionId}:elicitation:${ordinal}`; + return { + nativeItemId: nativeRequestId, + nativeRequestId, + questions, + }; + }), + correlatedTransportRequestId, + ); + const response = + userInput.answers === null + ? ({ action: { action: "cancel" } } as const) + : ({ + action: { + action: "accept", + content: elicitationContent( + userInput.answers, + new Set(Object.keys(params.requestedSchema.properties ?? {})), + ), + }, + } as const); + yield* userInput.acknowledgeNativeResponse; + return response; + }), + ); + if (flavor.registerExtensions !== undefined) { + yield* flavor.registerExtensions({ + runtime, + requestUserInput, + applyBackgroundTaskMutation: (mutation) => + Effect.gen(function* () { + // Direct Stop quarantine: drop residual task lifecycle from + // the stopped run instead of mutating wake machinery. + if (yield* Ref.get(stoppedRunQuarantine)) return; + // Root-session tasks only: a cancelled subagent's re-run in + // its child session must not gate root wake machinery. + if ((yield* Ref.get(activeSessionId)) !== mutation.sessionId) return; + yield* applyLateBackgroundMutation(mutation.sessionId, mutation); + }), + }); + } + }); + + const spawnAcpRuntime = Effect.fnUntraced(function* () { + if (runtimeScope !== undefined) { + yield* Scope.close(runtimeScope, Exit.void); + } + runtimeScope = yield* Scope.make(); + const runtimeGeneration = yield* Ref.get(runtimeCallbackGeneration); + runtime = yield* flavor + .makeRuntime(makeRuntimeInput(runtimeGeneration)) + .pipe( + Effect.provideService(Crypto.Crypto, options.crypto), + Effect.provideService(Scope.Scope, runtimeScope), + ); + }); - if (flavor.registerExtensions !== undefined) { - yield* flavor.registerExtensions({ runtime, requestUserInput }); - } + const restartAcpRuntime = Effect.fnUntraced(function* () { + yield* spawnAcpRuntime(); + yield* wireAcpRuntimeHandlers(); + }); + + yield* spawnAcpRuntime(); + yield* wireAcpRuntimeHandlers(); const started = yield* runtime.start(); yield* Ref.set(activeSessionId, started.sessionId); @@ -1899,8 +3885,11 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV const canLoadSession = started.initializeResult.agentCapabilities?.loadSession === true; const canResumeSession = started.initializeResult.agentCapabilities?.sessionCapabilities?.resume != null; - const supportsImagePrompts = - started.initializeResult.agentCapabilities?.promptCapabilities?.image === true; + const supportsImagePrompts = acpSupportsImagePrompts({ + flavorSupportsImagePrompts: flavor.supportsImagePrompts, + negotiatedImage: + started.initializeResult.agentCapabilities?.promptCapabilities?.image === true, + }); const activateSession = Effect.fnUntraced(function* (sessionId: string) { if (canLoadSession) { @@ -1998,16 +3987,90 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV completedAt, }); + const drainTrailingRootTurnChunks = Effect.fnUntraced(function* () { + if (!flavor.settleRootTurnWhenIdle) return; + // Projected via handleSessionUpdate, not getEvents(). Cooperative yield + // only — replay uses TestClock; Effect.sleep here would stall settlement. + yield* Effect.yieldNow; + yield* Effect.yieldNow; + }); + + const terminalizeOpenRunOwnedItems = Effect.fnUntraced(function* ( + context: ActiveAcpTurn, + options: { readonly terminalizeSubagents: boolean }, + ) { + for (const tool of context.tools.values()) { + const status = toolStatus(tool.status); + if (status === "pending" || status === "running") { + yield* emitTool(context, tool, "interrupted"); + } + } + if (!options.terminalizeSubagents) return; + for (const subagent of context.subagents.values()) { + if (subagent.task.status !== "running" && subagent.task.status !== "pending") { + continue; + } + yield* emitSubagent(context, { + nativeTaskId: subagent.task.nativeTaskRef?.nativeId ?? subagent.task.id, + prompt: subagent.task.prompt, + title: subagent.task.title, + model: subagent.task.model, + status: "interrupted", + childSessionId: subagent.childSessionId, + result: subagent.task.result, + suppressNormalTool: true, + }); + } + }); + + const terminalizeOpenForegroundTools = Effect.fnUntraced(function* ( + context: ActiveAcpTurn, + ) { + for (const tool of context.tools.values()) { + if (!acpCompletedTurnShouldTerminalizeTool(tool, flavor)) continue; + yield* emitTool(context, tool, "completed"); + } + }); + + const quarantineStoppedRun = Effect.fnUntraced(function* () { + yield* continuationPermit.withPermit( + Effect.gen(function* () { + yield* Ref.update(continuationGeneration, (value) => value + 1); + yield* Ref.set(stoppedRunQuarantine, true); + yield* Ref.set(wakeBuffer, []); + yield* Ref.set(continuationRequested, false); + yield* Ref.set(runningBackgroundTaskIds, new Set()); + yield* Ref.set(carryoverSubagents, null); + yield* Ref.set(lastTurnRoute, null); + }), + ); + }); + const finalizeTurn = Effect.fnUntraced(function* ( context: ActiveAcpTurn, status: "completed" | "interrupted" | "failed" | "cancelled", failure?: OrchestrationV2ProviderFailure, + options?: { readonly drainTrailingChunks?: boolean }, ) { if (context.finalized) return; + const settledStatus = context.interrupted ? "interrupted" : status; context.finalized = true; + if (options?.drainTrailingChunks === true) { + yield* drainTrailingRootTurnChunks(); + } + const directStopQuarantine = yield* Ref.get(stoppedRunQuarantine); + if (settledStatus === "completed") { + yield* terminalizeOpenForegroundTools(context); + } else if (settledStatus === "interrupted") { + // Direct Stop terminalizes every visible run-owned item. restart_active + // keeps live subagent lineages for in-process replacement carryover. + yield* terminalizeOpenRunOwnedItems(context, { + terminalizeSubagents: directStopQuarantine, + }); + } yield* closeTextStreams(context); const now = yield* DateTime.now; - const turn = providerTurnPayload(context, status, now); + const turn = providerTurnPayload(context, settledStatus, now); yield* Ref.update(providerTurns, (current) => { const updated = new Map(current); updated.set(String(turn.id), turn); @@ -2033,7 +4096,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }, }); yield* emitProviderEvent( - status === "failed" + settledStatus === "failed" ? { type: "turn.terminal", driver, @@ -2044,7 +4107,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV context, `terminal-failure:${context.providerTurnId}`, ), - status, + status: settledStatus, failure: failure ?? makeProviderFailure({ class: "provider_error" }), threadDisposition: "reusable", } @@ -2054,15 +4117,108 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV providerThreadId: context.input.providerThread.id, providerTurnId: context.providerTurnId, runOrdinal: context.input.runOrdinal, - status, + status: settledStatus, failure: null, threadDisposition: "reusable", }, ); + const liveSubagents = [...context.subagents.values()].filter( + (subagent) => subagent.task.status === "running" || subagent.task.status === "pending", + ); + // Direct Stop must not carry residual subagents into a later run. + if (liveSubagents.length > 0 && !directStopQuarantine) { + const sessionId = yield* Ref.get(activeSessionId); + if (sessionId !== null) { + yield* Ref.set(carryoverSubagents, { sessionId, subagents: liveSubagents }); + } + } yield* Ref.set(activeTurn, null); yield* Deferred.succeed(context.completed, undefined).pipe(Effect.ignore); }); + const trySettleRootTurnWhenIdle = Effect.fnUntraced(function* (context: ActiveAcpTurn) { + const pending = yield* Ref.get(pendingRuntimeRequests); + const hasPendingRuntimeRequest = acpTurnHasPendingRuntimeRequest( + context.providerTurnId, + pending, + ); + const hasRunningTool = [...context.tools.values()].some((tool) => { + const status = toolStatus(tool.status); + return status === "pending" || status === "running"; + }); + // Debounce already proved root-session quiescence; open segment handles + // without an explicit close should not block settlement. + const hasRunningSubagent = [...context.subagents.values()].some( + (subagent) => subagent.task.status === "running", + ); + if ( + !acpRootTurnIsIdle({ + finalized: context.finalized, + interrupted: context.interrupted, + assistantStreamOpen: false, + reasoningStreamOpen: false, + hasRunningTool, + hasPendingRuntimeRequest, + hasToolHistory: context.tools.size > 0, + hasRunningSubagent, + hasOutput: context.assistant.nextSegment > 0, + }) + ) { + return; + } + // Never session/cancel here. Speculative settle must not kill in-flight + // Grok work; late tools would arrive with activeTurn null and drop. + yield* finalizeTurn(context, "completed", undefined, { drainTrailingChunks: true }); + }); + + scheduleDeferredFinalize = (context) => + Effect.gen(function* () { + if (!flavor.deferFinalizeForBackgroundWork) return; + if (!context.promptSettled || context.finalized || context.interrupted) return; + if (hasDeferredBackgroundWork(context)) return; + context.backgroundFinalizeGeneration += 1; + const generation = context.backgroundFinalizeGeneration; + // Minimal quiet for all models (no per-model carveouts). Defer + + // awaitingBackgroundHydration hold the turn through monitors; this + // is only a short debounce after the last rearm so a slightly late + // first post-hydration assistant chunk is not dropped. Multi-second + // floors (4–20s) only prolonged Working on grok-4.5. + yield* Effect.gen(function* () { + yield* Effect.sleep("2000 millis"); + if ( + context.finalized || + context.interrupted || + context.backgroundFinalizeGeneration !== generation + ) { + return; + } + if (hasDeferredBackgroundWork(context)) return; + const status = context.promptSettledStatus ?? "completed"; + yield* finalizeTurn(context, status, undefined, { drainTrailingChunks: true }); + }).pipe(Effect.forkIn(sessionScope), Effect.asVoid); + }); + + scheduleSettleRootTurnWhenIdle = (context) => + Effect.gen(function* () { + if (!flavor.settleRootTurnWhenIdle) return; + context.settleScheduleGeneration += 1; + const generation = context.settleScheduleGeneration; + yield* Effect.gen(function* () { + yield* Effect.sleep(`${acpRootTurnSettleDebounceMs} millis`); + if (context.finalized || context.interrupted) return; + if (context.settleScheduleGeneration !== generation) return; + const active = yield* Ref.get(activeTurn); + if (active !== context) return; + yield* trySettleRootTurnWhenIdle(context); + }).pipe(Effect.forkIn(sessionScope), Effect.asVoid); + }); + + rearmRootTurnRecoveryTimers = (context) => + Effect.gen(function* () { + if (!acpRootTurnShouldRearmRecoveryTimers(context)) return; + yield* scheduleSettleRootTurnWhenIdle(context); + }); + const resolvePromptParts = Effect.fnUntraced(function* ( turnInput: ProviderAdapterV2TurnInput, ) { @@ -2112,8 +4268,26 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV return prompt; }); - const startTurn = Effect.fn("AcpAdapterV2.startTurn")( + const restartRuntimeAfterTeardownIfRequired = Effect.fnUntraced(function* () { + const restartRequired = yield* Ref.get(runtimeRestartRequired); + if (!restartRequired) return false; + yield* restartAcpRuntime(); + yield* Ref.set(runtimeRestartRequired, false); + yield* Ref.set(activeSessionId, null); + yield* Ref.set(activeSessionSetup, null); + yield* Ref.set(activeSelection, null); + yield* Ref.set(snapshot, { + order: [], + messages: new Map(), + loadingRole: null, + loadingIndex: 0, + }); + return true; + }); + + const startTurnUnlocked = Effect.fn("AcpAdapterV2.startTurn")( function* (turnInput: ProviderAdapterV2TurnInput) { + yield* awaitRuntimeTeardown(); const existing = yield* Ref.get(activeTurn); if (existing !== null) { return yield* new ProviderAdapterProtocolError({ @@ -2122,7 +4296,10 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }); } const requestedSessionId = nativeThreadId(driver, turnInput.providerThread); - if ((yield* Ref.get(activeSessionId)) !== requestedSessionId) { + const restartAfterInterrupt = yield* restartRuntimeAfterTeardownIfRequired(); + const needsSessionActivation = + (yield* Ref.get(activeSessionId)) !== requestedSessionId || restartAfterInterrupt; + if (needsSessionActivation) { const activated = yield* activateSession(requestedSessionId); yield* Ref.set(activeSessionId, activated.sessionId); yield* Ref.set(activeSessionSetup, activated); @@ -2149,11 +4326,32 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV yield* Ref.set(activeSelection, turnInput.modelSelection); } } - const prompt = yield* resolvePromptParts(turnInput); + yield* Ref.set(lastTurnRoute, { + threadId: turnInput.threadId, + providerThreadId: turnInput.providerThread.id, + }); + yield* Ref.set(suppressPostSettleMonitorPrompt, false); + yield* Ref.set(handledBackgroundTaskIdsInActiveTurn, new Set()); + // Continuation turns attach to wake traffic the agent already produced + // after the prior root turn settled; do not re-prompt the ACP session. + const isContinuationTurn = + postSettleContinuationEnabled && + turnInput.message.createdBy === "agent" && + turnInput.message.creationSource === "provider"; + // Drop a sticky continuation offer when any new turn starts so idle + // pin and further offers cannot wed on a completed or failed dispatch. + yield* continuationPermit.withPermit( + Effect.gen(function* () { + yield* Ref.update(continuationGeneration, (value) => value + 1); + yield* Ref.set(continuationRequested, false); + }), + ); + const prompt = isContinuationTurn ? null : yield* resolvePromptParts(turnInput); const startedAt = yield* DateTime.now; const nativeTurnId = `${requestedSessionId}:turn:${turnInput.providerTurnOrdinal}`; const providerTurnId = idAllocator.derive.providerTurn({ driver, nativeTurnId }); const completed = yield* Deferred.make(); + const promptWireSettled = yield* Deferred.make(); const context: ActiveAcpTurn = { input: turnInput, providerTurnId, @@ -2167,11 +4365,35 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV subagents: new Map(), subagentsBySessionId: new Map(), pendingSubagentNotifications: new Map(), + toolCallIdsByBackgroundTaskId: new Map(), + persistentBackgroundTaskIds: new Set(), + awaitingBackgroundHydration: new Set(), + pendingInjectedReport: new Set(), plan: null, interrupted: false, finalized: false, + settleScheduleGeneration: 0, + promptSettled: false, + promptSettledStatus: null, + promptWireSettled, + backgroundFinalizeGeneration: 0, }; + const carryover = yield* Ref.getAndSet(carryoverSubagents, null); + if (carryover !== null && carryover.sessionId === requestedSessionId) { + for (const subagent of carryover.subagents) { + const nativeId = subagent.task.nativeTaskRef?.nativeId ?? null; + if (nativeId !== null) { + context.subagents.set(nativeId, subagent); + } + if (subagent.childSessionId !== null) { + context.subagentsBySessionId.set(subagent.childSessionId, subagent); + } + } + } yield* Ref.set(activeTurn, context); + // Direct Stop closes and recreates the old runtime before reaching + // this reset. The quarantine remains session-scoped by design. + yield* Ref.set(stoppedRunQuarantine, false); const runningTurn = providerTurnPayload(context, "running", null); yield* Ref.update(providerTurns, (current) => { const updated = new Map(current); @@ -2208,35 +4430,100 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV createdAt: startedAt, updatedAt: startedAt, }); - yield* runtime.prompt({ prompt }).pipe( - Effect.flatMap((result) => { - const status = - result.stopReason === "cancelled" - ? context.interrupted - ? "interrupted" - : "cancelled" - : "completed"; - return finalizeTurn(context, status); - }), + if (isContinuationTurn) { + const drained = yield* Ref.modify(wakeBuffer, (current) => { + const next: Array = []; + return [current.slice(), next] as const; + }); + yield* Ref.set(continuationRequested, false); + // Treat attach mode as prompt-settled so deferred finalize / quiet + // windows can complete the continuation after wake traffic drains. + context.promptSettled = true; + context.promptSettledStatus = "completed"; + if (drained.length === 0) { + yield* finalizeTurn(context, "completed", undefined, { + drainTrailingChunks: true, + }); + return; + } + for (const notification of drained) { + yield* handleSessionUpdate(notification); + } + if (!context.finalized) { + if (hasDeferredBackgroundWork(context)) { + yield* rearmDeferredFinalize(context); + } else { + yield* scheduleDeferredFinalize(context); + } + } + return; + } + const promptGeneration = yield* Ref.get(runtimeCallbackGeneration); + yield* runtime.prompt({ prompt: prompt! }).pipe( + // Wire settlement precedes the completion callback's permit request so + // settled-soft classification can observe the native return even when + // the completion fiber has not yet set promptSettled under the permit. + Effect.tap(() => + Deferred.succeed(context.promptWireSettled, undefined).pipe(Effect.asVoid), + ), + Effect.flatMap((result) => + runRuntimeCallbackAtGeneration( + promptGeneration, + Effect.gen(function* () { + if (context.finalized) return; + const status = + result.stopReason === "cancelled" + ? context.interrupted + ? "interrupted" + : "cancelled" + : "completed"; + // Grok monitors (and async subagents) keep working after the root + // prompt RPC returns. Defer finalize so their later updates and + // wake-turn traffic still project onto this run. + if ( + flavor.deferFinalizeForBackgroundWork === true && + !context.interrupted && + hasDeferredBackgroundWork(context) + ) { + context.promptSettled = true; + context.promptSettledStatus = status; + return; + } + // Only completed turns drain trailing chunks. Interrupted turns + // must not wait for residual output from a stopped prompt. + yield* finalizeTurn(context, status, undefined, { + drainTrailingChunks: status === "completed", + }); + }), + ).pipe(Effect.asVoid), + ), + // Prompt failure is not wire-settled: only a successful resolve marks + // the signal. catchCause must not complete promptWireSettled. Effect.catchCause((cause) => - finalizeTurn( - context, - context.interrupted ? "interrupted" : "failed", - makeProviderFailure({ - cause: Cause.squash(cause), - class: "provider_error", + runRuntimeCallbackAtGeneration( + promptGeneration, + Effect.gen(function* () { + if (context.finalized) return; + yield* finalizeTurn( + context, + context.interrupted ? "interrupted" : "failed", + makeProviderFailure({ + cause: Cause.squash(cause), + class: "provider_error", + }), + ).pipe( + Effect.andThen( + Effect.logWarning("orchestration-v2.acp-prompt-failed", { + driver, + providerSessionId: input.providerSessionId, + providerThreadId: turnInput.providerThread.id, + providerTurnId, + cause, + }), + ), + ); }), - ).pipe( - Effect.andThen( - Effect.logWarning("orchestration-v2.acp-prompt-failed", { - driver, - providerSessionId: input.providerSessionId, - providerThreadId: turnInput.providerThread.id, - providerTurnId, - cause, - }), - ), - ), + ).pipe(Effect.asVoid), ), Effect.forkIn(sessionScope), ); @@ -2256,8 +4543,20 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ), ); + const startTurn = Effect.fn("AcpAdapterV2.startTurn.transition")(function* ( + turnInput: ProviderAdapterV2TurnInput, + ) { + return yield* runtimeTransitionPermit.withPermit(startTurnUnlocked(turnInput)); + }); + yield* Effect.addFinalizer(() => Effect.gen(function* () { + yield* continuationPermit.withPermit( + Effect.gen(function* () { + yield* Ref.set(continuationClosed, true); + yield* Ref.update(continuationGeneration, (value) => value + 1); + }), + ); const requests = [...(yield* Ref.get(pendingRuntimeRequests)).values()]; yield* Effect.forEach( requests, @@ -2267,14 +4566,40 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV : Deferred.succeed(request.answers, null).pipe(Effect.ignore), { discard: true }, ); + const closingError = new EffectAcpErrors.AcpTransportError({ + detail: "The ACP session closed before its admitted response reached the transport", + cause: "ACP session transport closed", + }); + yield* Effect.forEach( + requests, + (request) => Deferred.fail(request.nativeResponseAcknowledgement, closingError), + { discard: true }, + ); + const transportHadOutstandingResponses = yield* closeNativeTransport; + yield* options.testHooks?.afterNativeResponseTransportClosed?.() ?? Effect.void; const sessionCapabilities = started.initializeResult.agentCapabilities?.sessionCapabilities; if (sessionCapabilities?.close != null) { - yield* runtime.closeSession().pipe(Effect.ignore); + yield* runtimeTransitionPermit.withPermitsIfAvailable(1)( + Effect.gen(function* () { + const teardownState = yield* Ref.get(runtimeTeardownState); + const restartRequired = yield* Ref.get(runtimeRestartRequired); + if ( + teardownState._tag === "Idle" && + !restartRequired && + !transportHadOutstandingResponses + ) { + yield* runtime.closeSession().pipe(Effect.ignore); + } + }), + ); } if (flavor.assertComplete !== undefined) { yield* flavor.assertComplete.pipe(Effect.orDie); } + if (runtimeScope !== undefined) { + yield* Scope.close(runtimeScope, Exit.void).pipe(Effect.ignore); + } }), ); @@ -2284,6 +4609,16 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV providerSessionId: input.providerSessionId, providerSession, events: Stream.fromEffectRepeat(Queue.take(events)), + ...(postSettleContinuationEnabled + ? { + hasPendingBackgroundWork: Effect.gen(function* () { + if ((yield* Ref.get(wakeBuffer)).length > 0) return true; + if (yield* Ref.get(continuationRequested)) return true; + if ((yield* Ref.get(runningBackgroundTaskIds)).size > 0) return true; + return false; + }), + } + : {}), ensureThread: Effect.fn("AcpAdapterV2.ensureThread")( function* (threadInput: ProviderAdapterV2EnsureThreadInput) { const now = yield* DateTime.now; @@ -2322,32 +4657,38 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV readonly modelSelection?: ModelSelection; readonly runtimePolicy?: ProviderAdapterV2RuntimePolicy; }) { - const sessionId = nativeThreadId(driver, threadInput.providerThread); - if ((yield* Ref.get(activeSessionId)) !== sessionId) { - yield* Ref.set(snapshot, { - order: [], - messages: new Map(), - loadingRole: null, - loadingIndex: 0, - }); - const activated = yield* activateSession(sessionId); - yield* Ref.set(activeSessionId, activated.sessionId); - yield* Ref.set(activeSessionSetup, activated); - const nextSelection = threadInput.modelSelection ?? input.modelSelection; - yield* configureSession( - activated, - nextSelection, - threadInput.runtimePolicy ?? input.runtimePolicy, - ); - yield* Ref.set(activeSelection, nextSelection); - } - const now = yield* DateTime.now; - return { - ...threadInput.providerThread, - providerSessionId: input.providerSessionId, - status: "idle" as const, - updatedAt: now, - }; + return yield* runtimeTransitionPermit.withPermit( + Effect.gen(function* () { + yield* awaitRuntimeTeardown(); + const restartAfterInterrupt = yield* restartRuntimeAfterTeardownIfRequired(); + const sessionId = nativeThreadId(driver, threadInput.providerThread); + if ((yield* Ref.get(activeSessionId)) !== sessionId || restartAfterInterrupt) { + yield* Ref.set(snapshot, { + order: [], + messages: new Map(), + loadingRole: null, + loadingIndex: 0, + }); + const activated = yield* activateSession(sessionId); + yield* Ref.set(activeSessionId, activated.sessionId); + yield* Ref.set(activeSessionSetup, activated); + const nextSelection = threadInput.modelSelection ?? input.modelSelection; + yield* configureSession( + activated, + nextSelection, + threadInput.runtimePolicy ?? input.runtimePolicy, + ); + yield* Ref.set(activeSelection, nextSelection); + } + const now = yield* DateTime.now; + return { + ...threadInput.providerThread, + providerSessionId: input.providerSessionId, + status: "idle" as const, + updatedAt: now, + }; + }), + ); }, (effect, threadInput) => effect.pipe( @@ -2372,24 +4713,292 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ), interruptTurn: Effect.fn("AcpAdapterV2.interruptTurn")( function* (turnInput: ProviderAdapterV2InterruptInput) { - const context = yield* Ref.get(activeTurn); - if (context?.providerTurnId !== turnInput.providerTurnId) { - return yield* new ProviderAdapterProtocolError({ - driver, - detail: `ACP provider turn ${turnInput.providerTurnId} is not active`, - }); - } - context.interrupted = true; - yield* runtime.cancel.pipe(Effect.ensuring(cancelPendingRuntimeRequests())); - const stopped = yield* Deferred.await(context.completed).pipe( - Effect.timeoutOption("10 seconds"), + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + yield* restore(runtimeTransitionPermit.take(1)); + return yield* Effect.gen(function* () { + const transition = yield* Effect.gen(function* () { + const interruptContext = yield* Ref.get(activeTurn); + // Settled steering restart: the native prompt already + // returned and this is not a user Stop, so keep the + // process (and its background subagents) alive instead + // of hard-killing it. The turn still terminalizes below. + const softSteerCandidate = + flavor.preserveRuntimeOnSettledInterrupt === true && + turnInput.requestRuntimeRestart !== true && + interruptContext?.providerTurnId === turnInput.providerTurnId; + // Settlement is read under the callback permit after + // draining admitted responses: the native prompt can have + // returned on the wire while its completion callback has + // not yet run, and a cancel sent in that window is exactly + // what settled-soft mode exists to avoid. Also treat the + // turn as settled when the wire signal is already done + // (non-blocking; do not await under the permit). + let settledSoftInterrupt = false; + if (softSteerCandidate && interruptContext !== null) { + const promptSettledUnderPermit = yield* runtimeCallbackPermit.withPermit( + awaitAdmittedNativeResponses.pipe( + Effect.andThen( + Effect.sync(() => interruptContext.promptSettled === true), + ), + ), + ); + const wireSettled = yield* Deferred.isDone( + interruptContext.promptWireSettled, + ); + settledSoftInterrupt = promptSettledUnderPermit || wireSettled; + } + const restartRuntime = + !settledSoftInterrupt && + (turnInput.requestRuntimeRestart === true || + flavor.restartRuntimeOnEveryInterrupt === true); + const hardRestart = + restartRuntime && flavor.terminateRuntimeProcessGroupOnInterrupt === true; + if (hardRestart) { + const teardownState = yield* Ref.get(runtimeTeardownState); + if (teardownState._tag === "Failed") { + return yield* teardownState.error; + } + if (teardownState._tag === "InProgress") { + // Concurrent Stop/restart: wait for the in-flight hard + // teardown instead of failing the durable effect. + yield* Effect.logInfo( + "ACP interrupt awaiting in-progress hard teardown", + { + driver, + providerTurnId: turnInput.providerTurnId, + }, + ); + yield* Deferred.await(teardownState.completed); + return undefined; + } + } + const context = interruptContext; + if (context?.providerTurnId !== turnInput.providerTurnId) { + // A soft steering interrupt can clear the turn while + // intentionally leaving the process alive. A queued user + // Stop must still contain that orphan runtime. With a + // different live turn active this stays a pure no-op + // success instead: quarantine and teardown are + // session-global and would maim the replacement turn. + const containOrphanRuntime = + context === null && + hardRestart && + turnInput.requestRuntimeRestart === true; + // Transport death or a prior Stop already cleared the + // turn. Failing here caused effect-worker retries while + // the process was already gone; treat as success. + yield* Effect.logWarning( + containOrphanRuntime + ? "ACP Stop raced a soft interrupt that cleared the turn; containing the orphan runtime" + : "ACP interrupt raced transport teardown or a prior Stop; treating as already interrupted", + { + driver, + requestedProviderTurnId: turnInput.providerTurnId, + activeProviderTurnId: context?.providerTurnId ?? null, + hardRestart, + requestRuntimeRestart: turnInput.requestRuntimeRestart === true, + teardownState: (yield* Ref.get(runtimeTeardownState))._tag, + }, + ); + if (containOrphanRuntime) { + const teardownBarrier = yield* Deferred.make< + void, + ProviderAdapterProtocolError + >(); + yield* runtimeCallbackPermit.withPermit( + Effect.gen(function* () { + yield* awaitAdmittedNativeResponses; + const stoppedGeneration = yield* Ref.get(runtimeCallbackGeneration); + yield* quarantineNativeTransportAtGeneration(stoppedGeneration); + yield* Ref.set(runtimeTeardownState, { + _tag: "InProgress", + completed: teardownBarrier, + }); + yield* Ref.update( + runtimeCallbackGeneration, + (generation) => generation + 1, + ); + }), + ); + // Capture before quarantineStoppedRun clears carryover. + const orphanCarryover = yield* Ref.getAndSet(carryoverSubagents, null); + yield* quarantineStoppedRun(); + // Match the main hard-restart path: cancel pending + // approvals/elicitations after quarantine, before kill. + yield* cancelPendingRuntimeRequests(); + yield* terminalizeCarryoverSubagents(orphanCarryover); + if (runtime.terminateProcessGroup === undefined) { + const error = new ProviderAdapterProtocolError({ + driver, + detail: + "ACP runtime does not expose its required process-group teardown; the session is poisoned", + }); + yield* Ref.set(runtimeTeardownState, { _tag: "Failed", error }); + yield* Deferred.fail(teardownBarrier, error).pipe(Effect.ignore); + return yield* error; + } + const teardownExit = yield* runtime.terminateProcessGroup.pipe( + Effect.exit, + ); + if (Exit.isFailure(teardownExit)) { + const error = new ProviderAdapterProtocolError({ + driver, + detail: + "ACP orphan runtime process-group teardown failed; the session is poisoned", + payload: Cause.squash(teardownExit.cause), + }); + yield* Ref.set(runtimeTeardownState, { _tag: "Failed", error }); + yield* Deferred.fail(teardownBarrier, error).pipe(Effect.ignore); + return yield* error; + } + yield* Ref.set(runtimeRestartRequired, true); + yield* Ref.set(runtimeTeardownState, { _tag: "Idle" }); + yield* Deferred.succeed(teardownBarrier, undefined).pipe(Effect.ignore); + } + return undefined; + } + const teardownBarrier = hardRestart + ? yield* Deferred.make() + : null; + if (teardownBarrier !== null) { + yield* runtimeCallbackPermit.withPermit( + Effect.gen(function* () { + yield* awaitAdmittedNativeResponses; + const stoppedGeneration = yield* Ref.get(runtimeCallbackGeneration); + yield* quarantineNativeTransportAtGeneration(stoppedGeneration); + yield* Ref.set(runtimeTeardownState, { + _tag: "InProgress", + completed: teardownBarrier, + }); + yield* Ref.update( + runtimeCallbackGeneration, + (generation) => generation + 1, + ); + yield* ( + options.testHooks?.afterHardTeardownTransportDrained?.() ?? + Effect.void + ); + }), + ); + } + return { + context, + restartRuntime, + hardRestart, + teardownBarrier, + settledSoftInterrupt, + }; + }); + // Concurrent hard teardown already completed above. + if (transition === undefined) return; + const { + context, + restartRuntime, + hardRestart, + teardownBarrier, + settledSoftInterrupt, + } = transition; + const poisonTeardown = Effect.fnUntraced(function* ( + detail: string, + payload?: unknown, + ) { + const error = new ProviderAdapterProtocolError({ + driver, + detail, + ...(payload === undefined ? {} : { payload }), + }); + yield* Ref.set(runtimeTeardownState, { _tag: "Failed", error }); + yield* Deferred.fail(teardownBarrier!, error).pipe(Effect.ignore); + return error; + }); + const runTransition = Effect.gen(function* () { + context.interrupted = true; + // Quarantine only when this run is discarded (user Stop / + // hard process kill). Soft in-process restarts must keep + // carryoverSubagents so a still-running subagent can + // complete into the replacement turn. + if (hardRestart || turnInput.requestRuntimeRestart === true) { + yield* quarantineStoppedRun(); + } + yield* cancelPendingRuntimeRequests(); + // Finalize before process-group kill so projection/UI cannot + // lag a dead transport, and concurrent interrupt effects see + // activeTurn cleared (idempotent success) rather than racing. + if ( + (hardRestart || context.promptSettled || settledSoftInterrupt) && + !context.finalized + ) { + // hardRestart: always terminalize locally. + // promptSettled / settledSoftInterrupt (incl. wire-settled): + // native prompt already returned; only deferred background + // work remains, so session/cancel has nothing to acknowledge. + yield* finalizeTurn(context, "interrupted"); + } + if (hardRestart) { + if (runtime.terminateProcessGroup === undefined) { + return yield* poisonTeardown( + "ACP runtime does not expose its required process-group teardown; the session is poisoned", + ).pipe(Effect.flatMap(Effect.fail)); + } + const teardownExit = yield* runtime.terminateProcessGroup!.pipe( + Effect.exit, + ); + if (Exit.isFailure(teardownExit)) { + return yield* poisonTeardown( + "ACP runtime process-group teardown failed; the session is poisoned", + Cause.squash(teardownExit.cause), + ).pipe(Effect.flatMap(Effect.fail)); + } + // Process group is gone; the next turn must spawn a + // replacement even if the flavor only set hardRestart + // without restartRuntimeAfterInterrupt. + yield* Ref.set(runtimeRestartRequired, true); + yield* Ref.set(runtimeTeardownState, { _tag: "Idle" }); + yield* Deferred.succeed(teardownBarrier!, undefined).pipe(Effect.ignore); + } else { + // Settled soft interrupt: the native prompt already + // returned, so session/cancel has nothing to + // acknowledge and would only threaten still-running + // background subagents. Skip it and leave the runtime + // untouched for the replacement turn. + if (!settledSoftInterrupt) { + yield* runtime.cancel; + } + if (restartRuntime && flavor.restartRuntimeAfterInterrupt === true) { + yield* Ref.set(runtimeRestartRequired, true); + } + } + const stopped = yield* Deferred.await(context.completed).pipe( + Effect.timeoutOption("10 seconds"), + ); + if (Option.isNone(stopped)) { + return yield* new ProviderAdapterProtocolError({ + driver, + detail: `ACP provider turn ${turnInput.providerTurnId} did not acknowledge cancellation before the interrupt timeout`, + }); + } + }); + if (!hardRestart) { + return yield* restore(runTransition); + } + return yield* runTransition.pipe( + Effect.catchCause((cause) => + Effect.gen(function* () { + const state = yield* Ref.get(runtimeTeardownState); + if (state._tag === "InProgress") { + yield* poisonTeardown( + "ACP hard teardown failed unexpectedly; the session is poisoned", + Cause.squash(cause), + ); + } + return yield* Effect.failCause(cause); + }), + ), + ); + }).pipe(Effect.ensuring(runtimeTransitionPermit.release(1))); + }), ); - if (Option.isNone(stopped)) { - return yield* new ProviderAdapterProtocolError({ - driver, - detail: `ACP provider turn ${turnInput.providerTurnId} did not acknowledge cancellation before the interrupt timeout`, - }); - } }, (effect, turnInput) => effect.pipe( @@ -2405,28 +5014,43 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ), ), respondToRuntimeRequest: (requestInput) => - Effect.gen(function* () { - const pending = (yield* Ref.get(pendingRuntimeRequests)).get( - String(requestInput.requestId), - ); - if (pending === undefined) { - return yield* new ProviderAdapterProtocolError({ - driver, - detail: `No pending ACP runtime request ${requestInput.requestId}`, - }); - } - if (pending.type === "user_input") { - yield* Deferred.succeed(pending.answers, requestInput.answers ?? null); - return; - } - if (requestInput.decision === undefined) { - return yield* new ProviderAdapterProtocolError({ - driver, - detail: `ACP approval request ${requestInput.requestId} requires a decision`, - }); - } - yield* Deferred.succeed(pending.decision, requestInput.decision); - }).pipe( + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + yield* restore(runtimeTransitionPermit.take(1)); + return yield* Effect.gen(function* () { + yield* awaitRuntimeTeardown(); + const generation = yield* Ref.get(runtimeCallbackGeneration); + const pending = (yield* Ref.get(pendingRuntimeRequests)).get( + String(requestInput.requestId), + ); + if (pending === undefined || pending.generation !== generation) { + return yield* new ProviderAdapterProtocolError({ + driver, + detail: `No pending ACP runtime request ${requestInput.requestId}`, + }); + } + const settled = + pending.type === "user_input" + ? yield* Deferred.succeed(pending.answers, requestInput.answers ?? null) + : requestInput.decision === undefined + ? yield* new ProviderAdapterProtocolError({ + driver, + detail: `ACP approval request ${requestInput.requestId} requires a decision`, + }) + : yield* Deferred.succeed(pending.decision, requestInput.decision); + if (!settled) { + return yield* new ProviderAdapterProtocolError({ + driver, + detail: `ACP runtime request ${requestInput.requestId} was already resolved`, + }); + } + yield* awaitNativeResponseAcknowledgements([ + [pending.transportRequestId, pending.nativeResponseAcknowledgement], + ]); + yield* Deferred.await(pending.nativeResponseAcknowledgement); + }).pipe(Effect.ensuring(runtimeTransitionPermit.release(1))); + }), + ).pipe( Effect.mapError( (cause) => new ProviderAdapterRuntimeRequestResponseError({ @@ -2438,42 +5062,48 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ), readThreadSnapshot: Effect.fn("AcpAdapterV2.readThreadSnapshot")( function* (snapshotInput) { - const sessionId = nativeThreadId(driver, snapshotInput.providerThread); - if ((yield* Ref.get(activeSessionId)) !== sessionId) { - if (!capabilities.threads.canReadThreadSnapshot) { - return yield* new ProviderAdapterProtocolError({ - driver, - detail: "ACP driver does not support session/load snapshots", - }); - } - yield* Ref.set(snapshot, { - order: [], - messages: new Map(), - loadingRole: null, - loadingIndex: 0, - }); - const activated = yield* runtime.loadSession(sessionId); - yield* Ref.set(activeSessionId, activated.sessionId); - yield* Ref.set(activeSessionSetup, activated); - yield* Ref.set(activeSelection, null); - } - const state = yield* Ref.get(snapshot); - const now = yield* DateTime.now; - return { - providerThread: { - ...snapshotInput.providerThread, - providerSessionId: input.providerSessionId, - status: "idle" as const, - updatedAt: now, - }, - providerTurns: [...(yield* Ref.get(providerTurns)).values()], - messages: state.order.flatMap((key) => { - const message = state.messages.get(key); - return message === undefined ? [] : [message]; + return yield* runtimeTransitionPermit.withPermit( + Effect.gen(function* () { + yield* awaitRuntimeTeardown(); + yield* restartRuntimeAfterTeardownIfRequired(); + const sessionId = nativeThreadId(driver, snapshotInput.providerThread); + if ((yield* Ref.get(activeSessionId)) !== sessionId) { + if (!capabilities.threads.canReadThreadSnapshot) { + return yield* new ProviderAdapterProtocolError({ + driver, + detail: "ACP driver does not support session/load snapshots", + }); + } + yield* Ref.set(snapshot, { + order: [], + messages: new Map(), + loadingRole: null, + loadingIndex: 0, + }); + const activated = yield* runtime.loadSession(sessionId); + yield* Ref.set(activeSessionId, activated.sessionId); + yield* Ref.set(activeSessionSetup, activated); + yield* Ref.set(activeSelection, null); + } + const state = yield* Ref.get(snapshot); + const now = yield* DateTime.now; + return { + providerThread: { + ...snapshotInput.providerThread, + providerSessionId: input.providerSessionId, + status: "idle" as const, + updatedAt: now, + }, + providerTurns: [...(yield* Ref.get(providerTurns)).values()], + messages: state.order.flatMap((key) => { + const message = state.messages.get(key); + return message === undefined ? [] : [message]; + }), + runtimeRequests: [], + providerPayload: { protocol: ACP_PROTOCOL, sessionId }, + }; }), - runtimeRequests: [], - providerPayload: { protocol: ACP_PROTOCOL, sessionId }, - }; + ); }, (effect, snapshotInput) => effect.pipe( @@ -2498,42 +5128,48 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ), forkThread: Effect.fn("AcpAdapterV2.forkThread")( function* (forkInput) { - if (!capabilities.threads.canForkThread) { - return yield* new ProviderAdapterProtocolError({ - driver, - detail: "ACP driver did not negotiate session/fork", - }); - } - if (forkInput.providerTurnId !== undefined) { - return yield* new ProviderAdapterProtocolError({ - driver, - detail: "ACP session/fork can only fork the current session head", - }); - } - const sourceSessionId = nativeThreadId(driver, forkInput.sourceProviderThread); - const forked = yield* runtime.forkSession(sourceSessionId); - yield* Ref.set(activeSessionId, forked.sessionId); - yield* Ref.set(activeSessionSetup, forked); - yield* Ref.set(activeSelection, null); - const now = yield* DateTime.now; - return makeProviderThread({ - driver, - providerInstanceId: options.instanceId, - idAllocator, - appThreadId: forkInput.targetThreadId, - providerSessionId: input.providerSessionId, - nativeThreadId: forked.sessionId, - ...(forkInput.ownerNodeId === undefined - ? {} - : { ownerNodeId: forkInput.ownerNodeId }), - forkedFrom: { - providerThreadId: forkInput.sourceProviderThread.id, - ...(forkInput.providerTurnId === undefined - ? {} - : { providerTurnId: forkInput.providerTurnId }), - }, - now, - }); + return yield* runtimeTransitionPermit.withPermit( + Effect.gen(function* () { + yield* awaitRuntimeTeardown(); + yield* restartRuntimeAfterTeardownIfRequired(); + if (!capabilities.threads.canForkThread) { + return yield* new ProviderAdapterProtocolError({ + driver, + detail: "ACP driver did not negotiate session/fork", + }); + } + if (forkInput.providerTurnId !== undefined) { + return yield* new ProviderAdapterProtocolError({ + driver, + detail: "ACP session/fork can only fork the current session head", + }); + } + const sourceSessionId = nativeThreadId(driver, forkInput.sourceProviderThread); + const forked = yield* runtime.forkSession(sourceSessionId); + yield* Ref.set(activeSessionId, forked.sessionId); + yield* Ref.set(activeSessionSetup, forked); + yield* Ref.set(activeSelection, null); + const now = yield* DateTime.now; + return makeProviderThread({ + driver, + providerInstanceId: options.instanceId, + idAllocator, + appThreadId: forkInput.targetThreadId, + providerSessionId: input.providerSessionId, + nativeThreadId: forked.sessionId, + ...(forkInput.ownerNodeId === undefined + ? {} + : { ownerNodeId: forkInput.ownerNodeId }), + forkedFrom: { + providerThreadId: forkInput.sourceProviderThread.id, + ...(forkInput.providerTurnId === undefined + ? {} + : { providerTurnId: forkInput.providerTurnId }), + }, + now, + }); + }), + ); }, (effect, forkInput) => effect.pipe( diff --git a/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts index 1a65a39aab8..27f8a603801 100644 --- a/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts @@ -1,9 +1,25 @@ import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; import type * as EffectAcpSchema from "effect-acp/schema"; import { ProviderAdapterV2RuntimePolicy } from "../ProviderAdapter.ts"; -import { AcpProviderCapabilitiesV2, acpPermissionDisposition } from "./AcpAdapterV2.ts"; -import { GrokProviderCapabilitiesV2 } from "./GrokAdapterV2.ts"; +import { + AcpProviderCapabilitiesV2, + acpCompletedTurnShouldTerminalizeTool, + acpPermissionDisposition, + acpRootSessionUpdateIngestsOutput, + acpRootTurnCompletionDrainMs, + acpRootTurnHasIngestedOutput, + acpRootTurnIsIdle, + acpRootTurnSettleDebounceMs, + acpRootTurnShouldRearmRecoveryTimers, + acpSupportsImagePrompts, +} from "./AcpAdapterV2.ts"; +import { + makeGrokAcpAdapterFlavor, + GrokProviderCapabilitiesV2, + type GrokAdapterV2Options, +} from "./GrokAdapterV2.ts"; function permissionRequest( kind: EffectAcpSchema.ToolKind, @@ -37,7 +53,221 @@ function runtimePolicy(input: { }); } +describe("acpRootTurnSettleDebounceMs", () => { + it("keeps the historical debounce constant for re-enable experiments", () => { + assert.equal(acpRootTurnSettleDebounceMs, 2_000); + }); +}); + +describe("acpRootTurnCompletionDrainMs", () => { + it("gives trailing root chunks a short landing window", () => { + assert.equal(acpRootTurnCompletionDrainMs, 100); + }); +}); + +describe("acpRootSessionUpdateIngestsOutput", () => { + const sessionId = "session-1"; + + it("ignores empty assistant chunks used as Grok keepalives", () => { + assert.isFalse( + acpRootSessionUpdateIngestsOutput({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "" }, + }, + }), + ); + }); + + it("accepts non-empty assistant and reasoning chunks", () => { + assert.isTrue( + acpRootSessionUpdateIngestsOutput({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "hello" }, + }, + }), + ); + assert.isTrue( + acpRootSessionUpdateIngestsOutput({ + sessionId, + update: { + sessionUpdate: "agent_thought_chunk", + content: { type: "text", text: "thinking" }, + }, + }), + ); + }); + + it("accepts tool and plan updates", () => { + assert.isTrue( + acpRootSessionUpdateIngestsOutput({ + sessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-1", + title: "Read", + kind: "read", + status: "pending", + }, + }), + ); + assert.isTrue( + acpRootSessionUpdateIngestsOutput({ + sessionId, + update: { + sessionUpdate: "plan", + entries: [{ content: "Step 1", status: "pending", priority: "medium" }], + }, + }), + ); + }); +}); + +describe("acpRootTurnHasIngestedOutput", () => { + const empty = { + assistant: { current: null, nextSegment: 0 }, + reasoning: { current: null, nextSegment: 0 }, + tools: new Map(), + plan: null, + } as const; + + it("is false before any root turn items land", () => { + assert.isFalse(acpRootTurnHasIngestedOutput(empty)); + }); + + it("is true once assistant segments have streamed", () => { + assert.isTrue( + acpRootTurnHasIngestedOutput({ + ...empty, + assistant: { current: null, nextSegment: 1 }, + }), + ); + }); +}); + +describe("acpRootTurn recovery timer re-arm", () => { + it("re-arms idle settle after pending clears on active turns", () => { + assert.isTrue(acpRootTurnShouldRearmRecoveryTimers({ finalized: false, interrupted: false })); + }); + + it("skips re-arm when the turn is already terminal", () => { + assert.isFalse(acpRootTurnShouldRearmRecoveryTimers({ finalized: true, interrupted: false })); + assert.isFalse(acpRootTurnShouldRearmRecoveryTimers({ finalized: false, interrupted: true })); + }); +}); + +describe("acpRootTurnIsIdle", () => { + const quiet = { + finalized: false, + interrupted: false, + assistantStreamOpen: false, + reasoningStreamOpen: false, + hasRunningTool: false, + hasPendingRuntimeRequest: false, + hasToolHistory: false, + hasRunningSubagent: false, + hasOutput: true, + } as const; + + it("is false while assistant text is still streaming", () => { + assert.isFalse(acpRootTurnIsIdle({ ...quiet, assistantStreamOpen: true })); + }); + + it("is false while a tool is running", () => { + assert.isFalse(acpRootTurnIsIdle({ ...quiet, hasRunningTool: true })); + }); + + it("is false after tool history (prompt RPC owns terminalization)", () => { + assert.isFalse(acpRootTurnIsIdle({ ...quiet, hasToolHistory: true })); + }); + + it("is false while a native subagent task is still running", () => { + assert.isFalse(acpRootTurnIsIdle({ ...quiet, hasRunningSubagent: true })); + }); + + it("is false when only reasoning or tools have streamed", () => { + assert.isFalse(acpRootTurnIsIdle({ ...quiet, hasOutput: false })); + }); + + it("is false for assistant-only quiet (preamble-before-tools must not settle)", () => { + assert.isFalse(acpRootTurnIsIdle(quiet)); + }); + + it("is false when tools finished and root is quiet (no speculative multi-wave settle)", () => { + assert.isFalse( + acpRootTurnIsIdle({ + ...quiet, + hasToolHistory: true, + hasRunningTool: false, + }), + ); + }); +}); + describe("GrokAdapterV2 capabilities", () => { + it("wires hard Stop teardown but soft non-Stop interrupts in the constructor flavor", () => { + const flavor = makeGrokAcpAdapterFlavor({ + makeRuntime: () => Effect.never, + } as unknown as GrokAdapterV2Options); + + assert.isFalse(flavor.interruptPromptOnCancel); + // User Stop (requestRuntimeRestart) keeps the hard process-group kill and + // respawn: Grok cancel is detach-and-continue, so only a process kill + // stops the work. + assert.isTrue(flavor.restartRuntimeAfterInterrupt); + assert.isTrue(flavor.terminateRuntimeProcessGroupOnInterrupt); + // Non-Stop interrupts (steering, restart_active) reuse the process and + // session; the cancelled work backgrounds and the model decides its fate. + assert.isUndefined(flavor.restartRuntimeOnEveryInterrupt); + assert.isTrue(flavor.preserveRuntimeOnSettledInterrupt); + }); + + it("terminalizes only foreground tools under the actual Grok flavor", () => { + const flavor = makeGrokAcpAdapterFlavor({ + makeRuntime: () => Effect.never, + } as unknown as GrokAdapterV2Options); + const foreground = { + toolCallId: "foreground-1", + title: "Terminal", + status: "inProgress" as const, + data: { + rawInput: { command: "true" }, + rawOutput: { type: "Bash", exit_code: 0 }, + }, + }; + const monitor = { + toolCallId: "monitor-1", + title: "Monitor", + status: "inProgress" as const, + data: { + rawInput: { variant: "Monitor", command: "sleep 30" }, + rawOutput: { + type: "Monitor", + taskId: "019f44b8-8e98-7c80-a40e-df1e26a5f9e3", + }, + }, + }; + const subagent = { + toolCallId: "subagent-1", + title: "Task", + status: "inProgress" as const, + data: { + rawInput: { + description: "Inspect interrupt handling", + prompt: "Review the adapter.", + subagent_type: "generalPurpose", + }, + }, + }; + + assert.isTrue(acpCompletedTurnShouldTerminalizeTool(foreground, flavor)); + assert.isFalse(acpCompletedTurnShouldTerminalizeTool(monitor, flavor)); + assert.isFalse(acpCompletedTurnShouldTerminalizeTool(subagent, flavor)); + }); + it("keeps optional protocol features conservative until a flavor or handshake confirms them", () => { assert.isFalse(AcpProviderCapabilitiesV2.sessions.supportsModelSwitchInSession); assert.isFalse(AcpProviderCapabilitiesV2.sessions.supportsRuntimeModeSwitchInSession); @@ -45,6 +275,27 @@ describe("GrokAdapterV2 capabilities", () => { assert.isFalse(AcpProviderCapabilitiesV2.tools.supportsMcpTools); }); + it("overrides ACP image capability false so screenshot attachments can prompt", () => { + // Handshake alone would refuse attachments (Grok advertises image:false). + assert.isFalse( + acpSupportsImagePrompts({ + negotiatedImage: false, + }), + ); + // Flavor override unblocks image content blocks for Grok. + assert.isTrue( + acpSupportsImagePrompts({ + flavorSupportsImagePrompts: true, + negotiatedImage: false, + }), + ); + assert.isTrue( + acpSupportsImagePrompts({ + negotiatedImage: true, + }), + ); + }); + it("declares Grok Task envelopes as native subagents", () => { assert.isFalse(GrokProviderCapabilitiesV2.threads.canForkThread); assert.isTrue(GrokProviderCapabilitiesV2.subagents.supportsSubagents); diff --git a/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.ts index c4b652aa9f6..2d1ec669921 100644 --- a/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.ts @@ -21,17 +21,26 @@ import { resolveGrokAcpBaseModelId, } from "../../provider/acp/GrokAcpSupport.ts"; import { + extractXAiAcpBackgroundToolMutation, + extractXAiAcpSubagentEndNotice, + extractXAiAcpSubagentUpdate, extractXAiAskUserQuestionIdentity, extractXAiAskUserQuestions, - extractXAiAcpSubagentUpdate, + extractXAiBackgroundTaskCompletion, + extractXAiKilledBackgroundTasks, + extractXAiMonitorTaskId, + isXAiPersistentMonitor, makeXAiAskUserQuestionCancelledResponse, makeXAiAskUserQuestionResponse, + normalizeXAiAcpToolCallState, + registerXAiBackgroundTaskTracking, XAiAskUserQuestionRequest, } from "../../provider/acp/XAiAcpExtension.ts"; import { mergeProviderInstanceEnvironment } from "../../provider/ProviderInstanceEnvironment.ts"; import * as AcpSessionRuntime from "../../provider/acp/AcpSessionRuntime.ts"; import { ProviderEventLoggers } from "../../provider/Layers/ProviderEventLoggers.ts"; import { IdAllocatorV2 } from "../IdAllocator.ts"; +import { ProviderContinuationRequests } from "../ProviderContinuationRequests.ts"; import { ProviderAdapterV2 } from "../ProviderAdapter.ts"; import { ProviderAdapterDriverCreateError, @@ -41,6 +50,7 @@ import { import { AcpProviderCapabilitiesV2, makeAcpAdapterV2, + type AcpAdapterV2ExtensionContext, type AcpAdapterV2Flavor, type AcpAdapterV2RuntimeInput, } from "./AcpAdapterV2.ts"; @@ -89,6 +99,7 @@ export interface GrokAdapterV2Options { readonly idAllocator: IdAllocatorV2["Service"]; readonly serverConfig: ServerConfig["Service"]; readonly nativeLogging?: Parameters[0]["nativeLogging"]; + readonly continuationRequests?: Parameters[0]["continuationRequests"]; readonly makeRuntime?: ( input: AcpAdapterV2RuntimeInput, ) => Effect.Effect< @@ -102,7 +113,16 @@ export interface GrokAdapterV2Options { export const registerGrokAcpExtensions: NonNullable = ({ runtime, requestUserInput, + applyBackgroundTaskMutation, }) => + registerXAiBackgroundTaskTracking(runtime, applyBackgroundTaskMutation).pipe( + Effect.andThen(registerGrokAskUserQuestionExtensions({ runtime, requestUserInput })), + ); + +const registerGrokAskUserQuestionExtensions = ({ + runtime, + requestUserInput, +}: Pick) => Effect.forEach( ["x.ai/ask_user_question", "_x.ai/ask_user_question"] as const, (method) => @@ -116,37 +136,78 @@ export const registerGrokAcpExtensions: NonNullable - answers === null - ? makeXAiAskUserQuestionCancelledResponse() - : makeXAiAskUserQuestionResponse(params, answers), + Effect.flatMap(({ acknowledgeNativeResponse, answers }) => + Effect.succeed( + answers === null + ? makeXAiAskUserQuestionCancelledResponse() + : makeXAiAskUserQuestionResponse(params, answers), + ).pipe(Effect.tap(() => acknowledgeNativeResponse)), ), ); }), { discard: true }, ); -export function makeGrokAdapterV2(options: GrokAdapterV2Options) { - const flavor: AcpAdapterV2Flavor = { +export function makeGrokAcpAdapterFlavor(options: GrokAdapterV2Options): AcpAdapterV2Flavor { + return { driver: GROK_PROVIDER, capabilities: GrokProviderCapabilitiesV2, + // Idle settle over-settled preamble-before-tools turns and cancelled the + // prompt while Grok continued, freezing T3 projection mid-turn. + settleRootTurnWhenIdle: false, + interruptPromptOnCancel: false, + // User Stop (requestRuntimeRestart) still hard-kills the process group + // and respawns: Grok `session/cancel` is detach-and-continue (the CLI + // re-runs cancelled foreground work as a background task no client RPC + // can kill, E3 harness 2026-07-18), so only a process kill truly stops + // work. Non-Stop interrupts (mid-prompt steering, restart_active) omit + // requestRuntimeRestart and stay soft: the session survives in the same + // process, cancelled work backgrounds, and the model itself decides via + // kill_command_or_subagent whether it keeps running. + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + // Steering restarts on a settled turn additionally skip session/cancel so + // fire-and-forget subagents survive the steer (E1 harness confirmed the + // Grok CLI accepts a concurrent session/prompt in that state). + preserveRuntimeOnSettledInterrupt: true, + // Grok ACP initialize reports promptCapabilities.image:false but the agent + // still accepts image content blocks (verified with real screenshots). + supportsImagePrompts: true, resolveModelId: (selection) => resolveGrokAcpBaseModelId(selection.model), makeRuntime: options.makeRuntime ?? ((input) => makeGrokAcpRuntime({ ...input, + interruptPromptOnCancel: input.interruptPromptOnCancel ?? false, grokSettings: options.settings, environment: options.environment, childProcessSpawner: options.childProcessSpawner, })), registerExtensions: registerGrokAcpExtensions, extractSubagentUpdate: extractXAiAcpSubagentUpdate, + extractSubagentEndNotice: extractXAiAcpSubagentEndNotice, + normalizeToolCall: normalizeXAiAcpToolCallState, + extractBackgroundTaskId: extractXAiMonitorTaskId, + extractBackgroundToolMutation: extractXAiAcpBackgroundToolMutation, + extractBackgroundTaskCompletion: (toolCall) => [ + ...extractXAiBackgroundTaskCompletion(toolCall), + ...extractXAiKilledBackgroundTasks(toolCall), + ], + isPersistentBackgroundTool: isXAiPersistentMonitor, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, ...(options.assertComplete === undefined ? {} : { assertComplete: options.assertComplete }), }; +} + +export function makeGrokAdapterV2(options: GrokAdapterV2Options) { + const flavor = makeGrokAcpAdapterFlavor(options); return makeAcpAdapterV2({ instanceId: options.instanceId, flavor, @@ -155,6 +216,9 @@ export function makeGrokAdapterV2(options: GrokAdapterV2Options) { idAllocator: options.idAllocator, serverConfig: options.serverConfig, ...(options.nativeLogging === undefined ? {} : { nativeLogging: options.nativeLogging }), + ...(options.continuationRequests === undefined + ? {} + : { continuationRequests: options.continuationRequests }), }); } @@ -179,6 +243,7 @@ export const GrokAdapterV2Driver: ProviderAdapterDriver makeNativeLogger({ nativeEventLogger: providerEventLoggers.native, @@ -231,6 +297,7 @@ export const layer: Layer.Layer< const idAllocator = yield* IdAllocatorV2; const providerEventLoggers = yield* ProviderEventLoggers; const serverConfig = yield* ServerConfig; + const continuationRequests = yield* ProviderContinuationRequests; const makeNativeLogger = yield* makeAcpNativeLoggerFactory(); return makeGrokAdapterV2({ instanceId: GROK_DEFAULT_INSTANCE_ID, @@ -241,6 +308,7 @@ export const layer: Layer.Layer< fileSystem, idAllocator, serverConfig, + continuationRequests, nativeLogging: (threadId) => makeNativeLogger({ nativeEventLogger: providerEventLoggers.native, diff --git a/apps/server/src/orchestration-v2/EffectWorker.test.ts b/apps/server/src/orchestration-v2/EffectWorker.test.ts index 5b0e4d26203..583e16e2ec7 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.test.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.test.ts @@ -17,7 +17,11 @@ import * as Ref from "effect/Ref"; import { CheckpointRollbackServiceV2 } from "./CheckpointRollbackService.ts"; import type { OrchestrationEffectV2 } from "./EffectOutbox.ts"; -import { executorLayer, OrchestrationEffectExecutorV2 } from "./EffectWorker.ts"; +import { + executorLayer, + isNonRetryableProviderTurnControlFailure, + OrchestrationEffectExecutorV2, +} from "./EffectWorker.ts"; import { RunFinalizationService } from "./RunFinalizationService.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; import { ProviderTurnControlServiceV2 } from "./ProviderTurnControlService.ts"; @@ -133,6 +137,40 @@ function makeExecutorLayer(input: { return executorLayer.pipe(Layer.provide(dependencies)); } +it("does not retry pure interrupt races where the turn is already gone", () => { + assert.isTrue( + isNonRetryableProviderTurnControlFailure( + "provider-turn.interrupt", + "ProviderAdapterInterruptError: ... ACP provider turn provider-turn:x is not active", + ), + ); + assert.isTrue( + isNonRetryableProviderTurnControlFailure( + "provider-turn.interrupt", + "Provider session provider-session:x is not active.", + ), + ); + // Restart is compound (interrupt + detach + start). Do not swallow start failures. + assert.isFalse( + isNonRetryableProviderTurnControlFailure( + "provider-turn.restart", + "Provider session provider-session:x is not active.", + ), + ); + assert.isFalse( + isNonRetryableProviderTurnControlFailure( + "provider-turn.start", + "Provider session provider-session:x is not active.", + ), + ); + assert.isFalse( + isNonRetryableProviderTurnControlFailure( + "provider-turn.interrupt", + "ACP hard teardown failed unexpectedly; the session is poisoned", + ), + ); +}); + it.effect("detaches a handed-off session only after the old turn terminalizes", () => Effect.gen(function* () { const now = yield* DateTime.now; diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 612c26c44cf..b543edeff24 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -25,6 +25,30 @@ export class OrchestrationEffectExecutionError extends Schema.TaggedErrorClass= maxAttempts + // Prefer succeed for terminal interrupt races so the outbox does not + // keep a failed interrupt around; fail only when we must not retry. + const updated = nonRetryable + ? yield* outbox.succeed({ effectId: effect.id, workerId }) + : effect.attemptCount >= maxAttempts ? yield* outbox.fail({ effectId: effect.id, workerId, error }) : yield* outbox.retry({ effectId: effect.id, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 9122f8dd79c..d9e1c479d12 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -573,6 +573,7 @@ function activeLocalTurnItems( item, runs: projection.runs, attempts: projection.attempts, + items: projection.turnItems, }), ) .map((item, position) => ({ @@ -668,6 +669,7 @@ function visibleTurnItemsThroughRun(input: { isOrchestrationV2SupersededInterrupt({ item, attempts: input.sourceProjection.attempts, + items: input.sourceProjection.turnItems, }) ) { return false; diff --git a/apps/server/src/orchestration-v2/ProviderAdapter.ts b/apps/server/src/orchestration-v2/ProviderAdapter.ts index e24d221231a..d8f3595bcf3 100644 --- a/apps/server/src/orchestration-v2/ProviderAdapter.ts +++ b/apps/server/src/orchestration-v2/ProviderAdapter.ts @@ -407,6 +407,8 @@ export interface ProviderAdapterV2SteerInput { export interface ProviderAdapterV2InterruptInput { readonly providerThread: OrchestrationV2ProviderThread; readonly providerTurnId: ProviderTurnId; + /** When true, the next `startTurn` may respawn the provider runtime (Grok Stop recovery). */ + readonly requestRuntimeRestart?: boolean; } export interface ProviderAdapterV2RuntimeRequestResponseInput { diff --git a/apps/server/src/orchestration-v2/ProviderContinuationRequests.ts b/apps/server/src/orchestration-v2/ProviderContinuationRequests.ts index b972999e011..a5d256093b8 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationRequests.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationRequests.ts @@ -2,6 +2,7 @@ import { ProviderDriverKind, ProviderThreadId, ThreadId } from "@t3tools/contrac import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; export interface ProviderContinuationRequest { @@ -9,6 +10,9 @@ export interface ProviderContinuationRequest { readonly providerThreadId: ProviderThreadId; readonly driver: ProviderDriverKind; readonly detail: string | null; + readonly dispatchIfCurrent?: ( + effect: Effect.Effect, + ) => Effect.Effect, E, R>; } /** diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts new file mode 100644 index 00000000000..79b942709a1 --- /dev/null +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -0,0 +1,193 @@ +import { assert, describe, it } from "@effect/vitest"; +import { + ProviderDriverKind, + ProviderThreadId, + ThreadId, + type OrchestrationV2ThreadProjection, +} from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; +import * as Semaphore from "effect/Semaphore"; + +import { layer as idAllocatorLayer } from "./IdAllocator.ts"; +import { + type ProviderContinuationRequest, + ProviderContinuationRequests, + layer as continuationRequestsLayer, +} from "./ProviderContinuationRequests.ts"; +import { workerLive } from "./ProviderContinuationService.ts"; +import { ThreadManagementService } from "./ThreadManagementService.ts"; + +const threadId = ThreadId.make("thread-provider-continuation"); +const providerThreadId = ProviderThreadId.make("provider-thread-continuation"); +const driver = ProviderDriverKind.make("continuation-test"); +const projection = { + thread: { archivedAt: null }, + messages: [], +} as unknown as OrchestrationV2ThreadProjection; + +const request = ( + dispatchIfCurrent?: ProviderContinuationRequest["dispatchIfCurrent"], + detail: string | null = null, +): ProviderContinuationRequest => ({ + threadId, + providerThreadId, + driver, + detail, + ...(dispatchIfCurrent === undefined ? {} : { dispatchIfCurrent }), +}); + +const makeGuard = Effect.fnUntraced(function* (completed?: Deferred.Deferred) { + const generation = yield* Ref.make(0); + const permit = yield* Semaphore.make(1); + const capture = Effect.gen(function* () { + const captured = yield* Ref.updateAndGet(generation, (value) => value + 1); + return (effect: Effect.Effect) => + permit + .withPermit( + Effect.gen(function* () { + if ((yield* Ref.get(generation)) !== captured) return Option.none(); + return Option.some(yield* effect); + }), + ) + .pipe( + completed === undefined + ? (effect) => effect + : Effect.ensuring(Deferred.succeed(completed, undefined)), + ); + }); + return { + capture, + invalidate: permit.withPermit(Ref.update(generation, (value) => value + 1)), + }; +}); + +function testLayer(input: { + readonly dispatched: Queue.Queue; + readonly getThreadProjection: () => Effect.Effect; +}) { + const threads = Layer.mock(ThreadManagementService)({ + getThreadProjection: input.getThreadProjection, + dispatch: (command) => Queue.offer(input.dispatched, command).pipe(Effect.as({} as never)), + }); + const worker = workerLive.pipe( + Layer.provide(Layer.mergeAll(idAllocatorLayer, continuationRequestsLayer, threads)), + ); + return Layer.merge(continuationRequestsLayer, worker); +} + +describe("ProviderContinuationService", () => { + it.effect("dispatches a current request exactly once", () => { + return Effect.gen(function* () { + const dispatched = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests; + yield* requests.offer(request()); + yield* Queue.take(dispatched); + yield* Effect.yieldNow; + assert.isTrue(Option.isNone(yield* Queue.poll(dispatched))); + }).pipe( + Effect.provide( + testLayer({ dispatched, getThreadProjection: () => Effect.succeed(projection) }), + ), + Effect.scoped, + ); + }); + }); + + it.effect("drops a request invalidated before dispatch", () => { + return Effect.gen(function* () { + const dispatched = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests; + const guard = yield* makeGuard(); + const dispatchIfCurrent = yield* guard.capture; + yield* guard.invalidate; + yield* requests.offer(request(dispatchIfCurrent)); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + assert.isTrue(Option.isNone(yield* Queue.poll(dispatched))); + }).pipe( + Effect.provide( + testLayer({ dispatched, getThreadProjection: () => Effect.succeed(projection) }), + ), + Effect.scoped, + ); + }); + }); + + it.effect("drops a request invalidated while projection is blocked", () => { + return Effect.gen(function* () { + const dispatched = yield* Queue.unbounded(); + const projectionEntered = yield* Deferred.make(); + const releaseProjection = yield* Deferred.make(); + const guardCompleted = yield* Deferred.make(); + yield* Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests; + const guard = yield* makeGuard(guardCompleted); + const dispatchIfCurrent = yield* guard.capture; + yield* requests.offer(request(dispatchIfCurrent)); + yield* Deferred.await(projectionEntered); + yield* guard.invalidate; + yield* Deferred.succeed(releaseProjection, undefined); + yield* Deferred.await(guardCompleted); + assert.isTrue(Option.isNone(yield* Queue.poll(dispatched))); + }).pipe( + Effect.provide( + testLayer({ + dispatched, + getThreadProjection: () => + Deferred.succeed(projectionEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseProjection)), + Effect.as(projection), + ), + }), + ), + Effect.scoped, + ); + }); + }); + + it.effect("does not revive an old request when a later generation is current", () => { + return Effect.gen(function* () { + const dispatched = yield* Queue.unbounded(); + const firstProjectionEntered = yield* Deferred.make(); + const releaseFirstProjection = yield* Deferred.make(); + let projectionCalls = 0; + yield* Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests; + const guard = yield* makeGuard(); + const first = yield* guard.capture; + yield* requests.offer(request(first, "A")); + yield* Deferred.await(firstProjectionEntered); + const second = yield* guard.capture; + yield* requests.offer(request(second, "B")); + yield* Deferred.succeed(releaseFirstProjection, undefined); + const command = yield* Queue.take(dispatched); + assert.equal((command as { readonly text?: unknown }).text, "B"); + yield* Effect.yieldNow; + assert.isTrue(Option.isNone(yield* Queue.poll(dispatched))); + }).pipe( + Effect.provide( + testLayer({ + dispatched, + getThreadProjection: () => { + projectionCalls += 1; + return projectionCalls === 1 + ? Deferred.succeed(firstProjectionEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseFirstProjection)), + Effect.as(projection), + ) + : Effect.succeed(projection); + }, + }), + ), + Effect.scoped, + ); + }); + }); +}); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.ts index 0ad60d2eb94..14aa26bfbd9 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.ts @@ -32,14 +32,20 @@ export const workerLive = Layer.effectDiscard( threadId: request.threadId, providerThreadId: request.providerThreadId, }); + // Still run the guard so adapters can clear sticky continuationRequested. + if (request.dispatchIfCurrent !== undefined) { + yield* request.dispatchIfCurrent(Effect.void); + } return; } + // The ordinal is display metadata only: allocate.message appends a + // random UUID, so a stale projection read here cannot collide ids. const messageId = yield* ids.allocate.message({ threadId: request.threadId, ordinal: projection.messages.length + 1, }); const commandId = CommandId.make(`provider-continuation:${messageId}`); - yield* threads.dispatch({ + const dispatch = threads.dispatch({ type: "message.dispatch", commandId, threadId: request.threadId, @@ -50,6 +56,11 @@ export const workerLive = Layer.effectDiscard( createdBy: "agent", creationSource: "provider", }); + if (request.dispatchIfCurrent === undefined) { + yield* dispatch; + return; + } + yield* request.dispatchIfCurrent(dispatch); }, ); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index abc70ee5b7c..445ed40e5bc 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -1233,6 +1233,99 @@ it.effect( }), ); +it.effect("ProviderSessionManagerV2 does not apply a stale idle pin to a replacement session", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const firstCheck = yield* Ref.make(true); + const checkEntered = yield* Deferred.make(); + const checkGate = yield* Deferred.make(); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSinkV2; + const idAllocator = yield* IdAllocatorV2; + const manager = yield* ProviderSessionManagerV2; + const now = yield* DateTime.now; + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-stale-pin", + projectId: yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-stale-pin", + }), + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + + // Park the first idle fiber inside an uninterruptible pending-work probe. + yield* TestClock.adjust("1 second"); + yield* Deferred.await(checkEntered); + + // close removes the map entry first, then waits to interrupt the idle + // fiber (still uninterruptible). That window lets a replacement open + // under the same providerSessionId before the stale probe finishes. + const closeFiber = yield* manager.close(providerSessionId).pipe(Effect.forkDetach); + for (let i = 0; i < 20; i += 1) { + yield* Effect.yieldNow; + } + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + assert.equal((yield* Ref.get(state)).openCount, 2); + + // Stale probe reports pending work against the old runtime; the pin + // stamp must no-op on the replacement (runtime / generation mismatch). + yield* Deferred.succeed(checkGate, undefined); + yield* Fiber.join(closeFiber); + for (let i = 0; i < 10; i += 1) { + yield* Effect.yieldNow; + } + + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + assert.equal((yield* Ref.get(state)).closeCount, 1); + + // Replacement has no pending background work. After one idle window it + // must release. A stale pin stamp would have deferred release until + // maxIdlePinMs. + yield* TestClock.adjust("1 second"); + yield* Effect.yieldNow; + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + assert.equal((yield* Ref.get(state)).closeCount, 2); + }); + + yield* effect.pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 1000, + maxIdlePinMs: 60_000, + hasPendingBackgroundWork: Effect.uninterruptible( + Effect.gen(function* () { + if (yield* Ref.getAndSet(firstCheck, false)) { + yield* Deferred.succeed(checkEntered, undefined); + yield* Deferred.await(checkGate); + return true; + } + return false; + }), + ), + }), + ), + ); + }), +); + it.effect( "ProviderSessionManagerV2 keeps active sessions alive until the provider turn terminates", () => diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 48584ca7b2b..dc826ecc5d2 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -605,31 +605,47 @@ export const layerWithOptions = ( ) { return; } + // Capture runtime identity before yielding: a replacement session + // can reuse the same providerSessionId while this fiber is parked. + const probedRuntime = entry.runtime; const hasPendingWork = - entry.runtime.hasPendingBackgroundWork === undefined + probedRuntime.hasPendingBackgroundWork === undefined ? false - : yield* entry.runtime.hasPendingBackgroundWork.pipe( + : yield* probedRuntime.hasPendingBackgroundWork.pipe( Effect.catchCause(() => Effect.succeed(false)), ); if (hasPendingWork) { const now = yield* Clock.currentTimeMillis; const pinnedSinceMs = entry.pinnedSinceMs ?? now; if (now - pinnedSinceMs < maxIdlePinMs) { - yield* Effect.logInfo("orchestration-v2.driver-session.idle-release-deferred", { - providerSessionId: input.providerSessionId, - pinnedForMs: now - pinnedSinceMs, - }); - yield* Ref.update(sessions, (latest) => { + const shouldContinuePin = yield* Ref.modify(sessions, (latest) => { const latestEntry = latest.get(key); - if (latestEntry === undefined || latestEntry.busyCount > 0) { - return latest; + if ( + latestEntry === undefined || + latestEntry.busyCount > 0 || + latestEntry.idleGeneration !== input.generation || + latestEntry.runtime !== probedRuntime + ) { + return [false, latest] as const; } const updated = new Map(latest); updated.set(key, { ...latestEntry, pinnedSinceMs }); - return updated; + return [true, updated] as const; }); - yield* scheduleIdleReleaseInternal(input.providerSessionId); - return; + if (!shouldContinuePin) { + // Generation or runtime advanced while we probed pending work; + // the current owner of the entry owns idle release. + return; + } + yield* Effect.logInfo("orchestration-v2.driver-session.idle-release-deferred", { + providerSessionId: input.providerSessionId, + pinnedForMs: now - pinnedSinceMs, + }); + // Re-check on this fiber after another idle window. Do not call + // scheduleIdleReleaseInternal: that cancels entry.idleFiber, which + // is this fiber, and can self-deadlock on Fiber.interrupt. + yield* Effect.sleep(Duration.millis(idleTimeoutMs)); + return yield* releaseIfStillIdle(input); } yield* Effect.logWarning("orchestration-v2.driver-session.idle-release-pin-expired", { providerSessionId: input.providerSessionId, diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts index 4ee71642619..0ce3ce30fb7 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts @@ -129,6 +129,28 @@ export const layer: Layer.Layer< } const session = yield* sessions.get(input.providerSessionId); if (Option.isNone(session)) { + // Interrupt/restart against a already-released session must not fail + // the durable effect (and retry 5x). The turn may still look running + // in projection until recovery/finalization; there is no live adapter + // to interrupt. + if (input.operation === "interrupt" || input.operation === "restart") { + yield* Effect.logWarning( + "Provider interrupt/restart found no live session; treating as already stopped", + { + threadId: input.threadId, + operation: input.operation, + providerSessionId: input.providerSessionId, + providerTurnId: input.providerTurnId, + providerTurnStatus: providerTurn.status, + }, + ); + return { + projection, + providerThread: interruptProviderThread, + providerTurn, + session: Option.none(), + }; + } return yield* new ProviderTurnControlError({ threadId: input.threadId, operation: input.operation, @@ -147,6 +169,7 @@ export const layer: Layer.Layer< yield* loaded.session.value.interruptTurn({ providerThread: loaded.providerThread, providerTurnId: loaded.providerTurn.id, + requestRuntimeRestart: true, }); }).pipe( Effect.mapError((cause) => @@ -163,13 +186,28 @@ export const layer: Layer.Layer< interruptAndAwaitTerminal: (input) => Effect.gen(function* () { const loaded = yield* load({ ...input, operation: "restart" }); - if (Option.isSome(loaded.session)) { - yield* loaded.session.value.interruptTurn({ - providerThread: loaded.providerThread, - providerTurnId: loaded.providerTurn.id, - }); + if (Option.isNone(loaded.session)) { + // No live adapter: nothing can emit a terminal provider-turn update + // from interrupt. Do not poll for projection terminalization or the + // restart effect stalls; let detach/start proceed. + if (loaded.providerTurn.status === "running") { + yield* Effect.logWarning( + "Provider restart interrupt skipped; no live session for a still-projected running turn", + { + threadId: input.threadId, + providerSessionId: input.providerSessionId, + providerTurnId: input.providerTurnId, + }, + ); + } + return; } + yield* loaded.session.value.interruptTurn({ + providerThread: loaded.providerThread, + providerTurnId: loaded.providerTurn.id, + }); + for (let remaining = 1_000; remaining > 0; remaining -= 1) { const projection = yield* projections.getThreadProjection(input.threadId); const providerTurn = projection.providerTurns.find( diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index b8bd2b2fe84..693a00b57dd 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -439,6 +439,23 @@ export const layer: Layer.Layer< }), Effect.catchCause(() => Effect.succeed(false)), ), + hasUnpairedRunInterruptRequest: () => + projectionStore.getThreadProjection(projection.thread.id).pipe( + Effect.map((current) => { + const requestId = idAllocator.derive.runSignalTurnItem({ + runId: run.id, + signal: "interrupt-request", + }); + const resultId = idAllocator.derive.runSignalTurnItem({ + runId: run.id, + signal: "interrupt-result", + }); + const hasRequest = current.turnItems.some((item) => item.id === requestId); + const hasResult = current.turnItems.some((item) => item.id === resultId); + return hasRequest && !hasResult; + }), + Effect.catchCause(() => Effect.succeed(false)), + ), message: { messageId: message.id, text: diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index aaa2384d306..2822f60851b 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -31,7 +31,7 @@ import * as Stream from "effect/Stream"; import { ServerSettingsService } from "../serverSettings.ts"; import { CheckpointServiceV2 } from "./CheckpointService.ts"; import { EventSinkV2 } from "./EventSink.ts"; -import { layer as idAllocatorLayer } from "./IdAllocator.ts"; +import { IdAllocatorV2, layer as idAllocatorLayer } from "./IdAllocator.ts"; import type { ProviderAdapterV2Event, ProviderAdapterV2SessionRuntime } from "./ProviderAdapter.ts"; import { ProviderEventIngestorV2 } from "./ProviderEventIngestor.ts"; import { @@ -542,6 +542,194 @@ it.effect("does not pin ingestion on background items when the root turn is inte }), ); +it.effect("omits run_interrupt_result when a superseding attempt already owns the run", () => + Effect.gen(function* () { + const written = yield* captureInterruptTerminalTurnItems({ + key: "steer-supersede", + shouldFinalizeRun: () => Effect.succeed(false), + }); + assert.deepEqual( + written.map((item) => item.type), + [], + ); + }), +); + +it.effect("emits run_interrupt_result when superseded attempt still has a hard-stop request", () => + Effect.gen(function* () { + const written = yield* captureInterruptTerminalTurnItems({ + key: "stop-then-steer-supersede", + shouldFinalizeRun: () => Effect.succeed(false), + hasUnpairedRunInterruptRequest: () => Effect.succeed(true), + }); + assert.deepEqual( + written.map((item) => item.type), + ["run_interrupt_result"], + ); + const ids = backgroundScenarioIds("stop-then-steer-supersede"); + const expectedRequestId = yield* Effect.gen(function* () { + const idAllocator = yield* IdAllocatorV2; + return idAllocator.derive.runSignalTurnItem({ + runId: ids.runId, + signal: "interrupt-request", + }); + }).pipe(Effect.provide(idAllocatorLayer)); + assert.equal(written[0]?.parentItemId, expectedRequestId); + }), +); + +it.effect("omits run_interrupt_result when superseded attempt request is already paired", () => + Effect.gen(function* () { + const written = yield* captureInterruptTerminalTurnItems({ + key: "stop-then-steer-already-paired", + shouldFinalizeRun: () => Effect.succeed(false), + hasUnpairedRunInterruptRequest: () => Effect.succeed(false), + }); + assert.deepEqual( + written.map((item) => item.type), + [], + ); + }), +); + +it.effect("emits run_interrupt_result when hard-stop finalizes the active attempt", () => + Effect.gen(function* () { + const written = yield* captureInterruptTerminalTurnItems({ + key: "hard-stop", + shouldFinalizeRun: () => Effect.succeed(true), + }); + assert.deepEqual( + written.map((item) => item.type), + ["run_interrupt_result"], + ); + }), +); + +function captureInterruptTerminalTurnItems(input: { + readonly key: string; + readonly shouldFinalizeRun: () => Effect.Effect; + readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; +}) { + return Effect.gen(function* () { + const ids = backgroundScenarioIds(input.key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const writtenItems = yield* Ref.make< + ReadonlyArray<{ readonly type: string; readonly parentItemId: string | null }> + >([]); + const ingestionDone = yield* Deferred.make(); + const captureTurnItem = (payload: { + readonly type: string; + readonly parentItemId: string | null; + }) => + Ref.update(writtenItems, (current) => [ + ...current, + { type: payload.type, parentItemId: payload.parentItemId }, + ]); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: (payload) => + Effect.gen(function* () { + for (const event of payload.events) { + if (event.type === "turn-item.updated") { + yield* captureTurnItem(event.payload); + } + } + return []; + }), + writeWithEffects: (payload) => + Effect.gen(function* () { + for (const event of payload.events) { + if (event.type === "turn-item.updated") { + yield* captureTurnItem(event.payload); + } + } + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: () => Effect.succeed([]), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${input.key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${input.key}`), + session: { + events: Stream.empty, + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([rootTerminalEvent(ids, "interrupted")]), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { + id: ids.rootNodeId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${input.key}`), + } as OrchestrationV2CheckpointScope, + providerThread: { + id: ids.providerThreadId, + driver, + } as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + shouldFinalizeRun: input.shouldFinalizeRun, + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, + }), + message: { + messageId: MessageId.make(`message:${input.key}`), + text: "interrupt projection", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish"); + return yield* Ref.get(writtenItems); + }); +} + interface BackgroundScenarioIds { readonly threadId: ThreadId; readonly childThreadId: ThreadId; diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index e8750e36870..c833fdf7b36 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -260,6 +260,7 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly relatedProviderThreadIds?: ReadonlyArray; readonly shouldStartProviderTurn?: () => Effect.Effect; readonly shouldFinalizeRun?: () => Effect.Effect; + readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly message: ProviderAdapterV2TurnMessage; readonly modelSelection: ModelSelection; readonly runtimePolicy: ProviderAdapterV2RuntimePolicy; @@ -323,6 +324,7 @@ export const layer: Layer.Layer< readonly providerThread: OrchestrationV2ProviderThread; readonly attempt: OrchestrationV2RunAttempt; readonly shouldFinalizeRun?: () => Effect.Effect; + readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly terminal: ProviderTerminalEvent; readonly failureItemPersisted: boolean; }) => @@ -336,30 +338,37 @@ export const layer: Layer.Layer< const shouldFinalizeRun = input.shouldFinalizeRun === undefined ? true : yield* input.shouldFinalizeRun(); if (!shouldFinalizeRun) { - // A newer attempt already owns the run and the command that created - // it terminalized this attempt as superseded. Preserve that domain - // status while retaining the provider's interruption artifact. + // Superseded attempt (steer / selection restart). Emit + // run_interrupt_result only when hard Stop left an unpaired request + // for this run; plain steers and already-paired stops emit nothing. if (input.terminal.status === "interrupted") { - yield* eventSink.write({ - events: [ - { - id: yield* idAllocator.allocate.event({ threadId: input.run.threadId }), - type: "turn-item.updated" as const, - threadId: input.run.threadId, - runId: input.run.id, - nodeId: input.rootNode.id, - providerInstanceId: input.run.providerInstanceId, - occurredAt: completedAt, - payload: makeInterruptResultTurnItem({ - idAllocator, - run: input.run, - rootNode: input.rootNode, - providerThread: input.providerThread, - completedAt, - }), - }, - ], - }); + const hasUnpairedRequest = + input.hasUnpairedRunInterruptRequest === undefined + ? false + : yield* input.hasUnpairedRunInterruptRequest(); + if (hasUnpairedRequest) { + yield* eventSink.writeWithEffects({ + effects: [], + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: input.run.threadId }), + type: "turn-item.updated" as const, + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: makeInterruptResultTurnItem({ + idAllocator, + run: input.run, + rootNode: input.rootNode, + providerThread: input.providerThread, + completedAt, + }), + }, + ], + }); + } } return; } @@ -594,6 +603,11 @@ export const layer: Layer.Layer< ...(input.shouldFinalizeRun === undefined ? {} : { shouldFinalizeRun: input.shouldFinalizeRun }), + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, + }), terminal, failureItemPersisted: terminal.status === "failed", }).pipe( @@ -776,6 +790,12 @@ export const layer: Layer.Layer< ...(input.shouldFinalizeRun === undefined ? {} : { shouldFinalizeRun: input.shouldFinalizeRun }), + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: + input.hasUnpairedRunInterruptRequest, + }), terminal: makeFailedTerminalEvent( makeProviderFailure({ cause: Cause.squash(cause), @@ -847,6 +867,12 @@ export const layer: Layer.Layer< ...(input.shouldFinalizeRun === undefined ? {} : { shouldFinalizeRun: input.shouldFinalizeRun }), + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: + input.hasUnpairedRunInterruptRequest, + }), terminal: makeFailedTerminalEvent( makeProviderFailure({ cause: Cause.squash(cause), diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index 143a1b451e6..9d9a5f6a24b 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -490,7 +490,12 @@ it.live("restarts selection as a new attempt and retries after old-session clean projection.providerTurns.map((turn) => turn.status), ["interrupted", "completed"], ); - assert.isTrue(projection.turnItems.some((item) => item.type === "run_interrupt_result")); + assert.isFalse( + projection.turnItems.some( + (item) => item.type === "run_interrupt_request" || item.type === "run_interrupt_result", + ), + "selection restart supersede must not project hard-Stop interrupt items", + ); assert.equal(projection.runs[0]?.modelSelection.model, replacementSelection.model); assert.isTrue(captured.failedReplacementOpen); // The old pooled process remains available to its other threads; this diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/cursor_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/cursor_output.ts index b4482f762d3..33149449274 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/cursor_output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/cursor_output.ts @@ -24,7 +24,7 @@ export function assertCursorMessageSteeringOutput( const projection = projectionFor(result, transcript.scenario); assertSemanticProjectionIntegrity(projection); - assertTurnItemTypes(projection, ["user_message", "run_interrupt_result", "assistant_message"]); + assertTurnItemTypes(projection, ["user_message", "assistant_message"]); assertRuntimeRequestCounts(projection, { total: 0 }); assertUserMessagesInclude(projection, [ MESSAGE_STEERING_INITIAL_PROMPT, @@ -47,9 +47,10 @@ export function assertCursorMessageSteeringOutput( ); assert.equal(projection.runs[0]?.activeAttemptId, projection.attempts[1]?.id); assert.equal(projection.runs[0]?.rootNodeId, projection.attempts[1]?.rootNodeId); - assert.notInclude( - projection.visibleTurnItems.map((row) => row.item.type), - "run_interrupt_result", - "restart steering must keep its internal interruption out of visible chat history", + assert.isFalse( + projection.turnItems.some( + (item) => item.type === "run_interrupt_request" || item.type === "run_interrupt_result", + ), + "steer supersede must not project run_interrupt_* items (those are hard Stop only)", ); } diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/grok_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/grok_output.ts index 7fbd5d1f643..3587141c96f 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/grok_output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/grok_output.ts @@ -23,7 +23,7 @@ export function assertGrokMessageSteeringOutput( const projection = projectionFor(result, transcript.scenario); assertSemanticProjectionIntegrity(projection); - assertTurnItemTypes(projection, ["user_message", "run_interrupt_result", "assistant_message"]); + assertTurnItemTypes(projection, ["user_message", "assistant_message"]); assertRuntimeRequestCounts(projection, { total: 0 }); assertUserMessagesInclude(projection, [ MESSAGE_STEERING_INITIAL_PROMPT, @@ -46,9 +46,10 @@ export function assertGrokMessageSteeringOutput( ); assert.equal(projection.runs[0]?.activeAttemptId, projection.attempts[1]?.id); assert.equal(projection.runs[0]?.rootNodeId, projection.attempts[1]?.rootNodeId); - assert.notInclude( - projection.visibleTurnItems.map((row) => row.item.type), - "run_interrupt_result", - "restart steering must keep its internal interruption out of visible chat history", + assert.isFalse( + projection.turnItems.some( + (item) => item.type === "run_interrupt_request" || item.type === "run_interrupt_result", + ), + "steer supersede must not project run_interrupt_* items (those are hard Stop only)", ); } diff --git a/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts b/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts index b09a3170096..ecd3ce55d58 100644 --- a/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts +++ b/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts @@ -749,6 +749,40 @@ describe("AcpSessionRuntime", () => { ), ); + it.effect("completes ad-hoc loadSession after replay becomes idle while RPC stays pending", () => + Effect.gen(function* () { + const runtime = yield* AcpSessionRuntime.AcpSessionRuntime; + yield* runtime.start(); + const loaded = yield* runtime.loadSession("mock-session-1").pipe(Effect.timeout("2 seconds")); + + expect(loaded.sessionId).toBe("mock-session-1"); + expect(loaded.sessionSetupResult._meta).toMatchObject({ + t3SessionLoadReady: "replay_idle", + }); + }).pipe( + Effect.provide( + AcpSessionRuntime.layer({ + authMethodId: "test", + spawn: { + command: mockAgentCommand, + args: mockAgentArgs, + env: { + T3_ACP_HANG_LOAD_SESSION_AFTER_REPLAY: "1", + T3_ACP_LOAD_SESSION_DELAY_MS: "10000", + }, + }, + cwd: process.cwd(), + sessionLoadReplayIdleGap: "50 millis", + sessionLoadTimeout: "1 second", + clientInfo: { name: "t3-test", version: "0.0.0" }, + }), + ), + Effect.scoped, + Effect.provide(NodeServices.layer), + TestClock.withLive, + ), + ); + it.effect("rejects invalid config option values before sending session/set_config_option", () => { const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "acp-runtime-")); const requestLogPath = NodePath.join(tempDir, "requests.ndjson"); diff --git a/apps/server/src/provider/acp/AcpRuntimeModel.test.ts b/apps/server/src/provider/acp/AcpRuntimeModel.test.ts index 7682c5f5f9c..87dad45311f 100644 --- a/apps/server/src/provider/acp/AcpRuntimeModel.test.ts +++ b/apps/server/src/provider/acp/AcpRuntimeModel.test.ts @@ -8,6 +8,7 @@ import { parsePermissionRequest, parseSessionModeState, parseSessionUpdateEvent, + sessionUpdateCountsAsLoadReplayActivity, sessionUpdateIsReplay, syntheticLoadSessionResponseFromInitialize, } from "./AcpRuntimeModel.ts"; @@ -83,6 +84,98 @@ describe("AcpRuntimeModel", () => { ).toBe(false); }); + it("ignores Grok keepalive chunks when tracking session/load replay activity", () => { + expect( + sessionUpdateCountsAsLoadReplayActivity({ + sessionId: "session-1", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "" }, + }, + } satisfies EffectAcpSchema.SessionNotification), + ).toBe(false); + expect( + sessionUpdateCountsAsLoadReplayActivity({ + _meta: { isReplay: true }, + sessionId: "session-1", + update: { + sessionUpdate: "tool_call", + toolCallId: "replay-tool", + title: "Replay", + kind: "search", + status: "completed", + }, + } satisfies EffectAcpSchema.SessionNotification), + ).toBe(true); + }); + + it("ignores load-replay activity from other sessions while a load gate is active", () => { + expect( + sessionUpdateCountsAsLoadReplayActivity( + { + sessionId: "session-other", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "unrelated" }, + }, + } satisfies EffectAcpSchema.SessionNotification, + "session-loading", + ), + ).toBe(false); + expect( + sessionUpdateCountsAsLoadReplayActivity( + { + sessionId: "session-loading", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "replay" }, + }, + } satisfies EffectAcpSchema.SessionNotification, + "session-loading", + ), + ).toBe(true); + }); + + it("counts mode/config/session/usage updates as load-replay activity", () => { + expect( + sessionUpdateCountsAsLoadReplayActivity({ + sessionId: "session-1", + update: { + sessionUpdate: "current_mode_update", + currentModeId: "code", + }, + } satisfies EffectAcpSchema.SessionNotification), + ).toBe(true); + expect( + sessionUpdateCountsAsLoadReplayActivity({ + sessionId: "session-1", + update: { + sessionUpdate: "config_option_update", + configOptions: [], + }, + } satisfies EffectAcpSchema.SessionNotification), + ).toBe(true); + expect( + sessionUpdateCountsAsLoadReplayActivity({ + sessionId: "session-1", + update: { + sessionUpdate: "session_info_update", + title: "Restored", + }, + } satisfies EffectAcpSchema.SessionNotification), + ).toBe(true); + expect( + sessionUpdateCountsAsLoadReplayActivity({ + sessionId: "session-1", + update: { + sessionUpdate: "usage_update", + used: 1, + size: 10, + }, + } satisfies EffectAcpSchema.SessionNotification), + ).toBe(true); + }); + it("builds a synthetic load response from initialize model state", () => { const response = syntheticLoadSessionResponseFromInitialize({ protocolVersion: 1, diff --git a/apps/server/src/provider/acp/AcpRuntimeModel.ts b/apps/server/src/provider/acp/AcpRuntimeModel.ts index 2a3d4bf96b6..7aca8ffb32d 100644 --- a/apps/server/src/provider/acp/AcpRuntimeModel.ts +++ b/apps/server/src/provider/acp/AcpRuntimeModel.ts @@ -459,8 +459,38 @@ export function sessionUpdateIsReplay(params: EffectAcpSchema.SessionNotificatio return isRecord(meta) && meta.isReplay === true; } +/** Replay chunks and substantive updates during session/load; not Grok keepalives. */ +export function sessionUpdateCountsAsLoadReplayActivity( + params: EffectAcpSchema.SessionNotification, + gatedSessionId?: string, +): boolean { + if (gatedSessionId !== undefined && params.sessionId !== gatedSessionId) { + return false; + } + if (sessionUpdateIsReplay(params)) return true; + const update = params.update; + switch (update.sessionUpdate) { + case "agent_message_chunk": + case "agent_thought_chunk": + return update.content.type === "text" && update.content.text.length > 0; + case "tool_call": + case "tool_call_update": + case "plan": + case "available_commands_update": + case "current_mode_update": + case "config_option_update": + case "session_info_update": + case "usage_update": + return true; + default: + return false; + } +} + export interface SessionLoadGate { readonly active: boolean; + /** Only notifications for this session refresh load-replay idle activity. */ + readonly sessionId: string; readonly lastActivityAtMillis: number | undefined; readonly idleGap: Duration.Duration; readonly initializeResult: EffectAcpSchema.InitializeResponse; diff --git a/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts b/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts new file mode 100644 index 00000000000..1d263755c25 --- /dev/null +++ b/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts @@ -0,0 +1,1030 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; + +import { describe, expect, it } from "@effect/vitest"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Scope from "effect/Scope"; + +import { + capturePosixOwnershipLedger, + isSafeCgroupPath, + makeLinuxCgroupController, + makePosixProcessTreeController, + observePosixOwnershipLedger, + observePosixOwnershipLedgerContinuously, + parseCgroup2Mount, + parseCgroup2Mounts, + parseUnifiedCgroupPath, + posixProcessIsZombie, + resolveLinuxCgroupTargetCommand, + signalLinuxCgroupRootTerm, + sweepStaleLinuxCgroupSiblings, + terminateLinuxCgroupLease, + terminatePosixOwnedProcessTree, + wrapCommandForLinuxCgroup, + type AcpOwnedPosixProcess, + type AcpLinuxCgroupLease, + type AcpPosixOwnershipRoot, + type AcpPosixProcessIdentity, + type AcpPosixProcessTreeController, +} from "./AcpSessionRuntime.ts"; + +const threadSpawnHelperSource = NodeURL.fileURLToPath( + new URL("../../../scripts/acp-thread-spawn-helper.c", import.meta.url), +); + +const identity = ( + pid: number, + ppid: number, + pgid: number, + sid: number, + startTime = String(pid), + state = "S", +): AcpPosixProcessIdentity => ({ + executable: `/proc/${pid}/exe`, + pgid, + pid, + ppid, + sid, + startTime, + state, +}); + +function makeController(input: { + readonly processes: ReadonlyArray; + readonly onProcess?: ( + processes: Map, + pid: number, + signal: NodeJS.Signals, + ) => void; +}) { + const processes = new Map(input.processes.map((process) => [process.pid, process])); + const signals: Array = []; + const controller: AcpPosixProcessTreeController = { + childPidsOf: (pid) => + [...processes.values()] + .filter((process) => process.ppid === pid) + .map((process) => process.pid), + childrenOf: (pid) => [...processes.values()].filter((process) => process.ppid === pid), + identity: (pid) => processes.get(pid), + snapshot: () => [...processes.values()], + signalProcess: (pid, signal) => { + signals.push(`process:${pid}:${signal}`); + if (input.onProcess) return input.onProcess(processes, pid, signal); + processes.delete(pid); + }, + }; + return { controller, processes, signals }; +} + +const server = () => identity(process.pid, 1, process.pid, process.pid, "server"); + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +describe("terminatePosixOwnedProcessTree", () => { + it("parses unified cgroup paths and escaped cgroup2 mountinfo", () => { + expect(parseUnifiedCgroupPath("0::/user.slice/app.scope\n")).toBe("/user.slice/app.scope"); + expect(parseUnifiedCgroupPath("0::/one\n0::/two\n")).toBeUndefined(); + expect(isSafeCgroupPath("/user.slice/app.scope")).toBe(true); + expect(isSafeCgroupPath("/user.slice/../escape")).toBe(false); + expect(isSafeCgroupPath("/user.slice/app.scope (deleted)")).toBe(false); + expect( + parseCgroup2Mount( + "42 31 0:37 /user.slice /sys/fs/cgroup/user\\040mount rw - cgroup2 cgroup2 rw\n", + ), + ).toEqual({ mountPoint: "/sys/fs/cgroup/user mount", root: "/user.slice" }); + expect(parseCgroup2Mount("42 31 0:37 / /sys/fs/cgroup rw - tmpfs tmpfs rw\n")).toBeUndefined(); + expect( + parseCgroup2Mounts( + "41 31 0:36 /other /sys/fs/cgroup/other rw - cgroup2 cgroup2 rw\n" + + "42 31 0:37 /user.slice /sys/fs/cgroup/user rw - cgroup2 cgroup2 rw\n", + ), + ).toHaveLength(2); + }); + + it.live("sweeps empty t3-acp sibling leases owned by dead pids only", () => + Effect.gen(function* () { + const scratchRoot = NodePath.join(process.cwd(), "tmp"); + NodeFS.mkdirSync(scratchRoot, { recursive: true }); + const parent = NodeFS.mkdtempSync(NodePath.join(scratchRoot, "acp-cgroup-stale-siblings-")); + // Temp dirs are not cgroupfs: rmdir fails when control files exist, so the + // test injects populated/remove while still exercising the real selection + // and path-safety logic against a real parent directory tree. + const populatedByPath = new Map(); + const writeSibling = (name: string, populated: "0" | "1" | null) => { + const path = NodePath.join(parent, name); + NodeFS.mkdirSync(path); + if (populated !== null) populatedByPath.set(path, populated); + return path; + }; + const deadPid = 2_147_483_646; + // Synthetic live foreign pid (not this process, still reported alive by the probe). + const liveForeignPid = 2_147_483_645; + const staleEmpty = writeSibling(`t3-acp-${deadPid}-aaaa`, "0"); + const currentPidSibling = writeSibling(`t3-acp-${process.pid}-bbbb`, "0"); + const liveForeignSibling = writeSibling(`t3-acp-${liveForeignPid}-cccc`, "0"); + const stalePopulated = writeSibling(`t3-acp-${deadPid}-dddd`, "1"); + const unrelated = writeSibling("other-lease", "0"); + const isProcessAlive = (pid: number) => pid === process.pid || pid === liveForeignPid; + try { + sweepStaleLinuxCgroupSiblings(parent, { + isProcessAlive, + readPopulated: (path) => populatedByPath.get(path), + remove: (path) => { + populatedByPath.delete(path); + NodeFS.rmdirSync(path); + }, + }); + expect(NodeFS.existsSync(staleEmpty)).toBe(false); + expect(NodeFS.existsSync(currentPidSibling)).toBe(true); + expect(NodeFS.existsSync(liveForeignSibling)).toBe(true); + expect(NodeFS.existsSync(stalePopulated)).toBe(true); + expect(NodeFS.existsSync(unrelated)).toBe(true); + // Missing populated state must not remove a sibling. + writeSibling(`t3-acp-${deadPid}-eeee`, null); + sweepStaleLinuxCgroupSiblings(parent, { + isProcessAlive, + readPopulated: (path) => populatedByPath.get(path), + remove: (path) => { + populatedByPath.delete(path); + NodeFS.rmdirSync(path); + }, + }); + expect(NodeFS.existsSync(NodePath.join(parent, `t3-acp-${deadPid}-eeee`))).toBe(true); + expect(NodeFS.existsSync(currentPidSibling)).toBe(true); + } finally { + NodeFS.rmSync(parent, { recursive: true, force: true }); + } + }), + ); + + it.live("preserves exact argv and strips wrapper-only environment before exec", () => + Effect.gen(function* () { + if ((yield* HostProcessPlatform) !== "linux") return; + const lease = makeLinuxCgroupController().create(); + if (lease === undefined) return; + const scratchRoot = NodePath.join(process.cwd(), "tmp"); + NodeFS.mkdirSync(scratchRoot, { recursive: true }); + const scratch = NodeFS.mkdtempSync(NodePath.join(scratchRoot, "acp-cgroup-wrapper-")); + const linkedNode = NodePath.join(scratch, "T3 Code AppImage 'quoted'"); + const bareGrok = NodePath.join(scratch, "grok"); + const relativeBin = NodePath.join(scratch, "relative-bin"); + const directoryBin = NodePath.join(scratch, "directory-bin"); + const validBin = NodePath.join(scratch, "valid-bin"); + const outputPath = NodePath.join(scratch, "argv.json"); + NodeFS.mkdirSync(relativeBin); + NodeFS.mkdirSync(NodePath.join(directoryBin, "grok"), { recursive: true }); + NodeFS.mkdirSync(validBin); + NodeFS.symlinkSync(process.execPath, linkedNode); + NodeFS.symlinkSync(process.execPath, bareGrok); + NodeFS.symlinkSync(process.execPath, NodePath.join(relativeBin, "grok")); + NodeFS.symlinkSync(process.execPath, NodePath.join(validBin, "grok")); + expect(resolveLinuxCgroupTargetCommand("grok", scratch, { PATH: scratch })).toBe(bareGrok); + expect(resolveLinuxCgroupTargetCommand("grok", scratch, { PATH: "" })).toBe(bareGrok); + // Undefined PATH uses Node's default search path (/usr/bin:/bin), not cwd alone + // (bareGrok lives only in cwd / PATH=scratch). + expect(resolveLinuxCgroupTargetCommand("grok", scratch, { PATH: undefined })).toBeUndefined(); + expect(resolveLinuxCgroupTargetCommand("node", scratch, { PATH: undefined })).toBeDefined(); + expect(resolveLinuxCgroupTargetCommand("grok", scratch, { PATH: "relative-bin" })).toBe( + NodePath.join(relativeBin, "grok"), + ); + expect( + resolveLinuxCgroupTargetCommand("grok", scratch, { + PATH: `directory-bin${NodePath.delimiter}valid-bin`, + }), + ).toBe(NodePath.join(validBin, "grok")); + const target = [ + 'const fs = require("node:fs");', + "fs.writeFileSync(process.argv[1], JSON.stringify({", + " args: process.argv.slice(2),", + " electron: process.env.ELECTRON_RUN_AS_NODE,", + " wrapper: process.env.T3_ACP_CGROUP_WRAPPER,", + "}));", + ].join("\n"); + const wrapped = wrapCommandForLinuxCgroup(lease, linkedNode, [ + "-e", + target, + outputPath, + "space value", + "single'quote", + 'double"quote', + ]); + try { + const result = NodeChildProcess.spawnSync(wrapped.command, wrapped.args, { + encoding: "utf8", + env: { + ...process.env, + ELECTRON_RUN_AS_NODE: "1", + T3_ACP_CGROUP_WRAPPER: "1", + }, + }); + expect(result.status, result.stderr).toBe(0); + expect(NodeFS.readFileSync(outputPath, "utf8")).toBe( + '{"args":["space value","single\'quote","double\\"quote"]}', + ); + } finally { + yield* terminateLinuxCgroupLease(lease).pipe(Effect.ignore); + NodeFS.rmSync(scratch, { recursive: true, force: true }); + } + }), + ); + + it.live("kills a post-TERM detached double fork without touching an unrelated sentinel", () => + Effect.gen(function* () { + if ((yield* HostProcessPlatform) !== "linux") return; + const controller = makeLinuxCgroupController(); + const lease = controller.create(); + if (lease === undefined) return; + expect(parseUnifiedCgroupPath(NodeFS.readFileSync("/proc/self/cgroup", "utf8"))).not.toBe( + lease.relativePath, + ); + const scratchRoot = NodePath.join(process.cwd(), "tmp"); + NodeFS.mkdirSync(scratchRoot, { recursive: true }); + const scratch = NodeFS.mkdtempSync(NodePath.join(scratchRoot, "acp-cgroup-kill-")); + const readyPath = NodePath.join(scratch, "ready"); + const childPath = NodePath.join(scratch, "child"); + const childProgram = "setInterval(() => {}, 1000);"; + const launcherProgram = [ + 'const fs = require("node:fs");', + 'const cp = require("node:child_process");', + "const child = cp.spawn(process.execPath, ['-e', process.argv[2]], { detached: true, stdio: 'ignore' });", + "fs.writeFileSync(process.argv[1], String(child.pid));", + "child.unref();", + ].join("\n"); + const providerProgram = [ + 'const fs = require("node:fs");', + 'const cp = require("node:child_process");', + "fs.writeFileSync(process.argv[1], String(process.pid));", + "process.on('SIGTERM', () => {", + " const launcher = cp.spawn(process.execPath, ['-e', process.argv[3], process.argv[2], process.argv[4]], { detached: true, stdio: 'ignore' });", + " launcher.unref();", + " process.exit(0);", + "});", + "setInterval(() => {}, 1000);", + ].join("\n"); + const wrapped = wrapCommandForLinuxCgroup(lease, process.execPath, [ + "-e", + providerProgram, + readyPath, + childPath, + launcherProgram, + childProgram, + ]); + const provider = NodeChildProcess.spawn(wrapped.command, wrapped.args, { + detached: true, + env: { ...process.env, ELECTRON_RUN_AS_NODE: "1", T3_ACP_CGROUP_WRAPPER: "1" }, + stdio: "ignore", + }); + provider.unref(); + const sentinel = NodeChildProcess.spawn(process.execPath, ["-e", childProgram], { + detached: true, + stdio: "ignore", + }); + sentinel.unref(); + let detachedChildPid: number | undefined; + try { + yield* Effect.gen(function* () { + while (!NodeFS.existsSync(readyPath)) yield* Effect.sleep("10 millis"); + }).pipe(Effect.timeout("2 seconds")); + process.kill(provider.pid!, "SIGTERM"); + detachedChildPid = yield* Effect.gen(function* () { + while (true) { + try { + const pid = Number(NodeFS.readFileSync(childPath, "utf8")); + if (Number.isSafeInteger(pid) && pid > 1) return pid; + } catch { + // The TERM-time launcher has not published its child yet. + } + yield* Effect.sleep("10 millis"); + } + }).pipe(Effect.timeout("2 seconds")); + expect(processExists(detachedChildPid)).toBe(true); + yield* terminateLinuxCgroupLease(lease); + expect(processExists(sentinel.pid!)).toBe(true); + yield* Effect.gen(function* () { + while (processExists(detachedChildPid!)) yield* Effect.sleep("10 millis"); + }).pipe(Effect.timeout("2 seconds")); + expect(NodeFS.existsSync(lease.path)).toBe(false); + + const replacement = controller.create(); + expect(replacement?.path).not.toBe(lease.path); + if (replacement !== undefined) { + NodeFS.mkdirSync(NodePath.join(replacement.path, "nested")); + yield* terminateLinuxCgroupLease(replacement); + expect(NodeFS.existsSync(replacement.path)).toBe(false); + } + } finally { + yield* terminateLinuxCgroupLease(lease).pipe(Effect.ignore); + try { + process.kill(-provider.pid!, "SIGKILL"); + } catch { + // The contained provider already exited. + } + try { + process.kill(-sentinel.pid!, "SIGKILL"); + } catch { + // The unrelated sentinel already exited. + } + if (detachedChildPid !== undefined && processExists(detachedChildPid)) { + try { + process.kill(detachedChildPid, "SIGKILL"); + } catch { + // The detached child already exited. + } + } + NodeFS.rmSync(scratch, { recursive: true, force: true }); + } + }), + ); + + it.live("re-kills repopulated cgroups and verifies root removal after retryable errors", () => + Effect.gen(function* () { + let exists = true; + let killCalls = 0; + let populated = false; + let removeCalls = 0; + const lease: AcpLinuxCgroupLease = { + contains: () => false, + exists: () => exists, + path: "/test/t3-acp-repopulation", + relativePath: "/test/t3-acp-repopulation", + kill: () => { + killCalls += 1; + populated = false; + }, + populated: () => populated, + remove: () => { + removeCalls += 1; + if (removeCalls === 1) { + populated = true; + const error = new Error("repopulated") as NodeJS.ErrnoException; + error.code = "EBUSY"; + throw error; + } + if (removeCalls === 2) { + const error = new Error("nested path vanished") as NodeJS.ErrnoException; + error.code = "ENOENT"; + throw error; + } + exists = false; + }, + }; + + yield* terminateLinuxCgroupLease(lease); + expect(killCalls).toBeGreaterThanOrEqual(3); + expect(removeCalls).toBe(3); + expect(exists).toBe(false); + yield* terminateLinuxCgroupLease(lease); + expect(removeCalls).toBe(3); + }), + ); + + it("sends TERM only to the exact captured root while it remains in the child cgroup", () => { + const fixture = makeController({ + processes: [server(), identity(100, process.pid, 100, 100, "owned")], + }); + const ownedRoot: AcpOwnedPosixProcess = { + ...identity(100, process.pid, 100, 100, "owned"), + parentExecutable: server().executable, + parentStartTime: server().startTime, + }; + const root: AcpPosixOwnershipRoot = { captureAttempted: true, value: ownedRoot }; + const lease: AcpLinuxCgroupLease = { + contains: () => true, + exists: () => true, + path: "/test/t3-acp-root", + relativePath: "/test/t3-acp-root", + kill: () => undefined, + populated: () => true, + remove: () => undefined, + }; + signalLinuxCgroupRootTerm({ controller: fixture.controller, lease, root }); + expect(fixture.signals).toEqual(["process:100:SIGTERM"]); + + const reused = makeController({ + processes: [server(), identity(100, process.pid, 100, 100, "reused")], + }); + signalLinuxCgroupRootTerm({ controller: reused.controller, lease, root }); + expect(reused.signals).toEqual([]); + + const migrated = makeController({ + processes: [server(), identity(100, process.pid, 100, 100, "owned")], + }); + signalLinuxCgroupRootTerm({ + controller: migrated.controller, + lease: { ...lease, contains: () => false }, + root, + }); + expect(migrated.signals).toEqual([]); + }); + + it.live("finds a child forked by a non-leader pthread", () => + Effect.gen(function* () { + if ((yield* HostProcessPlatform) !== "linux") return; + const scratchRoot = NodePath.join(process.cwd(), "tmp"); + NodeFS.mkdirSync(scratchRoot, { recursive: true }); + const scratch = NodeFS.mkdtempSync(NodePath.join(scratchRoot, "acp-pthread-spawn-")); + const binary = NodePath.join(scratch, "helper"); + const pidPath = NodePath.join(scratch, "pids"); + expect(NodeFS.statSync(threadSpawnHelperSource).isFile()).toBe(true); + const compile = NodeChildProcess.spawnSync( + "cc", + ["-pthread", threadSpawnHelperSource, "-o", binary], + { + encoding: "utf8", + }, + ); + if ((compile.error as NodeJS.ErrnoException | undefined)?.code === "ENOENT") { + NodeFS.rmSync(scratch, { recursive: true, force: true }); + return; + } + expect(compile.status, compile.stderr).toBe(0); + const helper = NodeChildProcess.spawn(binary, [pidPath], { detached: true, stdio: "ignore" }); + helper.unref(); + try { + const published = yield* Effect.gen(function* () { + while (true) { + try { + const ids = NodeFS.readFileSync(pidPath, "utf8").trim().split(/\s+/).map(Number); + if (ids.length === 2 && ids.every(Number.isSafeInteger)) return ids; + } catch { + // The worker has not published its TID and child yet. + } + yield* Effect.sleep("10 millis"); + } + }).pipe(Effect.timeout("2 seconds")); + const [workerTid, childPid] = published; + expect(workerTid).not.toBe(helper.pid); + expect(makePosixProcessTreeController("linux").childrenOf(helper.pid!)).toContainEqual( + expect.objectContaining({ pid: childPid, ppid: helper.pid }), + ); + } finally { + try { + process.kill(-helper.pid!, "SIGKILL"); + } catch { + // The isolated fixture already exited. + } + NodeFS.rmSync(scratch, { recursive: true, force: true }); + } + }), + ); + + it.live("keeps ownership across a fork-before-exec executable transition", () => + Effect.gen(function* () { + const beforeExec = { ...identity(110, 100, 110, 110), executable: "/provider/fork" }; + const fixture = makeController({ + processes: [server(), identity(100, process.pid, 100, 100), beforeExec], + }); + const frontier = new Map(); + const childQueues = new Map>(); + const ledger = new Map(); + const root: AcpPosixOwnershipRoot = { value: undefined }; + observePosixOwnershipLedger({ + childQueues, + controller: fixture.controller, + frontier, + ledger, + root, + rootPid: 100, + }); + fixture.processes.set(110, { ...beforeExec, executable: "/usr/bin/bash" }); + observePosixOwnershipLedger({ + childQueues, + controller: fixture.controller, + frontier, + ledger, + root, + rootPid: 100, + }); + + expect([...ledger.values()].find((process) => process.pid === 110)?.executable).toBe( + "/usr/bin/bash", + ); + yield* terminatePosixOwnedProcessTree({ + controller: fixture.controller, + grace: 0, + ledger, + rootPid: 100, + }); + expect(fixture.signals).toContain("process:110:SIGTERM"); + expect(fixture.processes.has(110)).toBe(false); + }), + ); + + it("never adopts a replacement PID after initial root capture failed", () => { + const fixture = makeController({ processes: [server()] }); + const frontier = new Map(); + const childQueues = new Map>(); + const ledger = new Map(); + const root: AcpPosixOwnershipRoot = { value: undefined }; + const capture = () => { + try { + observePosixOwnershipLedger({ + childQueues, + controller: fixture.controller, + frontier, + ledger, + root, + rootPid: 100, + }); + return undefined; + } catch (error) { + return error; + } + }; + expect(capture()).toMatchObject({ + detail: "ACP process 100 exited before its ownership ledger was captured", + }); + fixture.processes.set(100, identity(100, process.pid, 100, 100, "reused")); + expect(capture()).toMatchObject({ + detail: "ACP process 100 exited before its ownership ledger was captured", + }); + expect(ledger.size).toBe(0); + }); + + it.live("rotates more than 64 live parents without scanning retained tombstones", () => + Effect.gen(function* () { + const parents = Array.from({ length: 130 }, (_, index) => + identity(1_000 + index, 100, 1_000 + index, 1_000 + index), + ); + let childListReads = 0; + let identityCalls = 0; + let snapshotCalls = 0; + const fixture = makeController({ + processes: [server(), identity(100, process.pid, 100, 100)], + }); + const controller: AcpPosixProcessTreeController = { + ...fixture.controller, + childPidsOf: (pid) => { + childListReads += 1; + return fixture.controller.childPidsOf(pid); + }, + identity: (pid) => { + identityCalls += 1; + return fixture.controller.identity(pid); + }, + snapshot: () => { + snapshotCalls += 1; + return fixture.controller.snapshot(); + }, + }; + const frontier = new Map(); + const childQueues = new Map>(); + const ledger = new Map(); + const root: AcpPosixOwnershipRoot = { value: undefined }; + for (let index = 0; index < 5_000; index += 1) { + const tombstone = identity(100_000 + index, 1, 100_000 + index, 100_000 + index); + ledger.set(`${tombstone.pid}:${tombstone.startTime}`, { + ...tombstone, + parentExecutable: undefined, + parentStartTime: "", + }); + } + observePosixOwnershipLedger({ + childQueues, + controller, + frontier, + ledger, + root, + rootPid: 100, + }); + for (const parent of parents) { + fixture.processes.set(parent.pid, parent); + fixture.processes.set( + parent.pid + 10_000, + identity(parent.pid + 10_000, parent.pid, parent.pgid, parent.sid), + ); + } + let passes = 0; + while ( + !parents.every((parent) => + [...ledger.values()].some((owned) => owned.pid === parent.pid + 10_000), + ) && + passes < 40 + ) { + childListReads = 0; + identityCalls = 0; + observePosixOwnershipLedger({ + childQueues, + controller, + frontier, + ledger, + maxProcesses: 64, + root, + rootPid: 100, + }); + expect(identityCalls + childListReads).toBeLessThanOrEqual(64); + passes += 1; + } + + const missing = parents.filter( + (parent) => ![...ledger.values()].some((owned) => owned.pid === parent.pid + 10_000), + ); + expect(missing, `missing after ${passes} passes`).toEqual([]); + expect(passes).toBeGreaterThan(2); + yield* terminatePosixOwnedProcessTree({ + controller, + grace: 0, + ledger, + rootPid: 100, + }); + expect(snapshotCalls).toBe(10); + expect(fixture.processes.size).toBe(1); + }), + ); + + it.live("polls only known descendants and stops with its scope", () => + Effect.gen(function* () { + let childrenCalls = 0; + let identityCalls = 0; + let snapshotCalls = 0; + const fixture = makeController({ + processes: [server(), identity(100, process.pid, 100, 100), identity(110, 100, 110, 110)], + }); + const controller: AcpPosixProcessTreeController = { + ...fixture.controller, + childPidsOf: (pid) => { + childrenCalls += 1; + return fixture.controller.childPidsOf(pid); + }, + identity: (pid) => { + identityCalls += 1; + return fixture.controller.identity(pid); + }, + snapshot: () => { + snapshotCalls += 1; + return fixture.controller.snapshot(); + }, + }; + const scope = yield* Scope.make(); + const frontier = new Map(); + const childQueues = new Map>(); + let ledgerValuesCalls = 0; + const ledger = new (class extends Map { + override values(): MapIterator { + ledgerValuesCalls += 1; + return super.values(); + } + })(); + const root: AcpPosixOwnershipRoot = { value: undefined }; + for (let index = 0; index < 5_000; index += 1) { + const tombstone = identity(10_000 + index, 1, 10_000 + index, 10_000 + index); + ledger.set(`${tombstone.pid}:${tombstone.startTime}`, { + ...tombstone, + parentExecutable: undefined, + parentStartTime: "", + }); + } + yield* observePosixOwnershipLedgerContinuously({ + childQueues, + controller, + frontier, + ledger, + root, + rootPid: 100, + }).pipe(Effect.forkIn(scope)); + const timerStartedAt = yield* Clock.currentTimeMillis; + yield* Effect.sleep(0); + const timerDelay = (yield* Clock.currentTimeMillis) - timerStartedAt; + yield* Effect.sleep("80 millis"); + + expect(timerDelay).toBeLessThan(250); + expect(ledgerValuesCalls).toBe(0); + expect(snapshotCalls).toBe(0); + expect(childrenCalls).toBeGreaterThan(0); + expect(childrenCalls).toBeLessThanOrEqual(12); + const callsAtClose = identityCalls + childrenCalls; + yield* Scope.close(scope, Exit.void); + yield* Effect.sleep("60 millis"); + expect(identityCalls + childrenCalls).toBe(callsAtClose); + }), + ); + + it.live("terminates nested owned groups bottom-up and catches a TERM fork race", () => + Effect.gen(function* () { + let forked = false; + const fixture = makeController({ + processes: [ + server(), + identity(100, process.pid, 100, 100), + identity(110, 100, 110, 110), + identity(115, 110, 115, 110), + identity(120, 110, 120, 120), + identity(121, 120, 120, 120), + ], + onProcess: (processes, pid, signal) => { + if (signal === "SIGTERM" && pid === 121 && !forked) { + forked = true; + processes.set(122, identity(122, 110, 122, 122)); + } + processes.delete(pid); + }, + }); + + yield* terminatePosixOwnedProcessTree({ + controller: fixture.controller, + grace: 0, + rootPid: 100, + }); + + expect(fixture.processes.has(100)).toBe(false); + expect(fixture.processes.has(122)).toBe(false); + expect(fixture.signals.indexOf("process:121:SIGTERM")).toBeLessThan( + fixture.signals.indexOf("process:120:SIGTERM"), + ); + expect(fixture.signals.indexOf("process:115:SIGTERM")).toBeLessThan( + fixture.signals.indexOf("process:110:SIGTERM"), + ); + }), + ); + + it.live("falls back to exact PIDs for unknown group members and a missing leader", () => + Effect.gen(function* () { + const unrelated = identity(999, 1, 110, 110, "unrelated"); + const fixture = makeController({ + processes: [ + server(), + identity(100, process.pid, 100, 100), + identity(110, 100, 110, 110), + identity(111, 110, 110, 110), + unrelated, + ], + onProcess: (processes, pid) => { + processes.delete(pid); + if (pid === 111) processes.delete(110); + }, + }); + + yield* terminatePosixOwnedProcessTree({ + controller: fixture.controller, + grace: 0, + rootPid: 100, + }); + + expect(fixture.signals).toContain("process:111:SIGTERM"); + expect(fixture.processes.get(999)).toEqual(unrelated); + }), + ); + + it.live("re-admits a still-owned child after PID reuse and never signals the T3 session", () => + Effect.gen(function* () { + const reused = identity(110, 100, 110, 110, "reused"); + const fixture = makeController({ + processes: [ + server(), + identity(100, process.pid, 100, 100), + identity(110, 100, 110, 110, "owned"), + identity(120, 100, 120, process.pid), + ], + onProcess: (processes, pid) => { + if (pid === 110) processes.set(110, reused); + else processes.delete(pid); + }, + }); + + const result = yield* Effect.exit( + terminatePosixOwnedProcessTree({ + controller: fixture.controller, + grace: 0, + rootPid: 100, + }), + ); + + // PID 110 morphs to a new identity under the owned root and never exits, so + // teardown fails closed after re-admitting and re-signalling the live child. + expect(Exit.isFailure(result)).toBe(true); + expect(fixture.processes.get(110)).toEqual(reused); + expect( + fixture.signals.filter((entry) => entry.startsWith("process:110:")).length, + ).toBeGreaterThan(0); + expect(fixture.signals.some((entry) => entry.includes(":120:"))).toBe(false); + }), + ); + + it.live("does not treat zombie residual entries as teardown survivors", () => + Effect.gen(function* () { + expect(posixProcessIsZombie({ ...identity(1, 0, 1, 1), state: "Z" })).toBe(true); + expect(posixProcessIsZombie({ ...identity(1, 0, 1, 1), state: "S" })).toBe(false); + const withoutState: AcpPosixProcessIdentity = { + executable: "/proc/1/exe", + pgid: 1, + pid: 1, + ppid: 0, + sid: 1, + startTime: "1", + }; + expect(posixProcessIsZombie(withoutState)).toBe(false); + + const root = identity(100, process.pid, 100, 100); + const zombieChild = identity(110, 100, 110, 110, "110", "Z"); + const fixture = makeController({ + processes: [server(), root, zombieChild], + onProcess: (processes, pid) => { + // Kill leaves a zombie with the same pid/starttime until reaped. + const current = processes.get(pid); + if (current === undefined) return; + processes.set(pid, { ...current, state: "Z" }); + }, + }); + yield* terminatePosixOwnedProcessTree({ + controller: fixture.controller, + grace: 0, + rootPid: 100, + }); + // Zombies remain visible but must not fail residual teardown. + expect(fixture.processes.get(100)?.state).toBe("Z"); + expect(fixture.processes.get(110)?.state).toBe("Z"); + }), + ); + + it("admits a live child after a free PID was previously reserved by a dead entry", () => { + const staleChild: AcpOwnedPosixProcess = { + ...identity(200, 100, 200, 200, "stale-child"), + parentExecutable: `/proc/100/exe`, + parentStartTime: "100", + }; + const liveChild = identity(200, 100, 200, 200, "live-child"); + const root = identity(100, process.pid, 100, 100); + const ownedRoot: AcpOwnedPosixProcess = { + ...root, + parentExecutable: `/proc/${process.pid}/exe`, + parentStartTime: "server", + }; + const ledger = new Map([ + [`100:${root.startTime}`, ownedRoot], + [`200:${staleChild.startTime}`, staleChild], + ]); + const controller = makeController({ + processes: [server(), root], + }).controller; + + // First capture: PID 200 is free, so the dead reservation is dropped. + capturePosixOwnershipLedger({ + controller, + ledger, + rootPid: 100, + table: [root], + }); + expect([...ledger.values()].some((entry) => entry.pid === 200)).toBe(false); + + // Second capture: a new process reuses PID 200 under the owned parent. + capturePosixOwnershipLedger({ + controller: makeController({ processes: [server(), root, liveChild] }).controller, + ledger, + rootPid: 100, + table: [root, liveChild], + }); + + const admitted = [...ledger.values()].find((entry) => entry.pid === 200); + expect(admitted?.startTime).toBe("live-child"); + }); + + it("admits a live child when a stale ledger entry still occupies the reused PID", () => { + const staleChild: AcpOwnedPosixProcess = { + ...identity(200, 100, 200, 200, "stale-child"), + parentExecutable: `/proc/100/exe`, + parentStartTime: "100", + }; + const liveChild = identity(200, 100, 200, 200, "live-child"); + const root = identity(100, process.pid, 100, 100); + const ownedRoot: AcpOwnedPosixProcess = { + ...root, + parentExecutable: `/proc/${process.pid}/exe`, + parentStartTime: "server", + }; + const ledger = new Map([ + [`100:${root.startTime}`, ownedRoot], + [`200:${staleChild.startTime}`, staleChild], + ]); + + capturePosixOwnershipLedger({ + controller: makeController({ processes: [server(), root, liveChild] }).controller, + ledger, + rootPid: 100, + table: [root, liveChild], + }); + + expect([...ledger.values()].some((entry) => entry.startTime === "stale-child")).toBe(false); + const admitted = [...ledger.values()].find((entry) => entry.pid === 200); + expect(admitted?.startTime).toBe("live-child"); + }); + + it.live("signals children forked under a still-live parent during the same pass", () => + Effect.gen(function* () { + let forked = false; + const fixture = makeController({ + processes: [ + server(), + identity(100, process.pid, 100, 100), + identity(110, 100, 110, 110), + identity(111, 110, 111, 111), + ], + onProcess: (processes, pid) => { + // Bottom-up: signal 111 first while 110 is still live, then fork 112 + // under 110 so the same pass must enqueue and signal 112. + if (pid === 111 && !forked) { + forked = true; + processes.set(112, identity(112, 110, 112, 112)); + } + processes.delete(pid); + }, + }); + + yield* terminatePosixOwnedProcessTree({ + controller: fixture.controller, + grace: 0, + rootPid: 100, + }); + + expect(forked).toBe(true); + expect(fixture.processes.has(112)).toBe(false); + expect(fixture.signals.some((entry) => entry.startsWith("process:112:"))).toBe(true); + }), + ); + + it.live("retains the ledger across root exit and an explicit failure for finalizer retry", () => + Effect.gen(function* () { + let ignoreSignals = true; + const ledger = new Map(); + const frontier = new Map(); + const childQueues = new Map>(); + const root: AcpPosixOwnershipRoot = { value: undefined }; + const fixture = makeController({ + processes: [ + server(), + identity(100, process.pid, 100, 100), + identity(110, 100, 110, 110), + identity(111, 110, 110, 110), + ], + onProcess: (processes, pid) => { + if (ignoreSignals && pid === 100) processes.delete(100); + if (!ignoreSignals) processes.delete(pid); + }, + }); + observePosixOwnershipLedger({ + childQueues, + controller: fixture.controller, + frontier, + ledger, + root, + rootPid: 100, + }); + + const explicit = yield* Effect.exit( + terminatePosixOwnedProcessTree({ + controller: fixture.controller, + grace: 0, + ledger, + rootPid: 100, + }), + ); + expect(Exit.isFailure(explicit)).toBe(true); + expect(fixture.processes.has(100)).toBe(false); + expect(fixture.processes.has(111)).toBe(true); + + fixture.processes.set(112, identity(112, 110, 112, 112)); + observePosixOwnershipLedger({ + childQueues, + controller: fixture.controller, + frontier, + ledger, + root, + rootPid: 100, + }); + ignoreSignals = false; + yield* terminatePosixOwnedProcessTree({ + controller: fixture.controller, + grace: 0, + ledger, + rootPid: 100, + }); + expect(fixture.processes.has(110)).toBe(false); + expect(fixture.processes.has(111)).toBe(false); + expect(fixture.processes.has(112)).toBe(false); + }), + ); + + it("fails closed where a stable POSIX identity provider is unavailable", () => { + try { + makePosixProcessTreeController("darwin"); + throw new Error("Expected Darwin process ownership to fail closed"); + } catch (error) { + expect(error).toMatchObject({ + detail: "Detached ACP descendant ownership is unsupported on darwin", + }); + } + }); +}); diff --git a/apps/server/src/provider/acp/AcpSessionRuntime.ts b/apps/server/src/provider/acp/AcpSessionRuntime.ts index b2b0c5aee4a..2d5a0b6943d 100644 --- a/apps/server/src/provider/acp/AcpSessionRuntime.ts +++ b/apps/server/src/provider/acp/AcpSessionRuntime.ts @@ -1,3 +1,8 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; @@ -5,11 +10,13 @@ import * as Crypto from "effect/Crypto"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; @@ -20,6 +27,7 @@ import * as EffectAcpErrors from "effect-acp/errors"; import type * as EffectAcpSchema from "effect-acp/schema"; import type * as EffectAcpProtocol from "effect-acp/protocol"; import { resolveSpawnCommand } from "@t3tools/shared/shell"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { collectSessionConfigOptionValues, @@ -28,6 +36,7 @@ import { mergeToolCallState, parseSessionModeState, parseSessionUpdateEvent, + sessionUpdateCountsAsLoadReplayActivity, sessionUpdateIsReplay, waitForSessionLoadReplayIdle, type SessionLoadGate, @@ -64,6 +73,15 @@ export interface AcpSessionRuntimeOptions { readonly sessionLoadTimeout?: Duration.Input; readonly sessionLoadReplayIdleGap?: Duration.Input; readonly interruptPromptOnCancel?: boolean; + readonly ownDetachedProcessGroup?: boolean; + readonly ownDescendantProcessGroups?: boolean; + readonly processGroupPlatform?: NodeJS.Platform; + readonly processGroupTerminationGrace?: Duration.Input; + readonly windowsProcessTreeTerminator?: ( + pid: number, + ) => Effect.Effect; + readonly posixProcessTreeController?: AcpPosixProcessTreeController; + readonly linuxCgroupController?: AcpLinuxCgroupController | null; readonly clientCapabilities?: EffectAcpSchema.InitializeRequest["clientCapabilities"]; readonly clientInfo: { readonly name: string; @@ -77,6 +95,10 @@ export interface AcpSessionRuntimeOptions { readonly logOutgoing?: boolean; readonly logger?: (event: EffectAcpProtocol.AcpProtocolLogEvent) => Effect.Effect; }; + readonly onIncomingRequest?: EffectAcpClient.AcpClientOptions["onIncomingRequest"]; + readonly onTermination?: (error: EffectAcpErrors.AcpError) => Effect.Effect; + readonly onOutgoingResponseFailure?: EffectAcpClient.AcpClientOptions["onOutgoingResponseFailure"]; + readonly onOutgoingResponse?: EffectAcpClient.AcpClientOptions["onOutgoingResponse"]; } export interface AcpSessionRequestLogEvent { @@ -87,6 +109,925 @@ export interface AcpSessionRequestLogEvent { readonly cause?: Cause.Cause; } +export class AcpProcessGroupTerminationError extends Schema.TaggedErrorClass()( + "AcpProcessGroupTerminationError", + { + detail: Schema.String, + pid: Schema.optionalKey(Schema.Int), + exitCode: Schema.optionalKey(Schema.Number), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return this.detail; + } +} + +const isAcpProcessGroupTerminationError = Schema.is(AcpProcessGroupTerminationError); + +export interface AcpPosixProcessIdentity { + readonly executable: string | undefined; + readonly pgid: number; + readonly pid: number; + readonly ppid: number; + readonly sid: number; + /** + * Linux `/proc//stat` state when known (`R`/`S`/`D`/`Z`/…). Omitted on + * platforms or fixtures that do not surface it. Zombies (`Z`) are already + * dead and must not count as residual teardown survivors. + */ + readonly state?: string; + readonly startTime: string; +} + +export interface AcpPosixProcessTreeController { + readonly childPidsOf: (pid: number) => ReadonlyArray; + readonly childrenOf: (pid: number) => ReadonlyArray; + readonly identity: (pid: number) => AcpPosixProcessIdentity | undefined; + readonly snapshot: () => ReadonlyArray; + readonly signalProcess: (pid: number, signal: NodeJS.Signals) => void; +} + +export interface AcpLinuxCgroupLease { + readonly contains: (pid: number) => boolean; + readonly exists: () => boolean; + readonly path: string; + readonly relativePath: string; + readonly kill: () => void; + readonly populated: () => boolean; + readonly remove: () => void; +} + +export interface AcpLinuxCgroupController { + readonly create: () => AcpLinuxCgroupLease | undefined; +} + +export interface AcpPosixOwnershipRoot { + captureAttempted?: boolean; + value: AcpOwnedPosixProcess | undefined; +} + +function decodeMountInfoPath(value: string): string { + return value.replace(/\\([0-7]{3})/g, (_match, octal: string) => + String.fromCharCode(Number.parseInt(octal, 8)), + ); +} + +export function parseUnifiedCgroupPath(contents: string): string | undefined { + const matches = [...contents.matchAll(/^0::(\/.*)$/gm)]; + return matches.length === 1 ? matches[0]![1] : undefined; +} + +export function isSafeCgroupPath(value: string): boolean { + return ( + value.startsWith("/") && + !value.includes("(deleted)") && + NodePath.posix.normalize(value) === value + ); +} + +export function parseCgroup2Mounts( + contents: string, +): ReadonlyArray<{ readonly mountPoint: string; readonly root: string }> { + const mounts: Array<{ readonly mountPoint: string; readonly root: string }> = []; + for (const line of contents.split("\n")) { + const [mount, filesystem] = line.split(" - "); + if (mount === undefined || filesystem?.split(" ")[0] !== "cgroup2") continue; + const fields = mount.split(" "); + if (fields.length < 5) continue; + mounts.push({ + mountPoint: decodeMountInfoPath(fields[4]!), + root: decodeMountInfoPath(fields[3]!), + }); + } + return mounts; +} + +export function parseCgroup2Mount( + contents: string, +): { readonly mountPoint: string; readonly root: string } | undefined { + return parseCgroup2Mounts(contents)[0]; +} + +export function wrapCommandForLinuxCgroup( + lease: AcpLinuxCgroupLease, + command: string, + args: ReadonlyArray, +): { readonly command: string; readonly args: ReadonlyArray } { + return { + command: process.execPath, + args: [ + "-e", + [ + 'const fs = require("node:fs");', + "try {", + ' fs.writeFileSync(process.argv[1] + "/cgroup.procs", String(process.pid) + "\\n");', + ' const actual = fs.readFileSync("/proc/self/cgroup", "utf8").split("\\n").find((line) => line.startsWith("0::"))?.slice(3);', + " if (actual !== process.argv[2]) process.exit(126);", + " const env = { ...process.env };", + " delete env.ELECTRON_RUN_AS_NODE;", + " delete env.T3_ACP_CGROUP_WRAPPER;", + " process.execve(process.argv[3], process.argv.slice(3), env);", + "} catch { process.exit(125); }", + ].join("\n"), + lease.path, + lease.relativePath, + command, + ...args, + ], + }; +} + +export function resolveLinuxCgroupTargetCommand( + command: string, + cwd: string, + environment: NodeJS.ProcessEnv, +): string | undefined { + // Match Node spawn PATH fallback when PATH is undefined. An explicitly empty + // PATH stays empty (lookup only via empty-segment → cwd). + const pathEnv = environment.PATH === undefined ? "/usr/bin:/bin" : environment.PATH; + const candidates = command.includes(NodePath.sep) + ? [NodePath.resolve(cwd, command)] + : pathEnv + .split(NodePath.delimiter) + .map((entry) => + NodePath.join( + entry.length === 0 + ? cwd + : NodePath.isAbsolute(entry) + ? entry + : NodePath.resolve(cwd, entry), + command, + ), + ); + for (const candidate of candidates) { + try { + if (!NodeFS.statSync(candidate).isFile()) continue; + NodeFS.accessSync(candidate, NodeFS.constants.X_OK); + return candidate; + } catch { + // Try the next PATH entry. + } + } + return undefined; +} + +const STALE_ACP_CGROUP_SIBLING = /^t3-acp-(\d+)-/; + +export interface SweepStaleLinuxCgroupSiblingsOptions { + readonly currentPid?: number; + readonly isProcessAlive?: (pid: number) => boolean; + readonly readPopulated?: (siblingPath: string) => "0" | "1" | undefined; + readonly remove?: (siblingPath: string) => void; +} + +function defaultCgroupOwnerIsAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException | undefined)?.code !== "ESRCH"; + } +} + +function defaultReadCgroupPopulated(siblingPath: string): "0" | "1" | undefined { + const state = /(?:^|\n)populated ([01])(?:\n|$)/.exec( + NodeFS.readFileSync(NodePath.join(siblingPath, "cgroup.events"), "utf8"), + )?.[1]; + return state === "0" || state === "1" ? state : undefined; +} + +/** Best-effort removal of empty `t3-acp--*` sibling leases under a parent cgroup. */ +export function sweepStaleLinuxCgroupSiblings( + parentPath: string, + options: SweepStaleLinuxCgroupSiblingsOptions = {}, +): void { + const currentPid = options.currentPid ?? process.pid; + const isProcessAlive = options.isProcessAlive ?? defaultCgroupOwnerIsAlive; + const readPopulated = options.readPopulated ?? defaultReadCgroupPopulated; + const remove = options.remove ?? ((siblingPath: string) => NodeFS.rmdirSync(siblingPath)); + try { + for (const entry of NodeFS.readdirSync(parentPath, { withFileTypes: true })) { + try { + if (!entry.isDirectory()) continue; + const match = STALE_ACP_CGROUP_SIBLING.exec(entry.name); + if (match === null) continue; + const ownerPid = Number(match[1]); + if (!Number.isSafeInteger(ownerPid) || ownerPid <= 1 || ownerPid === currentPid) continue; + if (isProcessAlive(ownerPid)) continue; + const siblingPath = NodePath.join(parentPath, entry.name); + if (NodePath.dirname(siblingPath) !== parentPath) continue; + if (readPopulated(siblingPath) !== "0") continue; + remove(siblingPath); + } catch { + // Ignore unreadable or busy siblings; never fail lease creation. + } + } + } catch { + // Ignore readdir failures on the parent path. + } +} + +function tryCreateLinuxCgroupLease( + currentRelative: string, + mount: { readonly mountPoint: string; readonly root: string }, +): AcpLinuxCgroupLease | undefined { + if ( + !isSafeCgroupPath(mount.root) || + !isSafeCgroupPath(mount.mountPoint) || + !( + mount.root === "/" || + currentRelative === mount.root || + currentRelative.startsWith(`${mount.root}/`) + ) + ) { + return undefined; + } + const mountRelative = + mount.root === "/" ? currentRelative : currentRelative.slice(mount.root.length); + const currentPath = NodePath.resolve(mount.mountPoint, `.${mountRelative}`); + const resolvedMount = NodePath.resolve(mount.mountPoint); + if (currentPath !== resolvedMount && !currentPath.startsWith(`${resolvedMount}${NodePath.sep}`)) { + return undefined; + } + sweepStaleLinuxCgroupSiblings(currentPath); + const childName = `t3-acp-${process.pid}-${NodeCrypto.randomUUID().replaceAll("-", "")}`; + const childPath = NodePath.join(currentPath, childName); + const childRelative = NodePath.posix.join(currentRelative, childName); + if (NodePath.dirname(childPath) !== currentPath) return undefined; + try { + NodeFS.mkdirSync(childPath, { mode: 0o700 }); + NodeFS.accessSync(NodePath.join(childPath, "cgroup.procs"), NodeFS.constants.W_OK); + NodeFS.accessSync(NodePath.join(childPath, "cgroup.kill"), NodeFS.constants.W_OK); + NodeFS.accessSync(NodePath.join(childPath, "cgroup.events"), NodeFS.constants.R_OK); + if (NodeFS.readFileSync(NodePath.join(childPath, "cgroup.type"), "utf8").trim() !== "domain") { + throw new Error("ACP cgroup is not a domain cgroup"); + } + if (typeof process.execve !== "function") throw new Error("process.execve is unavailable"); + } catch { + try { + NodeFS.rmdirSync(childPath); + } catch { + // The unavailable cgroup was never used for a child process. + } + return undefined; + } + return { + contains: (pid) => + parseUnifiedCgroupPath(NodeFS.readFileSync(`/proc/${pid}/cgroup`, "utf8")) === childRelative, + exists: () => NodeFS.existsSync(childPath), + path: childPath, + relativePath: childRelative, + kill: () => NodeFS.writeFileSync(NodePath.join(childPath, "cgroup.kill"), "1\n"), + populated: () => { + const state = /(?:^|\n)populated ([01])(?:\n|$)/.exec( + NodeFS.readFileSync(NodePath.join(childPath, "cgroup.events"), "utf8"), + )?.[1]; + if (state === undefined) throw new Error("ACP cgroup.events has no populated state"); + return state === "1"; + }, + remove: () => { + const directories = (path: string): ReadonlyArray => + NodeFS.readdirSync(path, { withFileTypes: true }).flatMap((entry) => + entry.isDirectory() + ? [...directories(NodePath.join(path, entry.name)), NodePath.join(path, entry.name)] + : [], + ); + for (const directory of directories(childPath)) NodeFS.rmdirSync(directory); + NodeFS.rmdirSync(childPath); + }, + }; +} + +export function makeLinuxCgroupController(): AcpLinuxCgroupController { + return { + create: () => { + let currentRelative: string | undefined; + let mounts: ReadonlyArray<{ readonly mountPoint: string; readonly root: string }> = []; + try { + currentRelative = parseUnifiedCgroupPath(NodeFS.readFileSync("/proc/self/cgroup", "utf8")); + mounts = parseCgroup2Mounts(NodeFS.readFileSync("/proc/self/mountinfo", "utf8")); + } catch { + return undefined; + } + if ( + currentRelative === undefined || + !currentRelative.startsWith("/") || + mounts.length === 0 + ) { + return undefined; + } + if (!isSafeCgroupPath(currentRelative)) return undefined; + // Prefer the first mount that successfully creates a writable child cgroup. + // Multi-mount hosts often list a read-only view before the delegated one. + for (const mount of mounts) { + const lease = tryCreateLinuxCgroupLease(currentRelative, mount); + if (lease !== undefined) return lease; + } + return undefined; + }, + }; +} + +// OS-state polls (cgroup emptiness, process trees) must use wall time. Under +// TestClock, Effect.sleep never resolves unless the test advances the clock. +const wallClock = Clock.Clock.defaultValue(); +const withWallClock = (effect: Effect.Effect): Effect.Effect => + Effect.provideService(effect, Clock.Clock, wallClock); + +export function terminateLinuxCgroupLease( + lease: AcpLinuxCgroupLease, +): Effect.Effect { + const failure = (name: string, cause: unknown) => + new AcpProcessGroupTerminationError({ + cause, + detail: `Failed to ${name} ACP cgroup ${lease.path}`, + }); + return Effect.gen(function* () { + for (let attempt = 0; attempt < 200; attempt += 1) { + if (!lease.exists()) return; + try { + lease.kill(); + } catch (cause) { + const code = (cause as NodeJS.ErrnoException | undefined)?.code; + if (code !== "ENOENT") return yield* failure("kill", cause); + if (!lease.exists()) return; + yield* Effect.sleep("10 millis"); + continue; + } + let populated: boolean; + try { + populated = lease.populated(); + } catch (cause) { + const code = (cause as NodeJS.ErrnoException | undefined)?.code; + if (code === "ENOENT" && !lease.exists()) return; + if (code === "ENOENT") { + yield* Effect.sleep("10 millis"); + continue; + } + return yield* failure("read state for", cause); + } + if (populated) { + yield* Effect.sleep("10 millis"); + continue; + } + try { + lease.remove(); + } catch (cause) { + const code = (cause as NodeJS.ErrnoException | undefined)?.code; + if (code === "ENOENT" && !lease.exists()) return; + if (code !== "EBUSY" && code !== "ENOTEMPTY" && code !== "ENOENT") { + return yield* failure("remove", cause); + } + } + if (!lease.exists()) return; + yield* Effect.sleep("10 millis"); + } + return yield* new AcpProcessGroupTerminationError({ + detail: `ACP cgroup ${lease.path} remained populated after cgroup.kill`, + }); + }).pipe(withWallClock); +} + +export function signalLinuxCgroupRootTerm(input: { + readonly controller: AcpPosixProcessTreeController; + readonly lease: AcpLinuxCgroupLease; + readonly root: AcpPosixOwnershipRoot; +}): void { + const root = input.root.value; + if (root === undefined) return; + try { + const observed = input.controller.identity(root.pid); + if (!samePosixProcessIdentity(root, observed) || !input.lease.contains(root.pid)) return; + input.controller.signalProcess(root.pid, "SIGTERM"); + } catch { + // TERM is optional. cgroup.kill remains the authoritative teardown. + } +} + +function readLinuxProcessIdentity(pid: number): AcpPosixProcessIdentity | undefined { + try { + const stat = NodeFS.readFileSync(`/proc/${pid}/stat`, "utf8"); + const commandEnd = stat.lastIndexOf(")"); + if (commandEnd < 0) return undefined; + const fields = stat + .slice(commandEnd + 2) + .trim() + .split(/\s+/); + // After comm: state, ppid, pgrp, session, … starttime (field 22 / index 19). + const state = fields[0]; + const ppid = Number(fields[1]); + const pgid = Number(fields[2]); + const sid = Number(fields[3]); + const startTime = fields[19]; + if ( + state === undefined || + !Number.isSafeInteger(ppid) || + !Number.isSafeInteger(pgid) || + !Number.isSafeInteger(sid) || + startTime === undefined + ) { + return undefined; + } + let executable: string | undefined; + try { + executable = NodeFS.readlinkSync(`/proc/${pid}/exe`); + } catch { + executable = undefined; + } + return { executable, pgid, pid, ppid, sid, startTime, state }; + } catch { + return undefined; + } +} + +/** Linux zombies keep pid/starttime until reaped; they are not live survivors. */ +export function posixProcessIsZombie(process: AcpPosixProcessIdentity): boolean { + return process.state === "Z"; +} + +function readLinuxChildPids(pid: number): ReadonlyArray { + const childPids = new Set(); + let taskIds: ReadonlyArray = []; + try { + taskIds = NodeFS.readdirSync(`/proc/${pid}/task`).filter((entry) => + /^[1-9][0-9]*$/.test(entry), + ); + } catch { + return []; + } + for (const taskId of taskIds) { + try { + for (const childPid of NodeFS.readFileSync(`/proc/${pid}/task/${taskId}/children`, "utf8") + .trim() + .split(/\s+/) + .filter(Boolean) + .map(Number)) { + childPids.add(childPid); + } + } catch { + // The task exited while its children were being read. + } + } + return [...childPids]; +} + +function makeLinuxProcessTreeController(): AcpPosixProcessTreeController { + return { + childPidsOf: readLinuxChildPids, + childrenOf: (pid) => + readLinuxChildPids(pid).flatMap((childPid) => { + const identity = readLinuxProcessIdentity(childPid); + return identity === undefined ? [] : [identity]; + }), + identity: readLinuxProcessIdentity, + snapshot: () => + NodeFS.readdirSync("/proc", { withFileTypes: true }).flatMap((entry) => { + if (!entry.isDirectory() || !/^[1-9][0-9]*$/.test(entry.name)) return []; + const identity = readLinuxProcessIdentity(Number(entry.name)); + return identity === undefined ? [] : [identity]; + }), + signalProcess: (pid, signal) => process.kill(pid, signal), + }; +} + +export function observePosixOwnershipLedger(input: { + readonly childQueues: Map>; + readonly controller: AcpPosixProcessTreeController; + readonly frontier: Map; + readonly ledger: Map; + readonly maxProcesses?: number; + readonly root: AcpPosixOwnershipRoot; + readonly rootPid: number; +}): void { + let rootIdentity: AcpPosixProcessIdentity | undefined = input.root.value; + if (rootIdentity === undefined) { + if (input.root.captureAttempted === true) { + throw new AcpProcessGroupTerminationError({ + detail: `ACP process ${input.rootPid} exited before its ownership ledger was captured`, + }); + } + input.root.captureAttempted = true; + rootIdentity = input.controller.identity(input.rootPid); + if (rootIdentity === undefined) { + throw new AcpProcessGroupTerminationError({ + detail: `ACP process ${input.rootPid} exited before its ownership ledger was captured`, + }); + } + const parent = input.controller.identity(rootIdentity.ppid); + const ownedRoot = { + ...rootIdentity, + parentExecutable: parent?.executable, + parentStartTime: parent?.startTime ?? "", + }; + input.ledger.set(processIdentityKey(rootIdentity), ownedRoot); + input.frontier.set(rootIdentity.pid, ownedRoot); + input.root.value = ownedRoot; + } + const maxProcesses = Math.max(1, input.maxProcesses ?? Number.POSITIVE_INFINITY); + let remainingBudget = maxProcesses; + const pending = [...input.frontier.entries()]; + const visited = new Set(); + while (pending.length > 0 && remainingBudget > 0) { + const entry = pending.shift(); + if (entry === undefined) continue; + const [parentPid, parent] = entry; + input.frontier.delete(parentPid); + if (visited.has(parent.pid)) continue; + visited.add(parent.pid); + remainingBudget -= 1; + const observedParent = input.controller.identity(parent.pid); + if (!samePosixProcessIdentity(parent, observedParent)) { + input.childQueues.delete(parent.pid); + continue; + } + const refreshedParent = { ...parent, ...observedParent }; + input.ledger.set(processIdentityKey(refreshedParent), refreshedParent); + input.frontier.set(refreshedParent.pid, refreshedParent); + let queuedChildren = input.childQueues.get(refreshedParent.pid); + if (queuedChildren === undefined) { + if (remainingBudget === 0) continue; + remainingBudget -= 1; + queuedChildren = [...input.controller.childPidsOf(refreshedParent.pid)]; + } + while (queuedChildren.length > 0 && remainingBudget > 0) { + const childPid = queuedChildren.shift(); + if (childPid === undefined) continue; + remainingBudget -= 1; + const child = input.controller.identity(childPid); + if (child === undefined) continue; + if (child.ppid !== parent.pid) continue; + let reservedPid = input.frontier.get(child.pid); + if (reservedPid !== undefined && !samePosixProcessIdentity(reservedPid, child)) { + // PID reused under an owned parent; drop the stale reservation. + input.frontier.delete(child.pid); + for (const [key, owned] of [...input.ledger.entries()]) { + if (owned.pid === child.pid) input.ledger.delete(key); + } + reservedPid = undefined; + } + const childKey = processIdentityKey(child); + const owned = + reservedPid === undefined + ? { + ...child, + parentExecutable: refreshedParent.executable, + parentStartTime: refreshedParent.startTime, + } + : { ...reservedPid, ...child }; + input.ledger.set(childKey, owned); + input.frontier.set(child.pid, owned); + } + if (queuedChildren.length === 0) input.childQueues.delete(refreshedParent.pid); + else input.childQueues.set(refreshedParent.pid, queuedChildren); + } +} + +export function observePosixOwnershipLedgerContinuously(input: { + readonly childQueues: Map>; + readonly controller: AcpPosixProcessTreeController; + readonly frontier: Map; + readonly ledger: Map; + readonly root: AcpPosixOwnershipRoot; + readonly rootPid: number; +}): Effect.Effect { + return Effect.forever( + Effect.gen(function* () { + const startedAt = yield* Clock.currentTimeMillis; + yield* Effect.sync(() => observePosixOwnershipLedger({ ...input, maxProcesses: 64 })).pipe( + Effect.exit, + ); + const targetInterval = input.frontier.size <= 32 ? 25 : input.frontier.size <= 128 ? 50 : 100; + const completedAt = yield* Clock.currentTimeMillis; + yield* Effect.sleep(`${Math.max(5, targetInterval - (completedAt - startedAt))} millis`); + }), + ).pipe(withWallClock); +} + +export function makePosixProcessTreeController( + platform: NodeJS.Platform, +): AcpPosixProcessTreeController { + if (platform === "linux") return makeLinuxProcessTreeController(); + throw new AcpProcessGroupTerminationError({ + detail: `Detached ACP descendant ownership is unsupported on ${platform}`, + }); +} + +function samePosixProcessIdentity( + expected: AcpPosixProcessIdentity, + observed: AcpPosixProcessIdentity | undefined, +): boolean { + return ( + observed !== undefined && + observed.pid === expected.pid && + observed.startTime === expected.startTime + ); +} + +export interface AcpOwnedPosixProcess extends AcpPosixProcessIdentity { + readonly parentExecutable: string | undefined; + readonly parentStartTime: string; +} + +function processIdentityKey(process: AcpPosixProcessIdentity): string { + return `${process.pid}:${process.startTime}`; +} + +function processDepth( + process: AcpPosixProcessIdentity, + ledgerByPid: ReadonlyMap, +): number { + let depth = 0; + let current = process; + const seen = new Set(); + while (!seen.has(current.pid)) { + seen.add(current.pid); + const parent = ledgerByPid.get(current.ppid); + if (parent === undefined) break; + depth += 1; + current = parent; + } + return depth; +} + +export function capturePosixOwnershipLedger(input: { + readonly controller: AcpPosixProcessTreeController; + readonly ledger: Map; + readonly root?: AcpPosixOwnershipRoot; + readonly rootPid: number; + readonly table?: ReadonlyArray; +}): ReadonlyArray { + let table: ReadonlyArray; + if (input.table === undefined) { + try { + table = input.controller.snapshot(); + } catch (cause) { + throw new AcpProcessGroupTerminationError({ + cause, + detail: `Failed to snapshot ACP process tree ${input.rootPid}`, + }); + } + } else { + table = input.table; + } + const byPid = new Map(table.map((process) => [process.pid, process])); + let rootIdentity: AcpPosixProcessIdentity | undefined = + input.root?.value ?? + [...input.ledger.values()].find((process) => process.pid === input.rootPid); + if (rootIdentity === undefined) { + rootIdentity = byPid.get(input.rootPid); + if (rootIdentity === undefined) { + throw new AcpProcessGroupTerminationError({ + detail: `ACP process ${input.rootPid} exited before its ownership ledger was captured`, + }); + } + const ownedRoot = { + ...rootIdentity, + parentExecutable: byPid.get(rootIdentity.ppid)?.executable, + parentStartTime: byPid.get(rootIdentity.ppid)?.startTime ?? "", + }; + input.ledger.set(processIdentityKey(rootIdentity), ownedRoot); + if (input.root !== undefined) input.root.value = ownedRoot; + } + // Drop non-root ledger entries whose PID is free or whose identity no longer + // matches the live process at that PID. Either case can block admission of a + // legitimate child after PID reuse. Keep the root reservation even if the + // root briefly disappears mid-teardown (finalizer retry still needs it). + for (const [key, owned] of [...input.ledger.entries()]) { + if (owned.pid === input.rootPid) continue; + const live = byPid.get(owned.pid); + if (live === undefined || !samePosixProcessIdentity(owned, live)) { + input.ledger.delete(key); + } + } + const children = new Map>(); + const retainedByPid = new Map( + [...input.ledger.values()] + .filter((owned) => samePosixProcessIdentity(owned, byPid.get(owned.pid))) + .map((owned) => [owned.pid, owned]), + ); + for (const process of table) { + const siblings = children.get(process.ppid) ?? []; + siblings.push(process); + children.set(process.ppid, siblings); + } + const pending = [...input.ledger.values()].filter((owned) => + samePosixProcessIdentity(owned, byPid.get(owned.pid)), + ); + const visited = new Set(); + while (pending.length > 0) { + const parent = pending.shift(); + if (parent === undefined || visited.has(parent.pid)) continue; + visited.add(parent.pid); + const observedParent = byPid.get(parent.pid); + if (!samePosixProcessIdentity(parent, observedParent)) continue; + const refreshedParent = { ...parent, ...observedParent }; + input.ledger.set(processIdentityKey(parent), refreshedParent); + for (const process of children.get(refreshedParent.pid) ?? []) { + const reservedPid = retainedByPid.get(process.pid); + if (reservedPid !== undefined && !samePosixProcessIdentity(reservedPid, process)) continue; + const owned = + reservedPid === undefined + ? { + ...process, + parentExecutable: refreshedParent.executable, + parentStartTime: refreshedParent.startTime, + } + : { ...reservedPid, ...process }; + input.ledger.set(processIdentityKey(process), owned); + retainedByPid.set(process.pid, owned); + pending.push(owned); + } + } + return table; +} + +export function terminatePosixOwnedProcessTree(input: { + readonly controller: AcpPosixProcessTreeController; + readonly discoveryPasses?: number; + readonly grace?: Duration.Input; + readonly ledger?: Map; + readonly root?: AcpPosixOwnershipRoot; + readonly rootPid: number; +}): Effect.Effect { + // This portable fallback can only admit processes whose exact parent chain is + // visible before it breaks. A descendant that double-forks before discovery + // requires pre-exec cgroup containment for a hard ownership guarantee. + const ledger = input.ledger ?? new Map(); + + const fail = (detail: string, cause?: unknown) => + new AcpProcessGroupTerminationError({ detail, ...(cause === undefined ? {} : { cause }) }); + const snapshot = () => { + try { + return input.controller.snapshot(); + } catch (cause) { + throw fail(`Failed to snapshot ACP process tree ${input.rootPid}`, cause); + } + }; + const discover = (table: ReadonlyArray) => + capturePosixOwnershipLedger({ + controller: input.controller, + ledger, + ...(input.root === undefined ? {} : { root: input.root }), + rootPid: input.rootPid, + table, + }); + const signalPhase = (signal: NodeJS.Signals, includeRoot: boolean) => { + const table = snapshot(); + discover(table); + const byPid = new Map(table.map((entry) => [entry.pid, entry])); + const current = input.controller.identity(process.pid); + if (current === undefined) throw fail("Cannot identify the current T3 process group"); + const ledgerByPid = new Map( + [...ledger.values()].map((process) => [process.pid, process] as const), + ); + const pending = [...ledger.values()].filter( + (candidate) => + (includeRoot || candidate.pid !== input.rootPid) && + samePosixProcessIdentity(candidate, byPid.get(candidate.pid)), + ); + const retainedByPid = new Map( + [...ledger.values()] + .filter((owned) => samePosixProcessIdentity(owned, byPid.get(owned.pid))) + .map((owned) => [owned.pid, owned]), + ); + const signalled = new Set(); + while (pending.length > 0) { + pending.sort( + (left, right) => processDepth(right, ledgerByPid) - processDepth(left, ledgerByPid), + ); + const candidate = pending.shift(); + if (candidate === undefined || signalled.has(candidate.pid)) continue; + const observed = input.controller.identity(candidate.pid); + if ( + candidate.pid <= 1 || + candidate.pid === process.pid || + observed === undefined || + observed.pgid === current.pgid || + observed.sid === current.sid || + !samePosixProcessIdentity(candidate, observed) + ) { + continue; + } + for (const child of input.controller.childrenOf(candidate.pid)) { + if (child.ppid !== candidate.pid) continue; + const reservedPid = retainedByPid.get(child.pid); + if (reservedPid !== undefined && !samePosixProcessIdentity(reservedPid, child)) continue; + const owned = + reservedPid === undefined + ? { + ...child, + parentExecutable: observed.executable, + parentStartTime: observed.startTime, + } + : { ...reservedPid, ...child }; + ledger.set(processIdentityKey(child), owned); + ledgerByPid.set(child.pid, owned); + retainedByPid.set(child.pid, owned); + // Signal newly discovered children in this same pass (fork-during-kill). + if ( + (includeRoot || child.pid !== input.rootPid) && + !signalled.has(child.pid) && + samePosixProcessIdentity(owned, input.controller.identity(child.pid)) + ) { + pending.push(owned); + } + } + try { + // Linux starttime prevents PID reuse across observations, but Node does + // not expose pidfd_send_signal. A final read-to-kill race remains. + input.controller.signalProcess(candidate.pid, signal); + signalled.add(candidate.pid); + } catch (cause) { + if (samePosixProcessIdentity(candidate, input.controller.identity(candidate.pid))) { + throw fail(`Failed to signal ACP process ${candidate.pid}`, cause); + } + } + } + }; + + return Effect.gen(function* () { + const passes = Math.max(2, Math.min(input.discoveryPasses ?? 4, 8)); + for (let pass = 0; pass < passes; pass += 1) { + yield* Effect.try({ + try: () => signalPhase("SIGTERM", false), + catch: (cause) => cause as AcpProcessGroupTerminationError, + }); + yield* Effect.sleep("10 millis"); + } + yield* Effect.try({ + try: () => signalPhase("SIGTERM", true), + catch: (cause) => cause as AcpProcessGroupTerminationError, + }); + const grace = input.grace ?? "1 second"; + if (grace !== 0) yield* Effect.sleep(grace); + for (let pass = 0; pass < passes; pass += 1) { + yield* Effect.try({ + try: () => signalPhase("SIGKILL", true), + catch: (cause) => cause as AcpProcessGroupTerminationError, + }); + yield* Effect.sleep("10 millis"); + } + const survivors = new Map(snapshot().map((entry) => [entry.pid, entry])); + const residual = [...ledger.values()].filter((owned) => { + const observed = survivors.get(owned.pid); + if (!samePosixProcessIdentity(owned, observed) || observed === undefined) { + return false; + } + // Already-killed zombies still appear in /proc until reaped; not live. + return !posixProcessIsZombie(observed); + }); + if (residual.length > 0) { + return yield* fail( + `ACP process tree ${input.rootPid} retained owned processes: ${residual.map((entry) => entry.pid).join(", ")}`, + ); + } + }).pipe(withWallClock); +} + +export function windowsTaskkillResultIsSuccess(exitCode: number, _output?: string): boolean { + return exitCode === 0; +} + +export const terminateWindowsProcessTreeWithTaskkill = ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + pid: number, +): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const taskkill = yield* spawner.spawn( + ChildProcess.make("taskkill", ["/PID", String(pid), "/T", "/F"]), + ); + const outputFiber = yield* Stream.decodeText(taskkill.all).pipe( + Stream.mkString, + Effect.forkScoped, + ); + const exitCode = Number(yield* taskkill.exitCode); + const output = yield* Fiber.join(outputFiber); + // A missing leader cannot prove that inherited descendants exited. + if (windowsTaskkillResultIsSuccess(exitCode, output)) return; + const trimmedOutput = output.trim(); + return yield* new AcpProcessGroupTerminationError({ + detail: `taskkill exited ${exitCode} for ACP process tree ${pid}`, + pid, + exitCode, + ...(trimmedOutput.length === 0 + ? {} + : { cause: trimmedOutput.length > 500 ? trimmedOutput.slice(0, 500) : trimmedOutput }), + }); + }), + ).pipe( + Effect.mapError((cause) => + isAcpProcessGroupTerminationError(cause) + ? cause + : new AcpProcessGroupTerminationError({ + cause, + detail: `Failed to run taskkill for ACP process tree ${pid}`, + pid, + }), + ), + ); + export function selectAcpAgentAuthMethod( authMethods: ReadonlyArray | undefined, preferredMethodId?: string, @@ -231,6 +1172,13 @@ export class AcpSessionRuntime extends Context.Service< * @see https://agentclientprotocol.com/protocol/schema#session/cancel */ readonly cancel: Effect.Effect; + readonly processContainment: + | "cgroup-v2" + | "process-ledger-reduced-guarantee" + | "process-group" + | "none"; + /** Terminates the opt-in detached ACP process group and all inherited work. */ + readonly terminateProcessGroup?: Effect.Effect; /** * Selects the active mode through the negotiated `mode` configuration option. * This is a no-op when the requested mode is already active. @@ -335,6 +1283,7 @@ export const make = ( const configOptionsRef = yield* Ref.make(sessionConfigOptionsFromSetup(undefined)); const startStateRef = yield* Ref.make({ _tag: "NotStarted" }); const promptSerializationSemaphore = yield* Semaphore.make(1); + const sessionLoadSemaphore = yield* Semaphore.make(1); const activePromptFiberRef = yield* Ref.make< Option.Option> >(Option.none()); @@ -376,12 +1325,70 @@ export const make = ( options.spawn.args, options.spawn.env ? { env: options.spawn.env, extendEnv: true } : {}, ); + const linuxCgroupLease = + options.ownDescendantProcessGroups === true && options.processGroupPlatform === "linux" + ? yield* Effect.sync(() => { + try { + if (options.linuxCgroupController === null) return undefined; + return (options.linuxCgroupController ?? makeLinuxCgroupController()).create(); + } catch { + return undefined; + } + }) + : undefined; + // This is lifecycle containment for inherited provider work. It is not a + // security boundary against a provider that can modify its delegated cgroup. + if (linuxCgroupLease !== undefined) { + yield* Scope.addFinalizer( + runtimeScope, + Effect.uninterruptible(terminateLinuxCgroupLease(linuxCgroupLease)).pipe( + Effect.ignoreCause({ log: true }), + ), + ); + } + const containedTargetCommand = + linuxCgroupLease === undefined || NodePath.isAbsolute(spawnCommand.command) + ? spawnCommand.command + : yield* Effect.gen(function* () { + const resolved = resolveLinuxCgroupTargetCommand( + spawnCommand.command, + options.spawn.cwd ?? options.cwd, + { ...process.env, ...options.spawn.env }, + ); + if (resolved !== undefined) return resolved; + return yield* new EffectAcpErrors.AcpSpawnError({ + command: options.spawn.command, + cause: new Error("Contained ACP command was not found on PATH"), + }); + }); + const containedSpawnCommand = + linuxCgroupLease === undefined + ? spawnCommand + : { + ...wrapCommandForLinuxCgroup( + linuxCgroupLease, + containedTargetCommand, + spawnCommand.args, + ), + shell: false, + }; + const spawnEnvironment = + linuxCgroupLease === undefined + ? options.spawn.env + : { + ...options.spawn.env, + ELECTRON_RUN_AS_NODE: "1", + T3_ACP_CGROUP_WRAPPER: "1", + }; const child = yield* spawner .spawn( - ChildProcess.make(spawnCommand.command, spawnCommand.args, { + ChildProcess.make(containedSpawnCommand.command, containedSpawnCommand.args, { ...(options.spawn.cwd ? { cwd: options.spawn.cwd } : {}), - ...(options.spawn.env ? { env: options.spawn.env, extendEnv: true } : {}), - shell: spawnCommand.shell, + ...(spawnEnvironment ? { env: spawnEnvironment, extendEnv: true } : {}), + ...(options.ownDetachedProcessGroup === undefined + ? {} + : { detached: options.ownDetachedProcessGroup }), + shell: containedSpawnCommand.shell, }), ) .pipe( @@ -395,6 +1402,155 @@ export const make = ( ), ); + const posixOwnershipLedger = new Map(); + const posixOwnershipFrontier = new Map(); + const posixOwnershipChildQueues = new Map>(); + const posixOwnershipRoot: AcpPosixOwnershipRoot = { value: undefined }; + const posixController = + options.ownDescendantProcessGroups === true + ? yield* Effect.try({ + try: () => { + if (options.posixProcessTreeController !== undefined) { + return options.posixProcessTreeController; + } + if (options.processGroupPlatform === undefined) { + throw new AcpProcessGroupTerminationError({ + detail: "POSIX ACP descendant ownership requires a supported host platform", + }); + } + return makePosixProcessTreeController(options.processGroupPlatform); + }, + catch: (cause) => + new EffectAcpErrors.AcpSpawnError({ command: options.spawn.command, cause }), + }) + : undefined; + if (posixController !== undefined) { + const observeOwnership = () => + observePosixOwnershipLedger({ + childQueues: posixOwnershipChildQueues, + controller: posixController, + frontier: posixOwnershipFrontier, + ledger: posixOwnershipLedger, + root: posixOwnershipRoot, + rootPid: Number(child.pid), + }); + yield* Effect.try({ + try: observeOwnership, + catch: (cause) => + new EffectAcpErrors.AcpSpawnError({ command: options.spawn.command, cause }), + }); + yield* observePosixOwnershipLedgerContinuously({ + childQueues: posixOwnershipChildQueues, + controller: posixController, + frontier: posixOwnershipFrontier, + ledger: posixOwnershipLedger, + root: posixOwnershipRoot, + rootPid: Number(child.pid), + }).pipe(Effect.forkIn(runtimeScope)); + } + + const signalOwnedProcessGroup = (signal: NodeJS.Signals) => + Effect.try({ + try: () => { + process.kill(-Number(child.pid), signal); + return true; + }, + catch: (cause) => + new AcpProcessGroupTerminationError({ + cause, + detail: `Failed to signal ACP process group ${child.pid} with ${signal}`, + }), + }).pipe( + Effect.catch((error) => { + const cause = error.cause as NodeJS.ErrnoException | undefined; + return cause?.code === "ESRCH" ? Effect.succeed(false) : Effect.fail(error); + }), + ); + const terminateWindowsProcessTree = + options.windowsProcessTreeTerminator ?? + ((pid: number) => terminateWindowsProcessTreeWithTaskkill(spawner, pid)); + const terminatePosixProcessTree = (grace: Duration.Input, platformOverride?: NodeJS.Platform) => + Effect.gen(function* () { + const platform = platformOverride ?? options.processGroupPlatform; + if (platform === undefined || platform === "win32") { + return yield* new AcpProcessGroupTerminationError({ + detail: "POSIX ACP descendant ownership requires a supported host platform", + }); + } + const controller = yield* Effect.try({ + try: () => posixController ?? makePosixProcessTreeController(platform), + catch: (cause) => + isAcpProcessGroupTerminationError(cause) + ? cause + : new AcpProcessGroupTerminationError({ + cause, + detail: `Failed to initialize ACP descendant ownership on ${platform}`, + }), + }); + yield* terminatePosixOwnedProcessTree({ + controller, + grace, + ledger: posixOwnershipLedger, + root: posixOwnershipRoot, + rootPid: Number(child.pid), + }); + }); + const terminateOwnedProcessGroupImpl = Effect.gen(function* () { + if (options.ownDetachedProcessGroup !== true) return; + if (options.processGroupPlatform === undefined) { + return yield* new AcpProcessGroupTerminationError({ + detail: "Detached ACP process-group ownership requires a host platform", + }); + } + if (options.processGroupPlatform === "win32") { + return yield* terminateWindowsProcessTree(Number(child.pid)); + } + if (linuxCgroupLease !== undefined) { + if (posixController !== undefined) { + yield* Effect.sync(() => + signalLinuxCgroupRootTerm({ + controller: posixController, + lease: linuxCgroupLease, + root: posixOwnershipRoot, + }), + ); + } + const grace = options.processGroupTerminationGrace ?? "1 second"; + if (grace !== 0) yield* Effect.sleep(grace); + return yield* terminateLinuxCgroupLease(linuxCgroupLease); + } + if (options.ownDescendantProcessGroups === true) { + return yield* terminatePosixProcessTree(options.processGroupTerminationGrace ?? "1 second"); + } + const groupExisted = yield* signalOwnedProcessGroup("SIGTERM"); + if (!groupExisted) return; + const grace = options.processGroupTerminationGrace ?? "1 second"; + if (grace !== 0) { + yield* Effect.sleep(grace); + } + yield* signalOwnedProcessGroup("SIGKILL"); + }).pipe(withWallClock); + const terminateProcessGroup = yield* Effect.cached( + Effect.uninterruptible(terminateOwnedProcessGroupImpl), + ); + if (options.ownDetachedProcessGroup === true) { + const hostPlatform = yield* HostProcessPlatform; + const forceTerminateOwnedProcessGroup = + hostPlatform === "win32" + ? terminateWindowsProcessTreeWithTaskkill(spawner, Number(child.pid)) + : linuxCgroupLease !== undefined + ? terminateLinuxCgroupLease(linuxCgroupLease) + : options.ownDescendantProcessGroups === true + ? terminatePosixProcessTree(0, hostPlatform) + : signalOwnedProcessGroup("SIGKILL").pipe(Effect.asVoid); + yield* Scope.addFinalizer( + runtimeScope, + Effect.uninterruptible(forceTerminateOwnedProcessGroup).pipe( + Effect.ignoreCause({ log: true }), + ), + ); + } + const acpContext = yield* Layer.build( EffectAcpClient.layerChildProcess(child, { ...(options.protocolLogging?.logIncoming !== undefined @@ -404,6 +1560,17 @@ export const make = ( ? { logOutgoing: options.protocolLogging.logOutgoing } : {}), ...(options.protocolLogging?.logger ? { logger: options.protocolLogging.logger } : {}), + ...(options.onIncomingRequest ? { onIncomingRequest: options.onIncomingRequest } : {}), + onTermination: (error) => + (options.onTermination?.(error) ?? Effect.void).pipe( + Effect.ensuring( + Scope.close(runtimeScope, Exit.fail(error)).pipe(Effect.forkDetach, Effect.asVoid), + ), + ), + ...(options.onOutgoingResponseFailure + ? { onOutgoingResponseFailure: options.onOutgoingResponseFailure } + : {}), + ...(options.onOutgoingResponse ? { onOutgoingResponse: options.onOutgoingResponse } : {}), }), ).pipe(Effect.provideService(Scope.Scope, runtimeScope)); @@ -413,14 +1580,16 @@ export const make = ( Effect.gen(function* () { const gate = yield* Ref.get(sessionLoadGateRef); if (Option.isSome(gate) && gate.value.active) { - const lastActivityAtMillis = yield* Clock.currentTimeMillis; - yield* Ref.set( - sessionLoadGateRef, - Option.some({ - ...gate.value, - lastActivityAtMillis, - }), - ); + if (sessionUpdateCountsAsLoadReplayActivity(notification, gate.value.sessionId)) { + const lastActivityAtMillis = yield* Clock.currentTimeMillis; + yield* Ref.set( + sessionLoadGateRef, + Option.some({ + ...gate.value, + lastActivityAtMillis, + }), + ); + } return; } if (sessionUpdateIsReplay(notification)) { @@ -543,20 +1712,121 @@ export const make = ( ): Effect.Effect => Effect.gen(function* () { const current = yield* getStartedState; + const meta = sessionSetupResult._meta; + const syntheticReplayIdle = + meta !== null && + typeof meta === "object" && + !Array.isArray(meta) && + (meta as { readonly t3SessionLoadReady?: unknown }).t3SessionLoadReady === "replay_idle"; + const extractedModelConfigId = extractModelConfigId(sessionSetupResult); + const nextModelConfigId = + extractedModelConfigId ?? (syntheticReplayIdle ? current.modelConfigId : undefined); + const nextModeState = parseSessionModeState(sessionSetupResult); + if (nextModeState !== undefined) { + yield* Ref.set(modeStateRef, nextModeState); + } else if (!syntheticReplayIdle) { + yield* Ref.set(modeStateRef, undefined); + } + // Synthetic replay-idle load responses only carry initialize model/mode + // meta; preserve live configOptions so setConfigOption still validates. + if ( + sessionSetupResult.configOptions !== undefined && + sessionSetupResult.configOptions !== null + ) { + yield* Ref.set(configOptionsRef, sessionConfigOptionsFromSetup(sessionSetupResult)); + } else if (!syntheticReplayIdle) { + yield* Ref.set(configOptionsRef, sessionConfigOptionsFromSetup(sessionSetupResult)); + } const nextState = { sessionId, initializeResult: current.initializeResult, sessionSetupResult, - modelConfigId: extractModelConfigId(sessionSetupResult), + modelConfigId: nextModelConfigId, } satisfies AcpStartedState; - yield* Ref.set(modeStateRef, parseSessionModeState(sessionSetupResult)); - yield* Ref.set(configOptionsRef, sessionConfigOptionsFromSetup(sessionSetupResult)); yield* Ref.set(toolCallsRef, new Map()); yield* Ref.set(assistantSegmentRef, { nextSegmentIndex: 0 }); yield* Ref.set(startStateRef, { _tag: "Started", result: nextState }); return nextState; }); + const runLoadSessionWithReplayIdle = ( + loadPayload: EffectAcpSchema.LoadSessionRequest, + initializeResult: EffectAcpSchema.InitializeResponse, + ): Effect.Effect => + sessionLoadSemaphore.withPermits(1)( + Effect.gen(function* () { + const sessionLoadTimeout = Duration.fromInputUnsafe( + options.sessionLoadTimeout ?? defaultSessionLoadTimeout, + ); + const sessionLoadReplayIdleGap = Duration.fromInputUnsafe( + options.sessionLoadReplayIdleGap ?? defaultSessionLoadReplayIdleGap, + ); + + yield* Ref.set( + sessionLoadGateRef, + Option.some({ + active: true, + sessionId: loadPayload.sessionId, + lastActivityAtMillis: undefined, + idleGap: sessionLoadReplayIdleGap, + initializeResult, + }), + ); + + return yield* Effect.gen(function* () { + yield* logRequest({ + method: "session/load", + payload: loadPayload, + status: "started", + }); + + const idleFiber = yield* waitForSessionLoadReplayIdle({ + gateRef: sessionLoadGateRef, + }).pipe(Effect.forkIn(runtimeScope)); + const loaded = yield* Effect.raceFirst( + acp.agent.loadSession(loadPayload), + Fiber.join(idleFiber), + ).pipe( + Effect.ensuring(Fiber.interrupt(idleFiber).pipe(Effect.ignore)), + Effect.timeoutOption(sessionLoadTimeout), + Effect.flatMap((result) => + Option.match(result, { + onNone: () => + Effect.fail( + new EffectAcpErrors.AcpTransportError({ + operation: "call-rpc", + method: "session/load", + detail: + "session/load timed out waiting for RPC response or replay idle gap", + cause: undefined, + }), + ), + onSome: Effect.succeed, + }), + ), + Effect.tap((result) => + logRequest({ + method: "session/load", + payload: loadPayload, + status: "succeeded", + result, + }), + ), + Effect.onError((cause) => + logRequest({ + method: "session/load", + payload: loadPayload, + status: "failed", + cause, + }), + ), + ); + + return loaded; + }).pipe(Effect.ensuring(Ref.set(sessionLoadGateRef, Option.none()))); + }), + ); + const setConfigOption = ( configId: string, value: string | boolean, @@ -661,75 +1931,9 @@ export const make = ( cwd: options.cwd, mcpServers: options.mcpServers ?? [], } satisfies EffectAcpSchema.LoadSessionRequest; - const sessionLoadTimeout = Duration.fromInputUnsafe( - options.sessionLoadTimeout ?? defaultSessionLoadTimeout, - ); - const sessionLoadReplayIdleGap = Duration.fromInputUnsafe( - options.sessionLoadReplayIdleGap ?? defaultSessionLoadReplayIdleGap, - ); - - yield* Ref.set( - sessionLoadGateRef, - Option.some({ - active: true, - lastActivityAtMillis: undefined, - idleGap: sessionLoadReplayIdleGap, - initializeResult, - }), - ); sessionId = options.resumeSessionId; - sessionSetupResult = yield* Effect.gen(function* () { - yield* logRequest({ - method: "session/load", - payload: loadPayload, - status: "started", - }); - - const idleFiber = yield* waitForSessionLoadReplayIdle({ - gateRef: sessionLoadGateRef, - }).pipe(Effect.forkIn(runtimeScope)); - const loaded = yield* Effect.raceFirst( - acp.agent.loadSession(loadPayload), - Fiber.join(idleFiber), - ).pipe( - Effect.ensuring(Fiber.interrupt(idleFiber).pipe(Effect.ignore)), - Effect.timeoutOption(sessionLoadTimeout), - Effect.flatMap((result) => - Option.match(result, { - onNone: () => - Effect.fail( - new EffectAcpErrors.AcpTransportError({ - operation: "call-rpc", - method: "session/load", - detail: - "session/load timed out waiting for RPC response or replay idle gap", - cause: undefined, - }), - ), - onSome: Effect.succeed, - }), - ), - Effect.tap((result) => - logRequest({ - method: "session/load", - payload: loadPayload, - status: "succeeded", - result, - }), - ), - Effect.onError((cause) => - logRequest({ - method: "session/load", - payload: loadPayload, - status: "failed", - cause, - }), - ), - ); - - return loaded; - }).pipe(Effect.ensuring(Ref.set(sessionLoadGateRef, Option.none()))); + sessionSetupResult = yield* runLoadSessionWithReplayIdle(loadPayload, initializeResult); } else { const createPayload = { cwd: options.cwd, @@ -800,6 +2004,14 @@ export const make = ( }); return { + processContainment: + linuxCgroupLease !== undefined + ? "cgroup-v2" + : options.ownDescendantProcessGroups === true + ? "process-ledger-reduced-guarantee" + : options.ownDetachedProcessGroup === true + ? "process-group" + : "none", handleRequestPermission: acp.handleRequestPermission, handleElicitation: acp.handleElicitation, handleReadTextFile: acp.handleReadTextFile, @@ -829,17 +2041,13 @@ export const make = ( getConfigOptions: Ref.get(configOptionsRef), loadSession: (sessionId) => start.pipe( - Effect.flatMap(() => { + Effect.flatMap((started) => { const requestPayload = { sessionId, cwd: options.cwd, mcpServers: options.mcpServers ?? [], } satisfies EffectAcpSchema.LoadSessionRequest; - return runLoggedRequest( - "session/load", - requestPayload, - acp.agent.loadSession(requestPayload), - ); + return runLoadSessionWithReplayIdle(requestPayload, started.initializeResult); }), Effect.flatMap((response) => adoptSession(sessionId, response)), ), @@ -960,6 +2168,7 @@ export const make = ( }), ), ), + ...(options.ownDetachedProcessGroup === true ? { terminateProcessGroup } : {}), setMode: (modeId) => Ref.get(modeStateRef).pipe( Effect.flatMap((modeState) => { diff --git a/apps/server/src/provider/acp/GrokAcpSupport.test.ts b/apps/server/src/provider/acp/GrokAcpSupport.test.ts index 02d60976b24..fb9c85cc4e8 100644 --- a/apps/server/src/provider/acp/GrokAcpSupport.test.ts +++ b/apps/server/src/provider/acp/GrokAcpSupport.test.ts @@ -5,9 +5,33 @@ import * as EffectAcpErrors from "effect-acp/errors"; import { applyGrokAcpModelSelection, buildGrokAcpSpawnInput, + grokAcpRuntimeProcessOwnership, resolveGrokAcpBaseModelId, } from "./GrokAcpSupport.ts"; +describe("grokAcpRuntimeProcessOwnership", () => { + it("opts Grok into detached process-tree ownership on the injected host platform", () => { + expect(grokAcpRuntimeProcessOwnership("linux")).toEqual({ + ownDescendantProcessGroups: true, + ownDetachedProcessGroup: true, + processGroupPlatform: "linux", + }); + }); + + it("uses the prior provider-group path on Darwin and Windows", () => { + expect(grokAcpRuntimeProcessOwnership("darwin")).toEqual({ + ownDescendantProcessGroups: false, + ownDetachedProcessGroup: true, + processGroupPlatform: "darwin", + }); + expect(grokAcpRuntimeProcessOwnership("win32")).toEqual({ + ownDescendantProcessGroups: false, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + }); + }); +}); + describe("resolveGrokAcpBaseModelId", () => { it("normalizes empty and custom Grok model ids", () => { expect(resolveGrokAcpBaseModelId(undefined)).toBe("grok-build"); diff --git a/apps/server/src/provider/acp/GrokAcpSupport.ts b/apps/server/src/provider/acp/GrokAcpSupport.ts index c928b3ed80e..7dd4ab46f5e 100644 --- a/apps/server/src/provider/acp/GrokAcpSupport.ts +++ b/apps/server/src/provider/acp/GrokAcpSupport.ts @@ -1,3 +1,4 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; import { type GrokSettings, ProviderDriverKind } from "@t3tools/contracts"; import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; @@ -6,6 +7,7 @@ import * as Scope from "effect/Scope"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; import * as EffectAcpErrors from "effect-acp/errors"; import type * as EffectAcpSchema from "effect-acp/schema"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { normalizeModelSlug } from "@t3tools/shared/model"; import * as AcpSessionRuntime from "./AcpSessionRuntime.ts"; @@ -51,6 +53,21 @@ function resolveGrokAuthMethodId(environment: NodeJS.ProcessEnv | undefined): st : GROK_AUTH_METHOD_CACHED_TOKEN; } +export function grokAcpRuntimeProcessOwnership( + processGroupPlatform: NodeJS.Platform, +): Pick< + AcpSessionRuntime.AcpSessionRuntimeOptions, + "ownDescendantProcessGroups" | "ownDetachedProcessGroup" | "processGroupPlatform" +> { + return { + // macOS keeps the prior provider-group teardown until a stable libproc + // identity provider can cover Grok's nested detached tool groups. + ownDescendantProcessGroups: processGroupPlatform === "linux", + ownDetachedProcessGroup: true, + processGroupPlatform, + }; +} + export const makeGrokAcpRuntime = ( input: GrokAcpRuntimeInput, ): Effect.Effect< @@ -59,11 +76,15 @@ export const makeGrokAcpRuntime = ( Crypto.Crypto | Scope.Scope > => Effect.gen(function* () { + const processGroupPlatform = yield* HostProcessPlatform.pipe( + Effect.provide(NodeServices.layer), + ); const acpContext = yield* Layer.build( AcpSessionRuntime.layer({ ...input, spawn: buildGrokAcpSpawnInput(input.grokSettings, input.cwd, input.environment), authMethodId: resolveGrokAuthMethodId(input.environment), + ...grokAcpRuntimeProcessOwnership(processGroupPlatform), }).pipe( Layer.provide( Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, input.childProcessSpawner), diff --git a/apps/server/src/provider/acp/XAiAcpExtension.test.ts b/apps/server/src/provider/acp/XAiAcpExtension.test.ts index c3318c175a0..f4d3e1359c1 100644 --- a/apps/server/src/provider/acp/XAiAcpExtension.test.ts +++ b/apps/server/src/provider/acp/XAiAcpExtension.test.ts @@ -1,42 +1,77 @@ -// @effect-diagnostics nodeBuiltinImport:off -import * as NodePath from "node:path"; -import * as NodeURL from "node:url"; - -import * as NodeServices from "@effect/platform-node/NodeServices"; import { it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Schema from "effect/Schema"; import { describe, expect } from "vite-plus/test"; import { - extractXAiAskUserQuestions, + extractXAiAcpBackgroundToolMutation, + extractXAiAcpSubagentEndNotice, extractXAiAcpSubagentUpdate, + extractXAiAskUserQuestions, + extractXAiBackgroundTaskCompletion, + extractXAiKilledBackgroundTasks, + extractXAiMonitorTaskId, + isGenericAcpToolTitle, + isXAiMonitorTool, + isXAiPersistentMonitor, makeXAiAskUserQuestionCancelledResponse, makeXAiAskUserQuestionResponse, makeXAiPromptCompletionRuntime, + normalizeXAiAcpToolCallState, + resolveXAiAcpToolTitle, + xAiBackgroundTaskLifecycleMutation, + xAiPromptCompleteFromSessionUpdate, XAiAskUserQuestionRequest, } from "./XAiAcpExtension.ts"; import * as AcpSessionRuntime from "./AcpSessionRuntime.ts"; -const __dirname = NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)); -const mockAgentPath = NodePath.join(__dirname, "../../../scripts/acp-mock-agent.ts"); +const decodeXAiAskUserQuestionRequest = Schema.decodeUnknownSync(XAiAskUserQuestionRequest); -const makePromptCompletionRuntime = (env: NodeJS.ProcessEnv) => - Effect.gen(function* () { - const runtime = yield* AcpSessionRuntime.make({ - spawn: { - command: process.execPath, - args: [mockAgentPath], - env, - }, - cwd: process.cwd(), - clientInfo: { name: "t3-test", version: "0.0.0" }, - authMethodId: "test", +describe("xAiPromptCompleteFromSessionUpdate", () => { + it("maps live turn_completed snake_case payloads", () => { + expect( + xAiPromptCompleteFromSessionUpdate({ + sessionId: "019f4428-4bf1-7e52-b7c6-c29b506543b1", + update: { + sessionUpdate: "turn_completed", + prompt_id: "t3-xai-prompt-1", + stop_reason: "end_turn", + }, + }), + ).toEqual({ + sessionId: "019f4428-4bf1-7e52-b7c6-c29b506543b1", + promptId: "t3-xai-prompt-1", + stopReason: "end_turn", }); - return yield* makeXAiPromptCompletionRuntime(runtime); }); -const decodeXAiAskUserQuestionRequest = Schema.decodeUnknownSync(XAiAskUserQuestionRequest); + it("ignores non-turn updates and task-completed prompt ids", () => { + expect( + xAiPromptCompleteFromSessionUpdate({ + sessionId: "root", + update: { sessionUpdate: "hook_execution", prompt_id: "t3-xai-prompt-1" }, + }), + ).toBeNull(); + expect( + xAiPromptCompleteFromSessionUpdate({ + sessionId: "root", + update: { + sessionUpdate: "turn_completed", + prompt_id: "task-completed-call-abc", + stop_reason: "end_turn", + }, + }), + ).toBeNull(); + expect( + xAiPromptCompleteFromSessionUpdate({ + sessionId: "root", + update: { sessionUpdate: "turn_completed", stop_reason: "end_turn" }, + }), + ).toBeNull(); + }); +}); describe("XAiAcpExtension", () => { it("recognizes Grok Task starts as native subagents", () => { @@ -62,6 +97,7 @@ describe("XAiAcpExtension", () => { status: "running", childSessionId: null, result: null, + suppressNormalTool: true, }); }); @@ -95,6 +131,792 @@ describe("XAiAcpExtension", () => { status: "completed", childSessionId: "019f0220-e192-7c41-9e9d-b406bc3459c8", result: "Server audit complete.", + suppressNormalTool: true, + }); + }); + + it("keeps async spawn_subagent ACKs running and binds subagent_id", () => { + expect( + extractXAiAcpSubagentUpdate({ + toolCallId: "call-spawn-1", + title: "spawn_subagent", + status: "completed", + data: { + rawInput: { + description: "Demo subagent wait and ls", + prompt: "Wait then ls.", + subagent_type: "general-purpose", + }, + rawOutput: { + type: "Text", + text: [ + "Subagent started in background.", + "subagent_id: 019f44a6-4820-7402-925d-bc862ee711dd", + "type: general-purpose", + "description: Demo subagent wait and ls", + "", + 'Use get_command_or_subagent_output with task_ids=["019f44a6-4820-7402-925d-bc862ee711dd"] and timeout_ms to wait for results.', + ].join("\n"), + }, + }, + }), + ).toEqual({ + nativeTaskId: "call-spawn-1", + prompt: "Wait then ls.", + title: "Demo subagent wait and ls", + model: null, + status: "running", + childSessionId: "019f44a6-4820-7402-925d-bc862ee711dd", + result: null, + suppressNormalTool: true, + }); + }); + + it("hydrates subagent completion from get_command_or_subagent_output", () => { + expect( + extractXAiAcpSubagentUpdate({ + toolCallId: "call-get-1", + title: "get_command_or_subagent_output", + status: "completed", + data: { + rawInput: { + task_ids: ["019f44a6-4820-7402-925d-bc862ee711dd"], + timeout_ms: 30000, + }, + rawOutput: { + type: "Text", + text: [ + "=== Task 019f44a6-4820-7402-925d-bc862ee711dd ===", + "Status: completed", + "Exit Code: 0", + "", + "=== Output ===", + "SUBAGENT_MARKER: 35 entries", + ].join("\n"), + }, + }, + }), + ).toEqual({ + nativeTaskId: "call-get-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: "019f44a6-4820-7402-925d-bc862ee711dd", + result: "SUBAGENT_MARKER: 35 entries", + suppressNormalTool: true, + }); + }); + + it("attributes multi-task get_output envelopes to the result task_id", () => { + expect( + extractXAiAcpSubagentUpdate({ + toolCallId: "call-get-multi", + title: "get_command_or_subagent_output", + status: "completed", + data: { + rawInput: { + task_ids: [ + "019f44a6-4820-7402-925d-bc862ee711dd", + "019f44b0-1b73-7f32-bb1b-1ff696f536e3", + ], + timeout_ms: 30000, + }, + rawOutput: { + type: "TaskOutput", + Result: { + task_id: "019f44b0-1b73-7f32-bb1b-1ff696f536e3", + status: "completed", + exit_code: 0, + output: "SECOND_SUB_DONE\n", + }, + }, + }, + }), + ).toMatchObject({ + childSessionId: "019f44b0-1b73-7f32-bb1b-1ff696f536e3", + status: "completed", + result: "SECOND_SUB_DONE", + }); + }); + + it("hydrates structured ACP TaskOutput tool envelopes", () => { + expect( + extractXAiAcpSubagentUpdate({ + toolCallId: "call-get-2", + title: "[subagent:general-purpose] Sleep then return SUBAGENT_DONE (019f44b0)", + status: "completed", + data: { + rawInput: { + variant: "TaskOutput", + task_ids: ["019f44b0-1b73-7f32-bb1b-1ff696f536e3"], + timeout_ms: 20000, + }, + rawOutput: { + type: "TaskOutput", + Result: { + task_id: "019f44b0-1b73-7f32-bb1b-1ff696f536e3", + status: "completed", + exit_code: 0, + output: + "SUBAGENT_DONE\n\nid=019f44b0-1b73-7f32-bb1b-1ff696f536e3\n", + }, + }, + }, + }), + ).toEqual({ + nativeTaskId: "call-get-2", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: "019f44b0-1b73-7f32-bb1b-1ff696f536e3", + result: "SUBAGENT_DONE", + suppressNormalTool: true, + }); + }); + + it("does not leak raw machine tags when get_output is only meta/result blocks", () => { + const metaOnlyToolCall = { + toolCallId: "call-get-meta-only", + title: "get_command_or_subagent_output", + status: "completed" as const, + data: { + rawInput: { + variant: "TaskOutput", + task_ids: ["019f44b0-1b73-7f32-bb1b-1ff696f536e3"], + timeout_ms: 20000, + }, + rawOutput: { + type: "TaskOutput", + Result: { + task_id: "019f44b0-1b73-7f32-bb1b-1ff696f536e3", + status: "completed", + exit_code: 0, + output: + "id=019f44b0-1b73-7f32-bb1b-1ff696f536e3\ndone\n", + }, + }, + }, + }; + expect(extractXAiAcpSubagentUpdate(metaOnlyToolCall)).toEqual({ + nativeTaskId: "call-get-meta-only", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: "019f44b0-1b73-7f32-bb1b-1ff696f536e3", + result: null, + suppressNormalTool: true, + }); + expect(extractXAiBackgroundTaskCompletion(metaOnlyToolCall)).toEqual([ + { + taskId: "019f44b0-1b73-7f32-bb1b-1ff696f536e3", + status: "completed", + appendOutput: "", + }, + ]); + }); + + it("keeps monitor start ACKs running and extracts task ids", () => { + const toolCall = { + toolCallId: "call-mon-1", + title: "monitor", + status: "completed" as const, + data: { + rawInput: { + command: "echo mon_line_1", + description: "Demo monitor", + }, + rawOutput: { + type: "Text", + text: "Monitor started (task 019f44a5-87d1-7640-8e35-6a4667ffc873, timeout 36000000ms).\nYou will be notified on each event.", + }, + }, + }; + expect(normalizeXAiAcpToolCallState(toolCall).status).toBe("inProgress"); + expect(extractXAiMonitorTaskId(toolCall)).toBe("019f44a5-87d1-7640-8e35-6a4667ffc873"); + }); + + it("completes Monitor variant tools from structured Bash exit codes", () => { + const toolCall = { + toolCallId: "call-mon-2", + title: "Tool", + status: "inProgress" as const, + data: { + rawInput: { + variant: "Monitor", + command: "echo MON_DONE", + description: "Stream mon lines", + }, + rawOutput: { + type: "Bash", + output: Array.from(new TextEncoder().encode("mon_line_1\nMON_DONE\n")), + output_for_prompt: "mon_line_1\nMON_DONE\n", + exit_code: 0, + }, + }, + }; + expect(isXAiMonitorTool(toolCall)).toBe(true); + expect(normalizeXAiAcpToolCallState(toolCall).status).toBe("completed"); + }); + + it("detects Monitor start ACKs from structured rawOutput when title is generic", () => { + const toolCall = { + toolCallId: "call-mon-generic-title", + title: "Tool", + status: "completed" as const, + data: { + rawInput: { + command: "echo mon_line", + description: "Demo monitor without variant", + }, + rawOutput: { + type: "Monitor", + taskId: "019f44a5-87d1-7640-8e35-6a4667ffc873", + timeoutMs: 36000000, + persistent: false, + }, + }, + }; + expect(isXAiMonitorTool(toolCall)).toBe(true); + expect(normalizeXAiAcpToolCallState(toolCall).status).toBe("inProgress"); + expect(extractXAiMonitorTaskId(toolCall)).toBe("019f44a5-87d1-7640-8e35-6a4667ffc873"); + }); + + it("completes wake re-reports of finished commands despite generic titles", () => { + // Post-settle wake replay: the finished monitor is re-reported with empty + // rawInput and a generic title, so monitor detection cannot match; the + // structured Bash result with exit_code must still terminalize it. + const toolCall = { + toolCallId: "call-wake-1", + title: "Tool", + status: "inProgress" as const, + data: { + rawInput: {}, + rawOutput: { + type: "Bash", + output: Array.from(new TextEncoder().encode("STREAM_1\nSTREAM_DONE\n")), + output_for_prompt: "STREAM_1\nSTREAM_DONE\n", + exit_code: 0, + command: "for i in 1 2; do echo STREAM_$i; done; echo STREAM_DONE", + }, + }, + }; + expect(isXAiMonitorTool(toolCall)).toBe(false); + const normalized = normalizeXAiAcpToolCallState(toolCall); + expect(normalized.status).toBe("completed"); + expect(normalized.title).toBe("for i in 1 2; do echo STREAM_$i; done; echo STREAM_DONE"); + expect( + normalizeXAiAcpToolCallState({ + ...toolCall, + data: { + ...toolCall.data, + rawOutput: { ...toolCall.data.rawOutput, exit_code: 2 }, + }, + }).status, + ).toBe("failed"); + }); + + it("keeps empty successful Bash updates running until native output or turn settlement", () => { + const normalized = normalizeXAiAcpToolCallState({ + toolCallId: "call-bash-permission-ack", + title: "Ran command", + status: "completed", + data: { + rawInput: { command: "sleep 40" }, + rawOutput: { type: "Bash", exit_code: 0 }, + }, + }); + expect(normalized.status).toBe("inProgress"); + }); + + it("keeps structured Monitor start ACKs running and extracts taskId", () => { + const toolCall = { + toolCallId: "call-mon-3", + title: "Start monitor: Wait 30s then list directory", + status: "completed" as const, + data: { + rawInput: { + variant: "Monitor", + command: "sleep 30 && ls", + description: "Wait 30s then list directory", + }, + rawOutput: { + type: "Monitor", + taskId: "019f44b8-8e98-7c80-a40e-df1e26a5f9e3", + timeoutMs: 36000000, + persistent: false, + }, + }, + }; + expect(normalizeXAiAcpToolCallState(toolCall).status).toBe("inProgress"); + expect(extractXAiMonitorTaskId(toolCall)).toBe("019f44b8-8e98-7c80-a40e-df1e26a5f9e3"); + }); + + it("replaces generic ACP titles with description / Monitor labels", () => { + expect(isGenericAcpToolTitle("Tool")).toBe(true); + expect(isGenericAcpToolTitle("Read package.json")).toBe(false); + expect( + resolveXAiAcpToolTitle({ + toolCallId: "call-mon-title", + title: "Tool", + status: "completed", + data: { + rawInput: { + variant: "Monitor", + command: "sleep 30 && ls", + description: "Wait 30s then list directory", + }, + }, + }), + ).toBe("Monitor: Wait 30s then list directory"); + const normalized = normalizeXAiAcpToolCallState({ + toolCallId: "call-mon-title", + title: "Tool", + status: "completed", + data: { + rawInput: { + variant: "Monitor", + command: "sleep 30 && ls", + description: "Wait 30s then list directory", + }, + rawOutput: { + type: "Monitor", + taskId: "019f44b8-8e98-7c80-a40e-df1e26a5f9e3", + timeoutMs: 36000000, + persistent: false, + }, + }, + }); + expect(normalized.title).toBe("Monitor: Wait 30s then list directory"); + expect(normalized.status).toBe("inProgress"); + // Non-generic titles from the CLI are preserved. + expect( + resolveXAiAcpToolTitle({ + toolCallId: "call-read", + title: "Read package.json", + status: "inProgress", + data: { rawInput: { path: "package.json" } }, + }), + ).toBe("Read package.json"); + }); + + it("hydrates monitor completion from TaskOutput get_command envelopes", () => { + expect( + extractXAiBackgroundTaskCompletion({ + toolCallId: "call-get-mon", + title: "Wait 30s then list directory", + status: "completed", + data: { + rawInput: { + variant: "TaskOutput", + task_ids: ["019f44b8-8e98-7c80-a40e-df1e26a5f9e3"], + }, + rawOutput: { + type: "TaskOutput", + Result: { + task_id: "019f44b8-8e98-7c80-a40e-df1e26a5f9e3", + command: "[monitor] Wait 30s then list directory", + status: "completed", + exit_code: 0, + output: "agents\nAGENTS.md\nnotes\n", + }, + }, + }, + }), + ).toEqual([ + { + taskId: "019f44b8-8e98-7c80-a40e-df1e26a5f9e3", + status: "completed", + appendOutput: "agents\nAGENTS.md\nnotes", + }, + ]); + }); + + it("hydrates monitor completion from a standalone TaskOutput frame without rawInput", () => { + // Post-settle shape observed live (2026-07-12): the final + // get_command_or_subagent_output update carries only rawOutput; the frame + // is parsed in isolation (no in-turn merge), and this completion is the + // ONLY end signal when the agent consumed the output itself (no "Monitor + // ended" reminder follows). + expect( + extractXAiBackgroundTaskCompletion({ + toolCallId: "call-c29e64dd-ce5d-4eac-a6ca-c20513fefac3-1", + title: "[monitor] Stream numbered markers every 3s (019f54a0)", + status: "completed", + data: { + rawOutput: { + type: "TaskOutput", + Result: { + task_id: "019f54a0-06a8-77f2-8214-e24937cad564", + command: "[monitor] Stream numbered markers every 3s", + status: "completed", + exit_code: 0, + output: "STREAM_1\nSTREAM_DONE\n", + }, + }, + }, + }), + ).toMatchObject([ + { + taskId: "019f54a0-06a8-77f2-8214-e24937cad564", + status: "completed", + }, + ]); + }); + + it("hydrates every requested monitor when TaskOutput has no result task_id", () => { + expect( + extractXAiBackgroundTaskCompletion({ + toolCallId: "call-get-multi-mon", + title: "get_command_or_subagent_output", + status: "completed", + data: { + rawInput: { + variant: "TaskOutput", + task_ids: [ + "019f44a6-4820-7402-925d-bc862ee711dd", + "019f44b0-1b73-7f32-bb1b-1ff696f536e3", + ], + }, + rawOutput: { + type: "TaskOutput", + Result: { + status: "completed", + exit_code: 0, + output: "BOTH_DONE\n", + }, + }, + }, + }), + ).toEqual([ + { + taskId: "019f44a6-4820-7402-925d-bc862ee711dd", + status: "completed", + appendOutput: "BOTH_DONE", + }, + { + taskId: "019f44b0-1b73-7f32-bb1b-1ff696f536e3", + status: "completed", + appendOutput: "BOTH_DONE", + }, + ]); + }); + + it("treats Exit Code: -1 text envelopes as failed", () => { + expect( + extractXAiBackgroundTaskCompletion({ + toolCallId: "call-get-neg", + title: "get_command_or_subagent_output", + status: "completed", + data: { + rawInput: { + variant: "TaskOutput", + task_ids: ["019f44b8-8e98-7c80-a40e-df1e26a5f9e3"], + }, + rawOutput: { + type: "Text", + text: ["=== Task 019f44b8-8e98-7c80-a40e-df1e26a5f9e3 ===", "Exit Code: -1"].join("\n"), + }, + }, + }), + ).toMatchObject([ + { + taskId: "019f44b8-8e98-7c80-a40e-df1e26a5f9e3", + status: "failed", + }, + ]); + }); + + it("tombstones tasks the model kills via kill_command_or_subagent", () => { + expect( + extractXAiKilledBackgroundTasks({ + toolCallId: "call-kill-1", + title: "kill_command_or_subagent", + status: "completed", + data: { + rawInput: { + variant: "Kill", + task_ids: ["call-bg-1", "call-bg-2", "call-bg-1"], + }, + }, + }), + ).toEqual([ + { taskId: "call-bg-1", status: "completed", appendOutput: "" }, + { taskId: "call-bg-2", status: "completed", appendOutput: "" }, + ]); + // Singular task_id shape, matched by variant when the title is generic. + expect( + extractXAiKilledBackgroundTasks({ + toolCallId: "call-kill-2", + title: "Tool", + status: "failed", + data: { rawInput: { variant: "kill", task_id: "call-bg-3" } }, + }), + ).toEqual([{ taskId: "call-bg-3", status: "completed", appendOutput: "" }]); + // In-flight kill calls and unrelated tools contribute nothing. + expect( + extractXAiKilledBackgroundTasks({ + toolCallId: "call-kill-3", + title: "kill_command_or_subagent", + status: "inProgress", + data: { rawInput: { task_ids: ["call-bg-4"] } }, + }), + ).toEqual([]); + expect( + extractXAiKilledBackgroundTasks({ + toolCallId: "call-get", + title: "get_command_or_subagent_output", + status: "completed", + data: { rawInput: { variant: "TaskOutput", task_ids: ["call-bg-5"] } }, + }), + ).toEqual([]); + }); + + it("maps task lifecycle notifications to background mutations", () => { + // task_backgrounded carries the cancelled call's tool_call_id as task_id. + expect( + xAiBackgroundTaskLifecycleMutation( + { + sessionId: "session-1", + update: { + sessionUpdate: "task_backgrounded", + tool_call_id: "call-bg-1", + task_id: "call-bg-1", + }, + }, + "running", + ), + ).toEqual({ sessionId: "session-1", taskId: "call-bg-1", status: "running" }); + // task_completed nests the id inside task_snapshot. + expect( + xAiBackgroundTaskLifecycleMutation( + { + sessionId: "session-1", + update: { + sessionUpdate: "task_completed", + task_snapshot: { task_id: "call-bg-1" }, + }, + }, + "completed", + ), + ).toEqual({ sessionId: "session-1", taskId: "call-bg-1", status: "completed" }); + expect( + xAiBackgroundTaskLifecycleMutation( + { sessionId: "session-1", update: { sessionUpdate: "task_completed" } }, + "completed", + ), + ).toBeNull(); + }); + + it("detects persistent Monitor start ACKs", () => { + expect( + isXAiPersistentMonitor({ + toolCallId: "call-mon-persist", + title: "Monitor", + status: "completed", + data: { + rawInput: { variant: "Monitor" }, + rawOutput: { + type: "Monitor", + taskId: "019f44b8-8e98-7c80-a40e-df1e26a5f9e3", + timeoutMs: 36000000, + persistent: true, + }, + }, + }), + ).toBe(true); + expect( + isXAiPersistentMonitor({ + toolCallId: "call-mon-ephemeral", + title: "Monitor", + status: "completed", + data: { + rawInput: { variant: "Monitor" }, + rawOutput: { + type: "Monitor", + taskId: "019f44b8-8e98-7c80-a40e-df1e26a5f9e3", + timeoutMs: 36000000, + persistent: false, + }, + }, + }), + ).toBe(false); + }); + + it("keeps a standalone TaskOutput completion terminal through normalize", () => { + // Must remain offer evidence post-settle: normalize must not demote it to + // inProgress the way monitor start ACKs are. + expect( + normalizeXAiAcpToolCallState({ + toolCallId: "call-c29e64dd-ce5d-4eac-a6ca-c20513fefac3-1", + title: "[monitor] Stream numbered markers every 3s (019f54a0)", + status: "completed", + data: { + rawOutput: { + type: "TaskOutput", + Result: { + task_id: "019f54a0-06a8-77f2-8214-e24937cad564", + status: "completed", + exit_code: 0, + output: "STREAM_DONE\n", + }, + }, + }, + }).status, + ).toBe("completed"); + }); + + it("parses monitor event lines and end reminders", () => { + expect( + extractXAiAcpBackgroundToolMutation( + '\n[Demo] mon_line_1\n', + ), + ).toEqual([ + { + taskId: "019f44a5-87d1-7640-8e35-6a4667ffc873", + status: "running", + appendOutput: "[Demo] mon_line_1\n", + }, + ]); + expect( + extractXAiAcpBackgroundToolMutation( + [ + "", + 'Monitor "019f44a5-87d1-7640-8e35-6a4667ffc873" ended: [monitor ended: exited (code 0)].', + "Description: Demo monitor", + "", + ].join("\n"), + ), + ).toMatchObject([ + { + taskId: "019f44a5-87d1-7640-8e35-6a4667ffc873", + status: "completed", + }, + ]); + // Description/output may contain "error" without meaning the monitor failed. + expect( + extractXAiAcpBackgroundToolMutation( + [ + "", + 'Monitor "019f44a5-87d1-7640-8e35-6a4667ffc873" ended: [monitor ended: exited (code 0)].', + "Description: watch error logs for failed deploys", + "error: nothing found", + "", + ].join("\n"), + ), + ).toMatchObject([ + { + taskId: "019f44a5-87d1-7640-8e35-6a4667ffc873", + status: "completed", + }, + ]); + }); + + it("parses every monitor-event and trailing end notice in one chunk", () => { + const mutations = extractXAiAcpBackgroundToolMutation( + [ + '', + "[Demo] mon_line_1", + "", + '', + "[Demo] mon_line_2", + "", + 'Monitor "019f44a5-87d1-7640-8e35-6a4667ffc873" ended: [monitor ended: exited (code 0)].', + ].join("\n"), + ); + expect(mutations).toHaveLength(3); + expect(mutations[0]).toEqual({ + taskId: "019f44a5-87d1-7640-8e35-6a4667ffc873", + status: "running", + appendOutput: "[Demo] mon_line_1\n", + }); + expect(mutations[1]).toEqual({ + taskId: "019f44a5-87d1-7640-8e35-6a4667ffc873", + status: "running", + appendOutput: "[Demo] mon_line_2\n", + }); + expect(mutations[2]).toMatchObject({ + taskId: "019f44a5-87d1-7640-8e35-6a4667ffc873", + status: "completed", + }); + }); + + it("parses batched monitor event blocks as running mutations", () => { + expect( + extractXAiAcpBackgroundToolMutation( + [ + "3 monitor events from 1 monitor (use get_command_or_subagent_output to identify each monitor):", + "", + '', + "[1] STREAM_7", + "[2] STREAM_8", + "[3] STREAM_9", + "", + ].join("\n"), + ), + ).toEqual([ + { + taskId: "019f545d-4d39-7001-b1c8-c7744c448ec1", + status: "running", + appendOutput: "[1] STREAM_7\n[2] STREAM_8\n[3] STREAM_9\n", + }, + ]); + }); + + it("parses the subagent completed reminder as an end notice", () => { + expect( + extractXAiAcpSubagentEndNotice( + [ + "", + 'Background subagent "019f5470-bf92-7a90-afb3-5a6cea5b34a3" (general-purpose: "Run sleep then echo token") completed successfully.', + "Duration: 28.8s | Tool calls: 1 | Turns: 1", + 'Use get_task_output("019f5470-bf92-7a90-afb3-5a6cea5b34a3") to see the full output.', + "", + ].join("\n"), + ), + ).toEqual({ + childSessionId: "019f5470-bf92-7a90-afb3-5a6cea5b34a3", + status: "completed", + }); + }); + + it("parses a failed subagent reminder as a failed end notice", () => { + expect( + extractXAiAcpSubagentEndNotice( + 'Background subagent "019f5470-bf92-7a90-afb3-5a6cea5b34a3" (general-purpose: "Run sleep then echo token") failed.', + ), + ).toEqual({ + childSessionId: "019f5470-bf92-7a90-afb3-5a6cea5b34a3", + status: "failed", + }); + }); + + it("ignores prompt text that merely mentions subagents", () => { + expect( + extractXAiAcpSubagentEndNotice("Live-test post-settle subagent completion."), + ).toBeUndefined(); + expect( + extractXAiAcpSubagentEndNotice( + 'Background subagent "019f5470-bf92-7a90-afb3-5a6cea5b34a3" (general-purpose: "Run sleep then echo token") is still running.', + ), + ).toBeUndefined(); + }); + + it("does not treat nested parentheses in the title as the outcome verb", () => { + expect( + extractXAiAcpSubagentEndNotice( + 'Background subagent "019f5470-bf92-7a90-afb3-5a6cea5b34a3" (general-purpose: "Check (backend) failed tests") is still running.', + ), + ).toBeUndefined(); + expect( + extractXAiAcpSubagentEndNotice( + 'Background subagent "019f5470-bf92-7a90-afb3-5a6cea5b34a3" (general-purpose: "Check (backend) failed tests") completed successfully.', + ), + ).toEqual({ + childSessionId: "019f5470-bf92-7a90-afb3-5a6cea5b34a3", + status: "completed", }); }); @@ -335,58 +1157,265 @@ describe("XAiAcpExtension", () => { }); }); - it.effect("resolves a hung standard prompt from xAI prompt completion", () => + it.effect("settles a hung prompt from a root-session prompt_complete notification", () => Effect.gen(function* () { - const runtime = yield* makePromptCompletionRuntime({ - T3_ACP_EMIT_XAI_PROMPT_COMPLETE_THEN_HANG: "1", - }); - yield* runtime.start(); + const handlers = new Map Effect.Effect>(); + const hungPrompt = yield* Deferred.make(); + const baseRuntime = { + start: () => + Effect.succeed({ + sessionId: "root-session", + initializeResult: {}, + sessionSetupResult: {}, + modelConfigId: undefined, + }), + prompt: () => Deferred.await(hungPrompt), + cancel: Effect.void, + handleExtNotification: ( + method: string, + _schema: unknown, + handler: (notification: unknown) => Effect.Effect, + ) => { + handlers.set(method, handler); + return Effect.void; + }, + handleExtRequest: () => Effect.void, + } as unknown as AcpSessionRuntime.AcpSessionRuntime["Service"]; - const promptResult = yield* runtime.prompt({ - prompt: [{ type: "text", text: "hi" }], + const runtime = yield* makeXAiPromptCompletionRuntime(baseRuntime); + const promptFiber = yield* runtime + .prompt({ prompt: [{ type: "text", text: "hi" }] }) + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + const promptCompleteHandler = handlers.get("_x.ai/session/prompt_complete"); + expect(promptCompleteHandler).toBeDefined(); + yield* promptCompleteHandler!({ + sessionId: "root-session", + stopReason: "end_turn", }); - const promptId = promptResult._meta?.promptId; + const response = yield* Fiber.join(promptFiber); + expect(response.stopReason).toBe("end_turn"); + }), + ); + it.effect("settles a hung prompt from _x.ai/session/update turn_completed", () => + Effect.gen(function* () { + const handlers = new Map Effect.Effect>(); + let capturedMeta: Record | null | undefined; + const hungPrompt = yield* Deferred.make(); + const baseRuntime = { + start: () => + Effect.succeed({ + sessionId: "root-session", + initializeResult: {}, + sessionSetupResult: {}, + modelConfigId: undefined, + }), + prompt: (payload: { readonly _meta?: Record | null }) => { + capturedMeta = payload._meta ?? null; + return Deferred.await(hungPrompt); + }, + cancel: Effect.void, + handleExtNotification: ( + method: string, + _schema: unknown, + handler: (notification: unknown) => Effect.Effect, + ) => { + handlers.set(method, handler); + return Effect.void; + }, + handleExtRequest: () => Effect.void, + } as unknown as AcpSessionRuntime.AcpSessionRuntime["Service"]; + + const runtime = yield* makeXAiPromptCompletionRuntime(baseRuntime); + const promptFiber = yield* runtime + .prompt({ prompt: [{ type: "text", text: "hi" }] }) + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + const promptId = capturedMeta?.promptId; expect(typeof promptId).toBe("string"); - expect(promptResult).toMatchObject({ - stopReason: "end_turn", - _meta: { - sessionId: "mock-session-1", - promptId, - requestId: promptId, + const sessionUpdateHandler = handlers.get("_x.ai/session/update"); + expect(sessionUpdateHandler).toBeDefined(); + yield* sessionUpdateHandler!({ + sessionId: "root-session", + update: { + sessionUpdate: "turn_completed", + prompt_id: promptId, + stop_reason: "end_turn", }, }); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + const response = yield* Fiber.join(promptFiber); + expect(response.stopReason).toBe("end_turn"); + }), ); - it.effect("ignores stale xAI completion from an already settled prompt", () => + it.effect("ignores turn_completed for non-pending prompt ids and task completions", () => Effect.gen(function* () { - const runtime = yield* makePromptCompletionRuntime({ - T3_ACP_EMIT_STALE_XAI_PROMPT_COMPLETE_BEFORE_SECOND_HANG: "1", - }); - yield* runtime.start(); + const handlers = new Map Effect.Effect>(); + const hungPrompt = yield* Deferred.make(); + const baseRuntime = { + start: () => + Effect.succeed({ + sessionId: "root-session", + initializeResult: {}, + sessionSetupResult: {}, + modelConfigId: undefined, + }), + prompt: () => Deferred.await(hungPrompt), + cancel: Effect.void, + handleExtNotification: ( + method: string, + _schema: unknown, + handler: (notification: unknown) => Effect.Effect, + ) => { + handlers.set(method, handler); + return Effect.void; + }, + handleExtRequest: () => Effect.void, + } as unknown as AcpSessionRuntime.AcpSessionRuntime["Service"]; - const firstPromptResult = yield* runtime.prompt({ - prompt: [{ type: "text", text: "first" }], + const runtime = yield* makeXAiPromptCompletionRuntime(baseRuntime); + const promptFiber = yield* runtime + .prompt({ prompt: [{ type: "text", text: "hi" }] }) + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + const sessionUpdateHandler = handlers.get("_x.ai/session/update"); + expect(sessionUpdateHandler).toBeDefined(); + yield* sessionUpdateHandler!({ + sessionId: "root-session", + update: { + sessionUpdate: "turn_completed", + prompt_id: "task-completed-call-abc", + stop_reason: "end_turn", + }, }); - expect(firstPromptResult).toMatchObject({ - stopReason: "end_turn", - _meta: { promptId: "mock-stale-xai-prompt-1" }, + yield* sessionUpdateHandler!({ + sessionId: "root-session", + update: { + sessionUpdate: "turn_completed", + prompt_id: "some-other-cli-turn", + stop_reason: "end_turn", + }, }); + yield* Effect.yieldNow; + expect(promptFiber.pollUnsafe()).toBeUndefined(); + yield* Fiber.interrupt(promptFiber); + }), + ); + + it.effect("ignores prompt_complete notifications for foreign session ids", () => + Effect.gen(function* () { + const handlers = new Map Effect.Effect>(); + const hungPrompt = yield* Deferred.make(); + const baseRuntime = { + start: () => + Effect.succeed({ + sessionId: "root-session", + initializeResult: {}, + sessionSetupResult: {}, + modelConfigId: undefined, + }), + prompt: () => Deferred.await(hungPrompt), + cancel: Effect.void, + handleExtNotification: ( + method: string, + _schema: unknown, + handler: (notification: unknown) => Effect.Effect, + ) => { + handlers.set(method, handler); + return Effect.void; + }, + handleExtRequest: () => Effect.void, + } as unknown as AcpSessionRuntime.AcpSessionRuntime["Service"]; - const secondPromptResult = yield* runtime.prompt({ - prompt: [{ type: "text", text: "second" }], + const runtime = yield* makeXAiPromptCompletionRuntime(baseRuntime); + const promptFiber = yield* runtime + .prompt({ prompt: [{ type: "text", text: "hi" }] }) + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + const promptCompleteHandler = handlers.get("_x.ai/session/prompt_complete"); + expect(promptCompleteHandler).toBeDefined(); + yield* promptCompleteHandler!({ + sessionId: "child-session", }); - const secondPromptId = secondPromptResult._meta?.promptId; - expect(typeof secondPromptId).toBe("string"); - expect(secondPromptId).not.toBe("mock-stale-xai-prompt-1"); - expect(secondPromptResult).toMatchObject({ - stopReason: "end_turn", - _meta: { - promptId: secondPromptId, - requestId: secondPromptId, + yield* Effect.yieldNow; + expect(promptFiber.pollUnsafe()).toBeUndefined(); + yield* Fiber.interrupt(promptFiber); + }), + ); + + it.effect( + "ignores promptId-less prompt_complete when multiple prompts are pending on the same session", + () => + Effect.gen(function* () { + const handlers = new Map Effect.Effect>(); + const hungPrompt = yield* Deferred.make(); + const baseRuntime = { + start: () => + Effect.succeed({ + sessionId: "root-session", + initializeResult: {}, + sessionSetupResult: {}, + modelConfigId: undefined, + }), + prompt: () => Deferred.await(hungPrompt), + cancel: Effect.void, + handleExtNotification: ( + method: string, + _schema: unknown, + handler: (notification: unknown) => Effect.Effect, + ) => { + handlers.set(method, handler); + return Effect.void; + }, + handleExtRequest: () => Effect.void, + } as unknown as AcpSessionRuntime.AcpSessionRuntime["Service"]; + + const runtime = yield* makeXAiPromptCompletionRuntime(baseRuntime); + const firstPromptFiber = yield* runtime + .prompt({ prompt: [{ type: "text", text: "first" }] }) + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + const secondPromptFiber = yield* runtime + .prompt({ prompt: [{ type: "text", text: "second" }] }) + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + const promptCompleteHandler = handlers.get("_x.ai/session/prompt_complete"); + expect(promptCompleteHandler).toBeDefined(); + yield* promptCompleteHandler!({ + sessionId: "root-session", + stopReason: "end_turn", + }); + yield* Effect.yieldNow; + expect(firstPromptFiber.pollUnsafe()).toBeUndefined(); + expect(secondPromptFiber.pollUnsafe()).toBeUndefined(); + yield* Fiber.interrupt(firstPromptFiber); + yield* Fiber.interrupt(secondPromptFiber); + }), + ); + + it.effect("injects promptId and requestId into prompt _meta", () => + Effect.gen(function* () { + let capturedMeta: Record | null | undefined; + const baseRuntime = { + start: () => Effect.succeed({ sessionId: "session-1" }), + prompt: (payload: { readonly _meta?: Record | null }) => { + capturedMeta = payload._meta ?? null; + return Effect.succeed({ stopReason: "end_turn" as const }); }, + cancel: Effect.void, + handleExtNotification: () => Effect.void, + handleExtRequest: () => Effect.void, + } as unknown as AcpSessionRuntime.AcpSessionRuntime["Service"]; + + const runtime = yield* makeXAiPromptCompletionRuntime(baseRuntime); + yield* runtime.prompt({ prompt: [{ type: "text", text: "hi" }] }); + + expect(typeof capturedMeta?.promptId).toBe("string"); + expect(capturedMeta).toMatchObject({ + promptId: capturedMeta?.promptId, + requestId: capturedMeta?.promptId, }); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + }), ); }); diff --git a/apps/server/src/provider/acp/XAiAcpExtension.ts b/apps/server/src/provider/acp/XAiAcpExtension.ts index 77bc9770114..3ad86010c77 100644 --- a/apps/server/src/provider/acp/XAiAcpExtension.ts +++ b/apps/server/src/provider/acp/XAiAcpExtension.ts @@ -1,13 +1,19 @@ import type { ProviderUserInputAnswers, UserInputQuestion } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; +import type * as EffectAcpErrors from "effect-acp/errors"; import type * as EffectAcpSchema from "effect-acp/schema"; import type * as AcpSessionRuntime from "./AcpSessionRuntime.ts"; import type { AcpToolCallState } from "./AcpRuntimeModel.ts"; +const xAiStopReasonMissingMetaKey = "xAiStopReasonMissing"; +const completedXAiPromptIdLimit = 128; + const XAiPromptCompleteNotification = Schema.Struct({ sessionId: Schema.String, promptId: Schema.optional(Schema.String), @@ -17,15 +23,60 @@ const XAiPromptCompleteNotification = Schema.Struct({ type XAiPromptCompleteNotification = typeof XAiPromptCompleteNotification.Type; +/** + * Live Grok CLI (0.2.x) emits root turn completion as an extension session + * update, not as top-level `_x.ai/session/prompt_complete`: + * `{ method: "_x.ai/session/update", params: { sessionId, update: { + * sessionUpdate: "turn_completed", prompt_id, stop_reason } } }`. + */ +const XAiSessionUpdateNotification = Schema.Struct({ + sessionId: Schema.String, + update: Schema.Struct({ + sessionUpdate: Schema.String, + prompt_id: Schema.optional(Schema.String), + promptId: Schema.optional(Schema.String), + stop_reason: Schema.optional(Schema.String), + stopReason: Schema.optional(Schema.String), + }), + _meta: Schema.optional(Schema.Unknown), +}); + +type XAiSessionUpdateNotification = typeof XAiSessionUpdateNotification.Type; + +const XAI_TASK_COMPLETED_PROMPT_ID_PREFIX = "task-completed-"; + +/** + * Map a Grok `_x.ai/session/update` payload to a prompt-complete shape when it + * is a root `turn_completed` for a real prompt id. Returns null for hooks, + * background task completions, and other update kinds. + */ +export function xAiPromptCompleteFromSessionUpdate( + notification: XAiSessionUpdateNotification, +): XAiPromptCompleteNotification | null { + const update = notification.update; + if (update.sessionUpdate !== "turn_completed") { + return null; + } + const promptId = nonEmptyString(update.prompt_id) ?? nonEmptyString(update.promptId); + // Require prompt id so we never session-fallback match a background task + // completion (`task-completed-*`) or an unrelated CLI turn. + if (promptId === undefined || promptId.startsWith(XAI_TASK_COMPLETED_PROMPT_ID_PREFIX)) { + return null; + } + const stopReason = nonEmptyString(update.stop_reason) ?? nonEmptyString(update.stopReason); + return { + sessionId: notification.sessionId, + promptId, + ...(stopReason === undefined ? {} : { stopReason }), + }; +} + interface PendingXAiPromptCompletion { readonly sessionId: string; readonly promptId: string; readonly deferred: Deferred.Deferred; } -const completedXAiPromptIdLimit = 128; -const xAiStopReasonMissingMetaKey = "xAiStopReasonMissing"; - export interface XAiAcpSubagentUpdate { readonly nativeTaskId: string; readonly prompt: string; @@ -34,8 +85,26 @@ export interface XAiAcpSubagentUpdate { readonly status: "running" | "completed" | "failed"; readonly childSessionId: string | null; readonly result: string | null; + /** + * When false, the ACP adapter also projects a normal tool turn item for this + * tool call (used for get_command_or_subagent_output hydration). Default true. + */ + readonly suppressNormalTool?: boolean; +} + +export interface XAiAcpBackgroundToolMutation { + readonly taskId: string; + readonly status: "running" | "completed" | "failed"; + readonly appendOutput: string; +} + +export interface XAiAcpSubagentEndNotice { + readonly childSessionId: string; + readonly status: "completed" | "failed"; } +const XAI_UUID_RE = "[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}"; + function unknownRecord(value: unknown): Record | undefined { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -46,10 +115,42 @@ function nonEmptyString(value: unknown): string | undefined { return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; } -function xAiTaskOutputText(toolCall: AcpToolCallState): string | undefined { +function titleKey(toolCall: AcpToolCallState): string { + return (toolCall.title ?? "").trim().toLowerCase(); +} + +function decodeByteText(value: unknown): string | undefined { + if (typeof value === "string") return nonEmptyString(value); + if (!Array.isArray(value)) return undefined; + if (value.length === 0) return undefined; + if (!value.every((entry) => typeof entry === "number")) return undefined; + try { + return nonEmptyString(new TextDecoder().decode(Uint8Array.from(value as number[]))); + } catch { + return undefined; + } +} + +function xAiToolOutputText(toolCall: AcpToolCallState): string | undefined { const rawOutput = unknownRecord(toolCall.data.rawOutput); - const rawOutputText = nonEmptyString(rawOutput?.text); - if (rawOutputText !== undefined) return rawOutputText; + if (rawOutput !== undefined) { + const direct = + nonEmptyString(rawOutput.text) ?? + nonEmptyString(rawOutput.output_for_prompt) ?? + decodeByteText(rawOutput.output); + if (direct !== undefined) return direct; + const result = unknownRecord(rawOutput.Result) ?? unknownRecord(rawOutput.result); + if (result !== undefined) { + const nested = + nonEmptyString(result.output) ?? + nonEmptyString(result.text) ?? + decodeByteText(result.output); + if (nested !== undefined) return nested; + } + } + if (typeof toolCall.data.rawOutput === "string") { + return nonEmptyString(toolCall.data.rawOutput); + } const content = toolCall.data.content; if (!Array.isArray(content)) return undefined; const text = content @@ -63,41 +164,631 @@ function xAiTaskOutputText(toolCall: AcpToolCallState): string | undefined { return text.length > 0 ? text : undefined; } +function firstUuidMatch(text: string, pattern: RegExp): string | undefined { + const match = pattern.exec(text); + return match?.[1]; +} + +function extractXAiChildSessionId(output: string | undefined): string | null { + if (output === undefined) return null; + return ( + firstUuidMatch(output, new RegExp(`(?:^|\\n)\\s*Agent ID:\\s*(${XAI_UUID_RE})\\b`, "i")) ?? + firstUuidMatch(output, new RegExp(`(?:^|\\n)\\s*subagent_id:\\s*(${XAI_UUID_RE})\\b`, "i")) ?? + firstUuidMatch(output, new RegExp(`===\\s*Task\\s+(${XAI_UUID_RE})\\s*===`, "i")) ?? + null + ); +} + +function isXAiAsyncSpawnAck(output: string | undefined): boolean { + if (output === undefined) return false; + if (/subagent started in background/i.test(output)) return true; + return ( + new RegExp(`subagent_id:\\s*${XAI_UUID_RE}`, "i").test(output) && + /get_command_or_subagent_output/i.test(output) + ); +} + +function isXAiMonitorStartAck(output: string | undefined): boolean { + if (output === undefined) return false; + return /monitor started\s*\(\s*task\s+/i.test(output); +} + +function isXAiSpawnOrTaskTool( + toolCall: AcpToolCallState, + rawInput: Record | undefined, +): boolean { + const title = titleKey(toolCall); + if (title === "task" || title === "spawn_subagent" || title.includes("spawn subagent")) { + return true; + } + const variant = nonEmptyString(rawInput?.variant)?.toLowerCase(); + if (variant === "cursortask" || variant === "task" || variant === "spawn_subagent") { + return true; + } + if (nonEmptyString(rawInput?.subagent_type) !== undefined) return true; + if (nonEmptyString(rawInput?.subagentType) !== undefined) return true; + return false; +} + +function isXAiGetSubagentOutputTool( + toolCall: AcpToolCallState, + rawInput: Record | undefined, +): boolean { + const title = titleKey(toolCall); + if ( + title === "get_command_or_subagent_output" || + title.includes("get_command_or_subagent_output") || + title === "wait_commands_or_subagents" + ) { + return true; + } + const variant = nonEmptyString(rawInput?.variant)?.toLowerCase(); + if (variant === "taskoutput" || variant === "task_output") return true; + const rawOutput = unknownRecord(toolCall.data.rawOutput); + const outputType = nonEmptyString(rawOutput?.type)?.toLowerCase(); + if (outputType === "taskoutput" || outputType === "task_output") return true; + // ACP often titles this tool with the finished subagent label. + if (title.startsWith("[subagent:") && Array.isArray(rawInput?.task_ids)) return true; + return false; +} + +export function isXAiMonitorTool(toolCall: AcpToolCallState): boolean { + const title = titleKey(toolCall); + if (title === "monitor" || title.startsWith("monitor ")) return true; + const rawInput = unknownRecord(toolCall.data.rawInput); + const variant = nonEmptyString(rawInput?.variant)?.toLowerCase(); + if (variant === "monitor") return true; + // Live start ACKs often keep a generic title ("Tool") and only mark the + // structured envelope as Monitor; title rewrite to description must not + // prevent monitor registration / inProgress forcing. + const rawOutput = unknownRecord(toolCall.data.rawOutput); + const outputType = nonEmptyString(rawOutput?.type)?.toLowerCase(); + return outputType === "monitor"; +} + +export function extractXAiMonitorTaskId(toolCall: AcpToolCallState): string | undefined { + if (!isXAiMonitorTool(toolCall)) return undefined; + const rawOutput = unknownRecord(toolCall.data.rawOutput); + // Live ACP start ACK: { type: "Monitor", taskId, timeoutMs, persistent } + if (rawOutput !== undefined && nonEmptyString(rawOutput.type)?.toLowerCase() === "monitor") { + const structured = nonEmptyString(rawOutput.taskId) ?? nonEmptyString(rawOutput.task_id); + if (structured !== undefined) return structured; + } + const output = xAiToolOutputText(toolCall); + if (output !== undefined) { + const fromAck = firstUuidMatch( + output, + new RegExp(`monitor started\\s*\\(\\s*task\\s+(${XAI_UUID_RE})\\b`, "i"), + ); + if (fromAck !== undefined) return fromAck; + } + const rawInput = unknownRecord(toolCall.data.rawInput); + return ( + nonEmptyString(rawInput?.task_id) ?? + nonEmptyString(rawInput?.taskId) ?? + (Array.isArray(rawInput?.task_ids) ? nonEmptyString(rawInput.task_ids[0]) : undefined) + ); +} + +/** + * Live Monitor start ACK `{ type: "Monitor", persistent: true }` (or matching + * rawInput). Persistent monitors should not hold root-turn deferred finalize. + */ +export function isXAiPersistentMonitor(toolCall: AcpToolCallState): boolean { + if (!isXAiMonitorTool(toolCall)) return false; + const rawOutput = unknownRecord(toolCall.data.rawOutput); + if (rawOutput !== undefined && nonEmptyString(rawOutput.type)?.toLowerCase() === "monitor") { + return rawOutput.persistent === true; + } + const rawInput = unknownRecord(toolCall.data.rawInput); + return rawInput?.persistent === true; +} + +/** + * When get_command_or_subagent_output / TaskOutput completes registered monitor + * task(s), return hydration for each completed background tool id. + */ +export function extractXAiBackgroundTaskCompletion(toolCall: AcpToolCallState): ReadonlyArray<{ + readonly taskId: string; + readonly status: "running" | "completed" | "failed"; + readonly appendOutput: string; +}> { + const rawInput = unknownRecord(toolCall.data.rawInput); + if (!isXAiGetSubagentOutputTool(toolCall, rawInput)) return []; + // Caller matches taskId against registered background tools (monitors). + // Subagent hydration stays on extractXAiAcpSubagentUpdate. + const text = xAiToolOutputText(toolCall); + const status = statusFromGetOutputTool(toolCall, text); + const appendOutput = resultFromGetOutputTool(toolCall, text) ?? ""; + // Prefer the completed identity from the result/header over the request list + // (one call can poll multiple task_ids). + const resultTaskId = resultTaskIdFromGetOutputTool(toolCall, text); + if (resultTaskId !== undefined) { + return [{ taskId: resultTaskId, status, appendOutput }]; + } + const requestIds: string[] = []; + const push = (value: unknown) => { + const id = nonEmptyString(value); + if (id !== undefined && new RegExp(`^${XAI_UUID_RE}$`, "i").test(id)) { + requestIds.push(id); + } + }; + if (Array.isArray(rawInput?.task_ids)) { + for (const entry of rawInput.task_ids) push(entry); + } + push(rawInput?.task_id); + push(rawInput?.taskId); + const unique = [...new Set(requestIds)]; + return unique.map((taskId) => ({ taskId, status, appendOutput })); +} + +/** + * A finished kill_command_or_subagent call is the genuine end signal for every + * task id it names: the Grok CLI emits no `_x.ai/task_completed` for a killed + * task. A failed kill usually means the task was already gone, so it ends the + * id too (failing open to a continuation beats pinning the running set). + */ +export function extractXAiKilledBackgroundTasks(toolCall: AcpToolCallState): ReadonlyArray<{ + readonly taskId: string; + readonly status: "running" | "completed" | "failed"; + readonly appendOutput: string; +}> { + const title = titleKey(toolCall); + const rawInput = unknownRecord(toolCall.data.rawInput); + const variant = nonEmptyString(rawInput?.variant)?.toLowerCase(); + if (!title.includes("kill_command_or_subagent") && variant !== "kill") return []; + if (toolCall.status !== "completed" && toolCall.status !== "failed") return []; + const taskIds: string[] = []; + const push = (value: unknown) => { + const id = nonEmptyString(value); + if (id !== undefined) taskIds.push(id); + }; + if (Array.isArray(rawInput?.task_ids)) { + for (const entry of rawInput.task_ids) push(entry); + } + push(rawInput?.task_id); + push(rawInput?.taskId); + return [...new Set(taskIds)].map((taskId) => ({ + taskId, + status: "completed" as const, + appendOutput: "", + })); +} + +const XAiTaskLifecycleNotification = Schema.Struct({ + sessionId: Schema.String, + update: Schema.Struct({ + sessionUpdate: Schema.String, + task_id: Schema.optional(Schema.String), + tool_call_id: Schema.optional(Schema.String), + task_snapshot: Schema.optional( + Schema.Struct({ + task_id: Schema.optional(Schema.String), + }), + ), + }), + _meta: Schema.optional(Schema.Unknown), +}); + +type XAiTaskLifecycleNotification = typeof XAiTaskLifecycleNotification.Type; + +export interface XAiBackgroundTaskLifecycleMutation { + readonly sessionId: string; + readonly taskId: string; + readonly status: "running" | "completed"; +} + +export function xAiBackgroundTaskLifecycleMutation( + notification: XAiTaskLifecycleNotification, + status: "running" | "completed", +): XAiBackgroundTaskLifecycleMutation | null { + const update = notification.update; + const taskId = + nonEmptyString(update.task_snapshot?.task_id) ?? + nonEmptyString(update.task_id) ?? + nonEmptyString(update.tool_call_id); + if (taskId === undefined) return null; + return { sessionId: notification.sessionId, taskId, status }; +} + +/** + * Grok `session/cancel` is detach-and-continue: the CLI reaps the foreground + * tool shell, then re-runs the command as a background task + * (`_x.ai/task_backgrounded`) that executes to completion + * (`_x.ai/task_completed`). Soft interrupts keep the process alive, so those + * tasks must be tracked like monitors: residual frames buffer instead of + * waking synthetic continuations, and pending-background-work stays truthful + * until the task ends (or the model kills it via kill_command_or_subagent). + */ +export const registerXAiBackgroundTaskTracking = ( + runtime: Pick, + apply: (mutation: XAiBackgroundTaskLifecycleMutation) => Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + yield* runtime.handleExtNotification( + "_x.ai/task_backgrounded", + XAiTaskLifecycleNotification, + (notification) => { + const mutation = xAiBackgroundTaskLifecycleMutation(notification, "running"); + return mutation === null ? Effect.void : apply(mutation); + }, + ); + yield* runtime.handleExtNotification( + "_x.ai/task_completed", + XAiTaskLifecycleNotification, + (notification) => { + const mutation = xAiBackgroundTaskLifecycleMutation(notification, "completed"); + return mutation === null ? Effect.void : apply(mutation); + }, + ); + }); + +/** + * Grok ACP often sends `title: "Tool"` even when rawInput has a useful + * description or variant (especially Monitor). Prefer description/variant so + * the timeline matches Claude-style tool names rather than a generic label. + */ +export function isGenericAcpToolTitle(title: string | undefined): boolean { + const key = (title ?? "").trim().toLowerCase(); + return key.length === 0 || key === "tool" || key === "tool call" || key === "terminal"; +} + +export function resolveXAiAcpToolTitle(toolCall: AcpToolCallState): string | undefined { + if (!isGenericAcpToolTitle(toolCall.title)) { + return nonEmptyString(toolCall.title); + } + const rawInput = unknownRecord(toolCall.data.rawInput); + const description = nonEmptyString(rawInput?.description); + const variant = nonEmptyString(rawInput?.variant)?.toLowerCase(); + if (description !== undefined) { + if (variant === "monitor" && !description.toLowerCase().startsWith("monitor")) { + return `Monitor: ${description}`; + } + return description; + } + if (variant === "monitor") return "Monitor"; + if (variant === "task" || variant === "cursortask" || variant === "spawn_subagent") { + return "Task"; + } + if (variant === "taskoutput" || variant === "task_output") return "Task output"; + if (variant !== undefined) { + return variant; + } + const command = + nonEmptyString(rawInput?.command) ?? + nonEmptyString(unknownRecord(toolCall.data.rawOutput)?.command); + if (command !== undefined) { + return command.length > 80 ? `${command.slice(0, 77)}...` : command; + } + return nonEmptyString(toolCall.title); +} + +/** + * Normalize Grok ACP tool presentation and monitor lifecycle: + * - replace generic titles ("Tool") with description / variant labels + * - structured Monitor start ACK stays running + * - text start ACK stays running + * - structured Bash results with exit_code are terminal (any tool: post-settle + * wake re-reports of a finished monitor arrive with empty rawInput and a + * generic title, so monitor detection cannot match; without this they replay + * as running and the timeline row spins forever) + */ +export function normalizeXAiAcpToolCallState(toolCall: AcpToolCallState): AcpToolCallState { + const resolvedTitle = resolveXAiAcpToolTitle(toolCall); + const withTitle = + resolvedTitle !== undefined && resolvedTitle !== toolCall.title + ? { ...toolCall, title: resolvedTitle } + : toolCall; + + const rawOutput = unknownRecord(withTitle.data.rawOutput); + const outputType = nonEmptyString(rawOutput?.type)?.toLowerCase(); + if (outputType === "bash") { + // Same key set as the adapter's command projection (commandExitCode). + const exitCode = ["exit_code", "exitCode", "code"] + .map((key) => rawOutput?.[key]) + .find((value) => typeof value === "number" && Number.isInteger(value)); + if (typeof exitCode === "number") { + if (exitCode === 0 && (xAiToolOutputText(withTitle)?.trim().length ?? 0) === 0) { + return { ...withTitle, status: "inProgress" }; + } + return { + ...withTitle, + status: exitCode === 0 ? "completed" : "failed", + }; + } + } + if (!isXAiMonitorTool(withTitle)) { + return withTitle; + } + if (outputType === "monitor") { + // Start registration only; process still running in the background. + return { ...withTitle, status: "inProgress" }; + } + const output = xAiToolOutputText(withTitle); + if (withTitle.status === "completed" && isXAiMonitorStartAck(output)) { + return { ...withTitle, status: "inProgress" }; + } + return withTitle; +} + +/** + * Parse Grok synthetic monitor traffic that arrives as root text chunks + * (`` lines, batched `` event blocks, and + * "Monitor ... ended" reminders). Coalesced chunks may contain several events; + * return every match so later progress / end notices are not dropped. + */ +export function extractXAiAcpBackgroundToolMutation( + text: string, +): ReadonlyArray { + const mutations: XAiAcpBackgroundToolMutation[] = []; + + const eventRe = new RegExp( + `\\s*([\\s\\S]*?)\\s*`, + "gi", + ); + for (const eventMatch of text.matchAll(eventRe)) { + if (eventMatch[1] === undefined) continue; + const line = (eventMatch[2] ?? "").trim(); + mutations.push({ + taskId: eventMatch[1], + status: "running", + appendOutput: line.length > 0 ? `${line}\n` : "", + }); + } + + // Batched form: "N monitor events from 1 monitor ...:\n\n[1] line\n". Still a running + // monitor; must track and filter like single-event chatter. + const batchedRe = new RegExp( + `]*task_id=["']?(${XAI_UUID_RE})["']?[^>]*>\\s*([\\s\\S]*?)\\s*`, + "gi", + ); + for (const batchedMatch of text.matchAll(batchedRe)) { + if (batchedMatch[1] === undefined) continue; + const lines = (batchedMatch[2] ?? "").trim(); + mutations.push({ + taskId: batchedMatch[1], + status: "running", + appendOutput: lines.length > 0 ? `${lines}\n` : "", + }); + } + + const endedRe = new RegExp(`Monitor\\s+["']?(${XAI_UUID_RE})["']?\\s+ended`, "gi"); + for (const endedMatch of text.matchAll(endedRe)) { + if (endedMatch[1] === undefined) continue; + // Only the "Monitor … ended …" clause is outcome text. Description/output + // after the header can contain words like "error" without meaning failure. + const outcomeClause = (endedMatch[0] ?? "").trim(); + const afterHeader = text.slice((endedMatch.index ?? 0) + endedMatch[0].length); + const outcomeTail = afterHeader.split(/\n/)[0] ?? ""; + const outcome = `${outcomeClause}${outcomeTail}`; + const summary = text.replace(/<\/?system-reminder>/gi, "").trim(); + mutations.push({ + taskId: endedMatch[1], + status: + /exited\s*\(\s*code\s*0\s*\)/i.test(outcome) || /ended cleanly/i.test(outcome) + ? "completed" + : /exited|failed|error|signal/i.test(outcome) + ? "failed" + : "completed", + appendOutput: summary.length > 0 ? `${summary}\n` : "Monitor ended.\n", + }); + } + + return mutations; +} + +/** + * Parse Grok's root-session subagent end reminder ("Background subagent + * "" (general-purpose: "...") completed successfully."). The agent is + * free to skip get_command_or_subagent_output hydration, so this reminder can + * be the only terminal signal for the subagent row; a row stuck on running + * holds deferred finalize (and the whole run) open until harness timeout. + */ +function stripLeadingBalancedParenGroup(text: string): string { + const trimmed = text.replace(/^\s+/, ""); + if (!trimmed.startsWith("(")) return trimmed; + let depth = 0; + for (let index = 0; index < trimmed.length; index += 1) { + const char = trimmed[index]; + if (char === "(") depth += 1; + else if (char === ")") { + depth -= 1; + if (depth === 0) return trimmed.slice(index + 1).replace(/^\s+/, ""); + } + } + return trimmed; +} + +export function extractXAiAcpSubagentEndNotice(text: string): XAiAcpSubagentEndNotice | undefined { + const match = text.match(new RegExp(`Background subagent\\s+["']?(${XAI_UUID_RE})["']?`, "i")); + if (match?.[1] === undefined) return undefined; + // Skip the parenthesized descriptor (subagent type and title) so its words + // cannot masquerade as the outcome verb. Balance parens so nested titles + // like `(general-purpose: "Check (backend) failed")` do not leak. + const tail = stripLeadingBalancedParenGroup(text.slice((match.index ?? 0) + match[0].length)); + if (/completed successfully/i.test(tail)) { + return { childSessionId: match[1], status: "completed" }; + } + if (/\b(failed|errored|error|crashed|cancelled|canceled|interrupted|timed out)\b/i.test(tail)) { + return { childSessionId: match[1], status: "failed" }; + } + if (/\b(completed|finished|ended)\b/i.test(tail)) { + return { childSessionId: match[1], status: "completed" }; + } + return undefined; +} + +function taskIdsFromGetOutputTool( + toolCall: AcpToolCallState, + rawInput: Record | undefined, +): ReadonlyArray { + const ids: string[] = []; + const push = (value: unknown) => { + const text = nonEmptyString(value); + if (text !== undefined && new RegExp(`^${XAI_UUID_RE}$`, "i").test(text)) { + ids.push(text); + } + }; + if (Array.isArray(rawInput?.task_ids)) { + for (const entry of rawInput.task_ids) push(entry); + } + push(rawInput?.task_id); + push(rawInput?.taskId); + const resultId = resultTaskIdFromGetOutputTool(toolCall, xAiToolOutputText(toolCall)); + if (resultId !== undefined) ids.push(resultId); + return [...new Set(ids)]; +} + +/** Identity of the task the get_output envelope actually reports (not the request list). */ +function resultTaskIdFromGetOutputTool( + toolCall: AcpToolCallState, + output: string | undefined, +): string | undefined { + const rawOutput = unknownRecord(toolCall.data.rawOutput); + const result = unknownRecord(rawOutput?.Result) ?? unknownRecord(rawOutput?.result); + const structured = + nonEmptyString(result?.task_id) ?? + nonEmptyString(result?.taskId) ?? + nonEmptyString(rawOutput?.task_id) ?? + nonEmptyString(rawOutput?.taskId); + if (structured !== undefined && new RegExp(`^${XAI_UUID_RE}$`, "i").test(structured)) { + return structured; + } + if (output === undefined) return undefined; + const taskHeader = firstUuidMatch( + output, + new RegExp(`===\\s*Task\\s+(${XAI_UUID_RE})\\s*===`, "i"), + ); + if (taskHeader !== undefined) return taskHeader; + return firstUuidMatch(output, new RegExp(`subagent_id:\\s*(${XAI_UUID_RE})\\b`, "i")); +} + +function resultFromGetOutputTool( + toolCall: AcpToolCallState, + output: string | undefined, +): string | null { + const rawOutput = unknownRecord(toolCall.data.rawOutput); + const result = unknownRecord(rawOutput?.Result) ?? unknownRecord(rawOutput?.result); + const structured = + nonEmptyString(result?.output) ?? + nonEmptyString(result?.text) ?? + decodeByteText(result?.output); + if (structured !== undefined) { + // Drop trailing machine meta blocks when present. + const cleaned = structured + .replace(/[\s\S]*?<\/subagent_meta>/gi, "") + .replace(/[\s\S]*?<\/subagent_result>/gi, "") + .trim(); + if (cleaned.length > 0) return cleaned; + return null; + } + if (output === undefined) return null; + const marker = output.match(/=== Output ===\s*([\s\S]*)$/i); + const body = (marker?.[1] ?? output).trim(); + return body.length > 0 ? body : null; +} + +function statusFromGetOutputTool( + toolCall: AcpToolCallState, + output: string | undefined, +): "running" | "completed" | "failed" { + if (toolCall.status === "failed") return "failed"; + const rawOutput = unknownRecord(toolCall.data.rawOutput); + const result = unknownRecord(rawOutput?.Result) ?? unknownRecord(rawOutput?.result); + const structuredStatus = nonEmptyString(result?.status)?.toLowerCase(); + if (structuredStatus === "completed" || structuredStatus === "success") return "completed"; + if (structuredStatus === "failed" || structuredStatus === "error") return "failed"; + if (structuredStatus === "running" || structuredStatus === "pending") return "running"; + if (typeof result?.exit_code === "number") { + return result.exit_code === 0 ? "completed" : "failed"; + } + if (output !== undefined) { + if (/Status:\s*failed/i.test(output) || /Exit Code:\s*-?(?!0\b)\d+/i.test(output)) { + return "failed"; + } + if (/Status:\s*completed/i.test(output) || /Status:\s*success/i.test(output)) { + return "completed"; + } + if (/Status:\s*running/i.test(output) || /Status:\s*pending/i.test(output)) { + return "running"; + } + } + return toolCall.status === "completed" ? "completed" : "running"; +} + /** - * Recognizes Grok's private `Task` tool envelope without teaching the generic - * ACP adapter about xAI tool names or result formatting. + * Recognizes Grok Task / spawn_subagent envelopes and get_command hydration + * without teaching the generic ACP adapter about xAI tool names. + * + * Current Grok CLI returns spawn_subagent immediately with a background ACK + * (`subagent_id: ...`). That must stay `running` until child work finishes or + * get_command_or_subagent_output reports a terminal status. */ export function extractXAiAcpSubagentUpdate( toolCall: AcpToolCallState, ): XAiAcpSubagentUpdate | undefined { const rawInput = unknownRecord(toolCall.data.rawInput); - const isTask = - toolCall.title === "Task" || - rawInput?.variant === "CursorTask" || - nonEmptyString(rawInput?.subagent_type) !== undefined; - if (!isTask || rawInput === undefined) return undefined; - - const output = xAiTaskOutputText(toolCall); - const childSessionId = - output?.match(/(?:^|\n)Agent ID:\s*([0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})\b/i)?.[1] ?? - null; - const result = - output?.replace(/(?:^|\n)Agent ID:\s*[0-9a-f-]+[^\n]*(?:\n|$)/gi, "\n").trim() || null; - const status = - toolCall.status === "failed" - ? "failed" - : toolCall.status === "completed" - ? "completed" - : "running"; + const output = xAiToolOutputText(toolCall); + + if (isXAiGetSubagentOutputTool(toolCall, rawInput)) { + const taskIds = taskIdsFromGetOutputTool(toolCall, rawInput); + // Prefer the completed task identity from the result/header over the first + // requested task_ids entry (one call can poll multiple task_ids). + const childSessionId = + resultTaskIdFromGetOutputTool(toolCall, output) ?? + taskIds[0] ?? + extractXAiChildSessionId(output); + if (childSessionId === null) return undefined; + // Prefer the durable subagent row as the completion surface (Claude/Codex + // style). Suppress the noisy TaskOutput tool card; monitor hydration is + // handled separately via extractBackgroundTaskCompletion. + return { + nativeTaskId: toolCall.toolCallId, + prompt: "", + title: null, + model: null, + status: statusFromGetOutputTool(toolCall, output), + childSessionId, + result: resultFromGetOutputTool(toolCall, output), + suppressNormalTool: true, + }; + } + + if (!isXAiSpawnOrTaskTool(toolCall, rawInput)) return undefined; + + const childSessionId = extractXAiChildSessionId(output); + const asyncSpawnAck = isXAiAsyncSpawnAck(output); + const legacyResult = + output + ?.replace(new RegExp(`(?:^|\\n)\\s*Agent ID:\\s*${XAI_UUID_RE}[^\\n]*(?:\\n|$)`, "gi"), "\n") + .replace( + new RegExp(`(?:^|\\n)\\s*subagent_id:\\s*${XAI_UUID_RE}[^\\n]*(?:\\n|$)`, "gi"), + "\n", + ) + .trim() || null; + + let status: "running" | "completed" | "failed"; + if (toolCall.status === "failed") { + status = "failed"; + } else if (asyncSpawnAck) { + // Spawn RPC completed, but the child is still running. + status = "running"; + } else if (toolCall.status === "completed") { + status = "completed"; + } else { + status = "running"; + } return { nativeTaskId: toolCall.toolCallId, - prompt: nonEmptyString(rawInput.prompt) ?? "", - title: nonEmptyString(rawInput.description) ?? null, - model: nonEmptyString(rawInput.model) ?? null, + prompt: nonEmptyString(rawInput?.prompt) ?? "", + title: nonEmptyString(rawInput?.description) ?? null, + model: nonEmptyString(rawInput?.model) ?? null, status, childSessionId, - result, + result: asyncSpawnAck ? null : legacyResult, + suppressNormalTool: true, }; } @@ -283,82 +974,50 @@ export function makeXAiAskUserQuestionCancelledResponse(): XAiAskUserQuestionCan return { outcome: "cancelled" }; } -/** - * Adds Grok's private prompt-completion fallback around a standards-only ACP runtime. - * The underlying runtime remains unaware of xAI methods and metadata. - */ -export const makeXAiPromptCompletionRuntime = Effect.fn("makeXAiPromptCompletionRuntime")( - function* (runtime: AcpSessionRuntime.AcpSessionRuntime["Service"]) { - const activeSessionIdRef = yield* Ref.make(undefined); - const pendingRef = yield* Ref.make>([]); - const completedPromptIdsRef = yield* Ref.make>([]); - let nextPromptFallbackId = 0; - const allocatePromptFallbackId = Effect.sync(() => { - nextPromptFallbackId += 1; - return `t3-xai-prompt-${nextPromptFallbackId}`; - }); - - yield* runtime.handleExtNotification( - "_x.ai/session/prompt_complete", - XAiPromptCompleteNotification, - (notification) => - resolveXAiPromptCompletionFallback({ - pendingRef, - completedPromptIdsRef, - notification, - }), - ); - - return { - ...runtime, - start: () => - runtime - .start() - .pipe(Effect.tap((started) => Ref.set(activeSessionIdRef, started.sessionId))), - prompt: (payload) => - Effect.gen(function* () { - const sessionId = yield* Ref.get(activeSessionIdRef); - if (sessionId === undefined) { - return yield* runtime.prompt(payload); - } +function promptIdFromResponse(response: EffectAcpSchema.PromptResponse): string | undefined { + const meta = response._meta; + if (meta === null || typeof meta !== "object") { + return undefined; + } + const promptId = meta.promptId ?? meta.requestId; + return typeof promptId === "string" && promptId.length > 0 ? promptId : undefined; +} - const promptId = yield* allocatePromptFallbackId; - const fallback = yield* registerXAiPromptCompletionFallback( - pendingRef, - sessionId, - promptId, - ); - const requestPayload = { - ...payload, - _meta: { - ...payload._meta, - promptId: fallback.promptId, - requestId: fallback.promptId, - }, - } satisfies Omit; +function normalizeXAiStopReason(value: string | undefined): EffectAcpSchema.StopReason { + switch (value) { + case "cancelled": + case "end_turn": + case "max_tokens": + case "max_turn_requests": + case "refusal": + return value; + default: + return "end_turn"; + } +} - return yield* Effect.raceFirst( - runtime.prompt(requestPayload), - Deferred.await(fallback.deferred), - ).pipe( - Effect.tap((response) => - rememberCompletedXAiPromptId(completedPromptIdsRef, response, fallback.promptId), - ), - Effect.ensuring(unregisterXAiPromptCompletionFallback(pendingRef, fallback.deferred)), - ); - }), - cancel: Ref.get(activeSessionIdRef).pipe( - Effect.flatMap((sessionId) => - sessionId === undefined - ? runtime.cancel - : abortPendingPromptCompletions(pendingRef, sessionId).pipe( - Effect.andThen(runtime.cancel), - ), - ), - ), - } satisfies AcpSessionRuntime.AcpSessionRuntime["Service"]; - }, -); +function promptResponseFromXAi( + notification: XAiPromptCompleteNotification, +): EffectAcpSchema.PromptResponse { + const stopReason = normalizeXAiStopReason(notification.stopReason); + const meta: Record = { + sessionId: notification.sessionId, + }; + if (notification.stopReason === undefined) { + meta[xAiStopReasonMissingMetaKey] = true; + } + if (notification.promptId !== undefined) { + meta.promptId = notification.promptId; + meta.requestId = notification.promptId; + } + if (notification.agentResult !== undefined) { + meta.agentResult = notification.agentResult; + } + return { + stopReason, + _meta: meta, + }; +} const registerXAiPromptCompletionFallback = ( pendingRef: Ref.Ref>, @@ -436,7 +1095,15 @@ const resolveXAiPromptCompletionFallback = ({ entry.sessionId === notification.sessionId && entry.promptId === notification.promptId, ) - : pending.findIndex((entry) => entry.sessionId === notification.sessionId); + : (() => { + const sessionPendingIndexes = pending.flatMap((entry, entryIndex) => + entry.sessionId === notification.sessionId ? [entryIndex] : [], + ); + if (sessionPendingIndexes.length !== 1) { + return -1; + } + return sessionPendingIndexes[0] ?? -1; + })(); if (index < 0) { return [Effect.void, pending] as const; } @@ -458,6 +1125,9 @@ const rememberCompletedXAiPromptId = ( fallbackPromptId: string, ) => { const promptId = promptIdFromResponse(response) ?? fallbackPromptId; + if (promptId.length === 0) { + return Effect.void; + } return Ref.update(completedPromptIdsRef, (completedPromptIds) => { if (completedPromptIds.includes(promptId)) { return completedPromptIds; @@ -466,14 +1136,111 @@ const rememberCompletedXAiPromptId = ( }); }; -function promptIdFromResponse(response: EffectAcpSchema.PromptResponse): string | undefined { - const meta = response._meta; - if (meta === null || typeof meta !== "object") { - return undefined; - } - const promptId = meta.promptId ?? meta.requestId; - return typeof promptId === "string" && promptId.length > 0 ? promptId : undefined; -} +/** + * Grok-specific ACP runtime wrapper. Races `session/prompt` against root-matched + * terminal notifications: + * - `_x.ai/session/update` + `turn_completed` + matching `prompt_id` (current CLI) + * - `_x.ai/session/prompt_complete` (legacy / alternate builds) + * + * Pending entries are keyed by root sessionId + T3-injected promptId, so + * foreign/child sessions and `task-completed-*` ids do not settle the root turn. + */ +export const makeXAiPromptCompletionRuntime = Effect.fn("makeXAiPromptCompletionRuntime")( + function* (runtime: AcpSessionRuntime.AcpSessionRuntime["Service"]) { + let nextPromptFallbackId = 0; + const allocatePromptFallbackId = Effect.sync(() => { + nextPromptFallbackId += 1; + return `t3-xai-prompt-${nextPromptFallbackId}`; + }); + const pendingXAiPromptCompletionsRef = yield* Ref.make< + ReadonlyArray + >([]); + const completedXAiPromptIdsRef = yield* Ref.make>([]); + + const settleFromPromptComplete = (notification: XAiPromptCompleteNotification) => + resolveXAiPromptCompletionFallback({ + pendingRef: pendingXAiPromptCompletionsRef, + completedPromptIdsRef: completedXAiPromptIdsRef, + notification, + }).pipe(Effect.catch(() => Effect.void)); + + yield* runtime.handleExtNotification( + "_x.ai/session/prompt_complete", + XAiPromptCompleteNotification, + settleFromPromptComplete, + ); + + yield* runtime.handleExtNotification( + "_x.ai/session/update", + XAiSessionUpdateNotification, + (notification) => { + const complete = xAiPromptCompleteFromSessionUpdate(notification); + if (complete === null) { + return Effect.void; + } + return settleFromPromptComplete(complete); + }, + ); + + return { + ...runtime, + prompt: (payload) => + Effect.gen(function* () { + const started = yield* runtime.start(); + const promptId = yield* allocatePromptFallbackId; + const fallback = yield* registerXAiPromptCompletionFallback( + pendingXAiPromptCompletionsRef, + started.sessionId, + promptId, + ); + const cancelledResponse = promptResponseFromXAi({ + sessionId: started.sessionId, + promptId: fallback.promptId, + stopReason: "cancelled", + agentResult: null, + }); + const promptRpcFiber = yield* runtime + .prompt({ + ...payload, + _meta: { + ...payload._meta, + promptId: fallback.promptId, + requestId: fallback.promptId, + }, + }) + .pipe(Effect.forkChild); + return yield* Effect.raceFirst( + Fiber.join(promptRpcFiber).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.succeed(cancelledResponse) + : Effect.failCause(cause), + ), + ), + Deferred.await(fallback.deferred), + ).pipe( + Effect.tap((response) => + rememberCompletedXAiPromptId(completedXAiPromptIdsRef, response, fallback.promptId), + ), + Effect.ensuring( + Effect.gen(function* () { + yield* Fiber.interrupt(promptRpcFiber).pipe(Effect.ignore); + yield* unregisterXAiPromptCompletionFallback( + pendingXAiPromptCompletionsRef, + fallback.deferred, + ); + }), + ), + ); + }), + cancel: Effect.gen(function* () { + const started = yield* runtime.start(); + yield* abortPendingPromptCompletions(pendingXAiPromptCompletionsRef, started.sessionId); + yield* runtime.cancel; + }), + } satisfies AcpSessionRuntime.AcpSessionRuntime["Service"]; + }, +); export function promptResponseHasMissingXAiStopReason( response: EffectAcpSchema.PromptResponse, @@ -481,39 +1248,3 @@ export function promptResponseHasMissingXAiStopReason( const meta = response._meta; return meta !== null && typeof meta === "object" && meta[xAiStopReasonMissingMetaKey] === true; } - -function promptResponseFromXAi( - notification: XAiPromptCompleteNotification, -): EffectAcpSchema.PromptResponse { - const stopReason = normalizeXAiStopReason(notification.stopReason); - const meta: Record = { - sessionId: notification.sessionId, - }; - if (notification.stopReason === undefined) { - meta[xAiStopReasonMissingMetaKey] = true; - } - if (notification.promptId !== undefined) { - meta.promptId = notification.promptId; - meta.requestId = notification.promptId; - } - if (notification.agentResult !== undefined) { - meta.agentResult = notification.agentResult; - } - return { - stopReason, - _meta: meta, - }; -} - -function normalizeXAiStopReason(value: string | undefined): EffectAcpSchema.StopReason { - switch (value) { - case "cancelled": - case "end_turn": - case "max_tokens": - case "max_turn_requests": - case "refusal": - return value; - default: - return "end_turn"; - } -} diff --git a/apps/web/src/components/ChatView.logic.test.ts b/apps/web/src/components/ChatView.logic.test.ts index 750a6c823f6..02e3c830f8b 100644 --- a/apps/web/src/components/ChatView.logic.test.ts +++ b/apps/web/src/components/ChatView.logic.test.ts @@ -5,7 +5,10 @@ import { ProviderInstanceId, ThreadId, RunId, + TurnItemId, + type OrchestrationV2ProjectedTurnItem, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import { describe, expect, it } from "vite-plus/test"; import type { Thread } from "../types"; @@ -15,6 +18,7 @@ import { MAX_HIDDEN_MOUNTED_TERMINAL_THREADS, buildExpiredTerminalContextToastCopy, createLocalDispatchSnapshot, + deriveCommittedServerUserMessageIds, deriveComposerSendState, getStartedThreadModelChangeBlockReason, hasServerAcknowledgedLocalDispatch, @@ -463,3 +467,103 @@ describe("hasServerAcknowledgedLocalDispatch", () => { expect(hasServerAcknowledgedLocalDispatch({ ...common, threadError: "failed" })).toBe(true); }); }); + +describe("deriveCommittedServerUserMessageIds", () => { + it("tracks only committed user turn items, not assistant rows or projection-only messages", () => { + const turnStartId = MessageId.make("message-turn-start"); + const steerId = MessageId.make("message-steer"); + const assistantId = MessageId.make("message-assistant"); + const committedAt = DateTime.makeUnsafe("2026-06-26T17:50:15.180Z"); + const runId = RunId.make("run:thread:thread-1:ordinal:1"); + const visibleTurnItems: ReadonlyArray = [ + { + position: 0, + visibility: "local", + sourceThreadId: threadId, + sourceItemId: TurnItemId.make("turn-item:message-turn-start"), + item: { + id: TurnItemId.make("turn-item:message-turn-start"), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "completed", + title: null, + startedAt: committedAt, + completedAt: committedAt, + updatedAt: committedAt, + createdBy: "user", + creationSource: "web", + type: "user_message", + messageId: turnStartId, + inputIntent: "turn_start", + text: "start", + attachments: [], + }, + }, + { + position: 1, + visibility: "local", + sourceThreadId: threadId, + sourceItemId: TurnItemId.make("turn-item:message-assistant"), + item: { + id: TurnItemId.make("turn-item:message-assistant"), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "completed", + title: null, + startedAt: committedAt, + completedAt: committedAt, + updatedAt: committedAt, + type: "assistant_message", + messageId: assistantId, + text: "working", + streaming: false, + }, + }, + { + position: 2, + visibility: "local", + sourceThreadId: threadId, + sourceItemId: TurnItemId.make("turn-item:message-steer"), + item: { + id: TurnItemId.make("turn-item:message-steer"), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 3, + status: "completed", + title: null, + startedAt: committedAt, + completedAt: committedAt, + updatedAt: committedAt, + createdBy: "user", + creationSource: "web", + type: "user_message", + messageId: steerId, + inputIntent: "steer", + text: "continue", + attachments: [], + }, + }, + ]; + + expect(deriveCommittedServerUserMessageIds(visibleTurnItems)).toEqual( + new Set([turnStartId, steerId]), + ); + }); +}); diff --git a/apps/web/src/components/ChatView.logic.ts b/apps/web/src/components/ChatView.logic.ts index e3f2529f52c..069f3e0366f 100644 --- a/apps/web/src/components/ChatView.logic.ts +++ b/apps/web/src/components/ChatView.logic.ts @@ -3,6 +3,7 @@ import { isProviderDriverKind, ProjectId, type ModelSelection, + type OrchestrationV2ProjectedTurnItem, type ProviderDriverKind, type ServerProvider, type ScopedThreadRef, @@ -392,6 +393,22 @@ export function createLocalDispatchSnapshot( }; } +/** + * The timeline renders committed user rows from `visibleTurnItems`, but + * `message.updated` can land in `projection.messages` one event earlier than + * the matching `turn-item.updated`. Basing optimistic eviction on visible user + * turn items avoids dropping steer rows in that gap. + */ +export function deriveCommittedServerUserMessageIds( + visibleTurnItems: ReadonlyArray, +): ReadonlySet { + return new Set( + visibleTurnItems.flatMap((row) => + row.item.type === "user_message" ? [row.item.messageId] : [], + ), + ); +} + export function hasServerAcknowledgedLocalDispatch(input: { localDispatch: LocalDispatchSnapshot | null; phase: SessionPhase; diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 23cfef5e7f1..71239eec578 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -247,6 +247,7 @@ import { buildLocalDraftThread, collectUserMessageBlobPreviewUrls, createLocalDispatchSnapshot, + deriveCommittedServerUserMessageIds, deriveComposerSendState, hasServerAcknowledgedLocalDispatch, getStartedThreadModelChangeBlockReason, @@ -1154,8 +1155,8 @@ function ChatViewContent(props: ChatViewProps) { const serverProjection = serverThreadProjection?.projection ?? null; const serverVisibleTurnItems = useThreadVisibleTurnItems(routeThreadRef); const committedServerMessageIds = useMemo( - () => new Set(serverProjection?.messages.map((message) => message.id) ?? []), - [serverProjection], + () => deriveCommittedServerUserMessageIds(serverVisibleTurnItems), + [serverVisibleTurnItems], ); const markThreadVisited = useUiStateStore((store) => store.markThreadVisited); const activeThreadLastVisitedAt = useUiStateStore( @@ -1444,7 +1445,11 @@ function ChatViewContent(props: ChatViewProps) { const isLocalDraftThread = !isServerThread && localDraftThread !== undefined; const canCheckoutPullRequestIntoThread = isLocalDraftThread; const activeThreadId = activeThread?.id ?? null; - const activeMessageCount = isServerThread ? committedServerMessageIds.size : 0; + // Prefer the larger of turn-item-committed ids and projection messages so + // env lock does not unlock while turn items lag projection hydration. + const activeMessageCount = isServerThread + ? Math.max(committedServerMessageIds.size, serverProjection?.messages.length ?? 0) + : 0; const runningTerminalIds = useThreadRunningTerminalIds({ environmentId: activeThread?.environmentId ?? null, threadId: activeThreadId, diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index 26caa1bf6cd..3721e2ee563 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -1268,10 +1268,6 @@ function ProposedPlanTimelineRow({ type V2EventTone = "muted" | "warning" | "danger" | "success"; -function subagentDisplayTitle(title: string): string { - return title.replace(/^Subagent:\s*/i, ""); -} - function v2EventPresentation(item: OrchestrationV2TurnItem): { readonly label: string; readonly detail: string | null; @@ -1328,14 +1324,6 @@ function v2EventPresentation(item: OrchestrationV2TurnItem): { icon: MinusIcon, }; } - case "subagent": - return { - label: subagentDisplayTitle(item.title ?? "Subagent"), - detail: item.result ?? item.progress ?? item.prompt, - tone: - item.status === "failed" ? "danger" : item.status === "completed" ? "success" : "muted", - icon: BotIcon, - }; case "approval_request": return { label: "Approval requested", @@ -1410,7 +1398,14 @@ function V2EventTimelineRow({ row }: { row: Extract {presentation.label} {item.status !== "completed" ? ( - + {item.status} ) : null} @@ -2265,8 +2260,19 @@ function workEntryPreview( workEntry: Pick, workspaceRoot: string | undefined, ) { + // Prefer stdout/detail so completed shell/monitor results are visible collapsed + // (command alone hid ls listings behind expand-only inspector JSON). + if (workEntry.detail?.trim()) { + const lines = workEntry.detail + .trim() + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0); + if (lines.length > 0) { + return lines.slice(0, 3).join(" · "); + } + } if (workEntry.command) return workEntry.command; - if (workEntry.detail) return workEntry.detail; if ((workEntry.changedFiles?.length ?? 0) === 0) return null; const [firstPath] = workEntry.changedFiles ?? []; if (!firstPath) return null; diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.ts b/packages/client-runtime/src/state/orchestrationV2Projection.ts index c8b7335e6b5..fcdff7365b2 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.ts @@ -39,6 +39,7 @@ function shouldShowLocalTurnItem( item, runs: projection.runs, attempts: projection.attempts, + items: projection.turnItems, }); } diff --git a/packages/effect-acp/src/client.ts b/packages/effect-acp/src/client.ts index 61b3d71b49d..e64d0095089 100644 --- a/packages/effect-acp/src/client.ts +++ b/packages/effect-acp/src/client.ts @@ -26,6 +26,10 @@ export interface AcpClientOptions { readonly logIncoming?: boolean; readonly logOutgoing?: boolean; readonly logger?: (event: AcpProtocol.AcpProtocolLogEvent) => Effect.Effect; + readonly onIncomingRequest?: AcpProtocol.AcpPatchedProtocolOptions["onIncomingRequest"]; + readonly onTermination?: AcpProtocol.AcpPatchedProtocolOptions["onTermination"]; + readonly onOutgoingResponseFailure?: AcpProtocol.AcpPatchedProtocolOptions["onOutgoingResponseFailure"]; + readonly onOutgoingResponse?: AcpProtocol.AcpPatchedProtocolOptions["onOutgoingResponse"]; } type AcpClientRaw = { @@ -403,6 +407,12 @@ export const make = Effect.fn("effect-acp/AcpClient.make")(function* ( ...(options.logIncoming !== undefined ? { logIncoming: options.logIncoming } : {}), ...(options.logOutgoing !== undefined ? { logOutgoing: options.logOutgoing } : {}), ...(options.logger ? { logger: options.logger } : {}), + ...(options.onIncomingRequest ? { onIncomingRequest: options.onIncomingRequest } : {}), + ...(options.onTermination ? { onTermination: options.onTermination } : {}), + ...(options.onOutgoingResponseFailure + ? { onOutgoingResponseFailure: options.onOutgoingResponseFailure } + : {}), + ...(options.onOutgoingResponse ? { onOutgoingResponse: options.onOutgoingResponse } : {}), onNotification: dispatchNotification, onExtRequest: dispatchExtRequest, }); diff --git a/packages/effect-acp/src/protocol.test.ts b/packages/effect-acp/src/protocol.test.ts index ece068dfc88..e88557490dd 100644 --- a/packages/effect-acp/src/protocol.test.ts +++ b/packages/effect-acp/src/protocol.test.ts @@ -3,8 +3,12 @@ import * as AcpError from "./errors.ts"; import * as Effect from "effect/Effect"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; +import * as Exit from "effect/Exit"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Sink from "effect/Sink"; +import * as Stdio from "effect/Stdio"; import * as Stream from "effect/Stream"; import * as Ref from "effect/Ref"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; @@ -457,6 +461,177 @@ it.layer(NodeServices.layer)("effect-acp protocol", (it) => { }), ); + it.effect("acknowledges outgoing responses and reports encoding failures by request id", () => + Effect.gen(function* () { + const { stdio } = yield* makeInMemoryStdio(); + const acknowledged = yield* Deferred.make(); + const failed = yield* Deferred.make(); + const transport = yield* AcpProtocol.makeAcpPatchedProtocol({ + stdio, + serverRequestMethods: new Set(), + onOutgoingResponse: (requestId) => + Deferred.succeed(acknowledged, requestId).pipe(Effect.asVoid), + onOutgoingResponseFailure: (requestId, error) => + Deferred.succeed(failed, [requestId, error]).pipe(Effect.asVoid), + }); + + yield* transport.serverProtocol.send(0, { + _tag: "Exit", + requestId: "success-1", + exit: { _tag: "Success", value: { ok: true } }, + }); + assert.equal(yield* Deferred.await(acknowledged), "success-1"); + + const circular: { self?: unknown } = {}; + circular.self = circular; + yield* transport.serverProtocol + .send(0, { + _tag: "Exit", + requestId: "failure-1", + exit: { _tag: "Success", value: circular }, + }) + .pipe(Effect.exit); + const [requestId, error] = yield* Deferred.await(failed); + assert.equal(requestId, "failure-1"); + assert.instanceOf(error, AcpError.AcpProtocolParseError); + }), + ); + + it.effect("acknowledges a response only after the stdout write completes", () => + Effect.gen(function* () { + const writeStarted = yield* Deferred.make(); + const releaseWrite = yield* Deferred.make(); + const acknowledged = yield* Deferred.make(); + const stdio = Stdio.make({ + args: Effect.succeed([]), + stdin: Stream.never, + stdout: () => + Sink.forEach(() => + Deferred.succeed(writeStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseWrite)), + ), + ), + stderr: () => Sink.drain, + }); + const transport = yield* AcpProtocol.makeAcpPatchedProtocol({ + stdio, + serverRequestMethods: new Set(), + onOutgoingResponse: (requestId) => + Deferred.succeed(acknowledged, requestId).pipe(Effect.asVoid), + }); + const sendFiber = yield* transport.serverProtocol + .send(0, { + _tag: "Exit", + requestId: "held-write", + exit: { _tag: "Success", value: { ok: true } }, + }) + .pipe(Effect.forkScoped); + + yield* Deferred.await(writeStarted); + assert.isUndefined(sendFiber.pollUnsafe()); + assert.isFalse(yield* Deferred.isDone(acknowledged)); + yield* Deferred.succeed(releaseWrite, undefined); + yield* Fiber.join(sendFiber); + assert.equal(yield* Deferred.await(acknowledged), "held-write"); + }), + ); + + it.effect("fails current, queued, and future responses when the stdout writer fails", () => + Effect.gen(function* () { + const writeStarted = yield* Deferred.make(); + const releaseFailure = yield* Deferred.make(); + const failures = yield* Queue.unbounded(); + const terminated = yield* Deferred.make(); + const stdio = Stdio.make({ + args: Effect.succeed([]), + stdin: Stream.never, + stdout: () => + Sink.forEach(() => + Deferred.succeed(writeStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseFailure)), + Effect.andThen(Effect.die("stdout failed")), + ), + ), + stderr: () => Sink.drain, + }); + const transport = yield* AcpProtocol.makeAcpPatchedProtocol({ + stdio, + serverRequestMethods: new Set(), + onOutgoingResponseFailure: (requestId) => Queue.offer(failures, requestId), + onTermination: (error) => Deferred.succeed(terminated, error).pipe(Effect.asVoid), + }); + const send = (requestId: string) => + transport.serverProtocol + .send(0, { + _tag: "Exit", + requestId, + exit: { _tag: "Success", value: { ok: true } }, + }) + .pipe(Effect.exit); + const current = yield* send("current").pipe(Effect.forkScoped); + yield* Deferred.await(writeStarted); + const queued = yield* send("queued").pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.isUndefined(current.pollUnsafe()); + assert.isUndefined(queued.pollUnsafe()); + yield* Deferred.succeed(releaseFailure, undefined); + assert.instanceOf(yield* Deferred.await(terminated), AcpError.AcpTransportError); + yield* Fiber.join(current); + yield* Fiber.join(queued); + yield* send("future"); + + assert.deepEqual( + new Set(yield* Queue.takeN(failures, 3)), + new Set(["current", "queued", "future"]), + ); + }), + ); + + it.effect( + "atomically rejects an admitted response when the writer closes before queue offer", + () => + Effect.gen(function* () { + const admitted = yield* Deferred.make(); + const releaseAdmission = yield* Deferred.make(); + const failures = yield* Queue.unbounded(); + const protocolScope = yield* Scope.make(); + const stdio = Stdio.make({ + args: Effect.succeed([]), + stdin: Stream.never, + stdout: () => Sink.drain, + stderr: () => Sink.drain, + }); + const transport = yield* AcpProtocol.makeAcpPatchedProtocol({ + stdio, + serverRequestMethods: new Set(), + onOutgoingResponseFailure: (requestId) => Queue.offer(failures, requestId), + testHooks: { + onOutgoingWriteAdmitted: () => + Deferred.succeed(admitted, undefined).pipe( + Effect.andThen(Deferred.await(releaseAdmission)), + ), + }, + }).pipe(Effect.provideService(Scope.Scope, protocolScope)); + const send = (requestId: string) => + transport.serverProtocol + .send(0, { + _tag: "Exit", + requestId, + exit: { _tag: "Success", value: { ok: true } }, + }) + .pipe(Effect.exit); + const admittedSend = yield* send("admitted").pipe(Effect.forkScoped); + + yield* Deferred.await(admitted); + yield* Scope.close(protocolScope, Exit.void); + yield* Deferred.succeed(releaseAdmission, undefined); + yield* Fiber.join(admittedSend); + yield* send("future"); + + assert.deepEqual(yield* Queue.takeN(failures, 2), ["admitted", "future"]); + }), + ); + it.effect("cleans up interrupted extension requests before a late response arrives", () => Effect.gen(function* () { const { stdio, input, output } = yield* makeInMemoryStdio(); @@ -562,6 +737,31 @@ it.layer(NodeServices.layer)("effect-acp protocol", (it) => { }), ); + it.effect("does not treat intentional serverProtocol.end as a transport failure", () => + Effect.gen(function* () { + const { stdio, input } = yield* makeInMemoryStdio(); + const terminations = yield* Ref.make(0); + const writerExit = yield* Deferred.make<{ readonly intentionalEnd: boolean }>(); + const transport = yield* AcpProtocol.makeAcpPatchedProtocol({ + stdio, + serverRequestMethods: new Set(), + onTermination: () => Ref.update(terminations, (count) => count + 1), + testHooks: { + onOutgoingWriterExit: (detail) => + Deferred.succeed(writerExit, detail).pipe(Effect.asVoid), + }, + }); + + yield* transport.serverProtocol.end(0); + const exitDetail = yield* Deferred.await(writerExit); + assert.isTrue(exitDetail.intentionalEnd); + assert.equal(yield* Ref.get(terminations), 0); + + // Tear down the forever input reader so the scoped test can finish. + yield* Queue.end(input); + }), + ); + it.effect("does not emit a second process-exit error after a decode failure", () => Effect.gen(function* () { const handle = yield* makeHandle({ diff --git a/packages/effect-acp/src/protocol.ts b/packages/effect-acp/src/protocol.ts index 27c619296c0..cd07dcdb4aa 100644 --- a/packages/effect-acp/src/protocol.ts +++ b/packages/effect-acp/src/protocol.ts @@ -1,6 +1,7 @@ import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as Deferred from "effect/Deferred"; +import * as Exit from "effect/Exit"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -48,6 +49,11 @@ export interface AcpPatchedProtocolOptions { readonly logIncoming?: boolean; readonly logOutgoing?: boolean; readonly logger?: (event: AcpProtocolLogEvent) => Effect.Effect; + readonly onIncomingRequest?: ( + requestId: string, + method: string, + payload: unknown, + ) => Effect.Effect; readonly onNotification?: ( notification: AcpIncomingNotification, ) => Effect.Effect; @@ -56,6 +62,18 @@ export interface AcpPatchedProtocolOptions { params: unknown, ) => Effect.Effect; readonly onTermination?: (error: AcpError.AcpError) => Effect.Effect; + readonly onOutgoingResponseFailure?: ( + requestId: string, + error: AcpError.AcpError, + ) => Effect.Effect; + readonly onOutgoingResponse?: (requestId: string) => Effect.Effect; + readonly testHooks?: { + readonly onOutgoingWriteAdmitted?: () => Effect.Effect; + /** Invoked from the stdout writer exit path (after intentional/unexpected handling). */ + readonly onOutgoingWriterExit?: (input: { + readonly intentionalEnd: boolean; + }) => Effect.Effect; + }; } export interface AcpPatchedProtocol { @@ -71,6 +89,17 @@ interface AcpPendingRequest { readonly method: string; } +interface AcpOutgoingWrite { + readonly acknowledgement?: Deferred.Deferred; + readonly payload: string | Uint8Array; +} + +interface AcpOutgoingWriterState { + readonly outstandingAcknowledgements: ReadonlySet>; + readonly intentionalEnd: boolean; + readonly terminalError?: AcpError.AcpError; +} + const decodeSessionUpdate = Schema.decodeUnknownEffect(AcpSchema.SessionNotification); const decodeElicitationComplete = Schema.decodeUnknownEffect( AcpSchema.ElicitationCompleteNotification, @@ -85,7 +114,11 @@ export const makeAcpPatchedProtocol = Effect.fn("makeAcpPatchedProtocol")(functi const clientQueue = yield* Queue.unbounded(); const notificationQueue = yield* Queue.unbounded(); const disconnects = yield* Queue.unbounded(); - const outgoing = yield* Queue.unbounded>(); + const outgoing = yield* Queue.unbounded>(); + const outgoingWriterState = yield* Ref.make({ + intentionalEnd: false, + outstandingAcknowledgements: new Set(), + }); const nextRequestId = yield* Ref.make(1n); const terminationHandled = yield* Ref.make(false); const extPending = yield* Ref.make(new Map()); @@ -132,7 +165,66 @@ export const makeAcpPatchedProtocol = Effect.fn("makeAcpPatchedProtocol")(functi payload: typeof encoded === "string" ? encoded : new TextDecoder().decode(encoded), }); - yield* Queue.offer(outgoing, encoded).pipe(Effect.asVoid); + const acknowledgement = + message._tag === "Exit" ? yield* Deferred.make() : undefined; + const admissionError = yield* Ref.modify(outgoingWriterState, (state) => { + if (state.terminalError !== undefined) { + return [state.terminalError, state] as const; + } + if (acknowledgement === undefined) { + return [undefined, state] as const; + } + return [ + undefined, + { + ...state, + outstandingAcknowledgements: new Set(state.outstandingAcknowledgements).add( + acknowledgement, + ), + }, + ] as const; + }); + if (admissionError !== undefined) { + return yield* admissionError; + } + if (options.testHooks?.onOutgoingWriteAdmitted !== undefined) { + yield* options.testHooks.onOutgoingWriteAdmitted(); + } + yield* Queue.offer(outgoing, { + payload: encoded, + ...(acknowledgement === undefined ? {} : { acknowledgement }), + }).pipe( + Effect.mapError( + (cause) => + new AcpError.AcpTransportError({ + detail: "Failed to queue an outgoing ACP message", + cause, + }), + ), + Effect.filterOrFail( + (accepted) => accepted, + () => + new AcpError.AcpTransportError({ + detail: "The ACP output queue was closed before accepting a message", + cause: "Output queue closed", + }), + ), + Effect.tapError((error) => + acknowledgement === undefined + ? Effect.void + : Ref.update(outgoingWriterState, (state) => { + const updated = new Set(state.outstandingAcknowledgements); + updated.delete(acknowledgement); + return { ...state, outstandingAcknowledgements: updated }; + }).pipe(Effect.andThen(Deferred.fail(acknowledgement, error)), Effect.asVoid), + ), + ); + if (acknowledgement !== undefined) { + yield* Deferred.await(acknowledgement); + } + if (message._tag === "Exit" && options.onOutgoingResponse !== undefined) { + yield* options.onOutgoingResponse(message.requestId); + } } }); @@ -308,8 +400,11 @@ export const makeAcpPatchedProtocol = Effect.fn("makeAcpPatchedProtocol")(functi }); } + const observeIncoming = + options.onIncomingRequest?.(message.id, message.tag, message.payload) ?? Effect.void; + if (!options.serverRequestMethods.has(message.tag)) { - return handleExtRequest(message).pipe( + return observeIncoming.pipe(Effect.andThen(handleExtRequest(message))).pipe( Effect.catchTags({ AcpProtocolParseError: (error) => Effect.logWarning(error).pipe( @@ -334,7 +429,7 @@ export const makeAcpPatchedProtocol = Effect.fn("makeAcpPatchedProtocol")(functi ); } - return Queue.offer(serverQueue, message).pipe(Effect.asVoid); + return observeIncoming.pipe(Effect.andThen(Queue.offer(serverQueue, message)), Effect.asVoid); }; const handleExitEncoded = (message: RpcMessage.ResponseExitEncoded) => @@ -473,7 +568,112 @@ export const makeAcpPatchedProtocol = Effect.fn("makeAcpPatchedProtocol")(functi Effect.forkScoped, ); - yield* Stream.fromQueue(outgoing).pipe(Stream.run(options.stdio.stdout()), Effect.forkScoped); + const failOutgoingWrites = (error: AcpError.AcpError) => + Ref.modify(outgoingWriterState, (state) => { + const terminalError = state.terminalError ?? error; + return [ + [terminalError, state.outstandingAcknowledgements] as const, + { + ...state, + terminalError, + outstandingAcknowledgements: new Set>(), + }, + ] as const; + }).pipe( + Effect.flatMap((acknowledgements) => + Effect.forEach( + acknowledgements[1], + (acknowledgement) => Deferred.fail(acknowledgement, acknowledgements[0]), + { concurrency: "unbounded", discard: true }, + ), + ), + Effect.andThen(Queue.fail(outgoing, error)), + Effect.asVoid, + ); + const completeOutgoingWrites = () => + Ref.modify(outgoingWriterState, (state) => { + return [ + state.outstandingAcknowledgements, + { + ...state, + outstandingAcknowledgements: new Set>(), + }, + ] as const; + }).pipe( + Effect.flatMap((acknowledgements) => + Effect.forEach( + acknowledgements, + (acknowledgement) => Deferred.succeed(acknowledgement, undefined), + { concurrency: "unbounded", discard: true }, + ), + ), + Effect.asVoid, + ); + const acknowledgeOutgoingWrite = (acknowledgement: Deferred.Deferred) => + Ref.update(outgoingWriterState, (state) => { + const updated = new Set(state.outstandingAcknowledgements); + updated.delete(acknowledgement); + return { ...state, outstandingAcknowledgements: updated }; + }).pipe(Effect.andThen(Deferred.succeed(acknowledgement, undefined)), Effect.asVoid); + yield* Stream.fromQueue(outgoing).pipe( + Stream.flatMap((write) => { + const acknowledgement = write.acknowledgement; + const completion = + acknowledgement === undefined + ? Stream.empty + : Stream.fromEffect(acknowledgeOutgoingWrite(acknowledgement)).pipe(Stream.drain); + return Stream.make(write.payload).pipe(Stream.concat(completion)); + }), + Stream.run(options.stdio.stdout()), + Effect.onExit((exit) => + Exit.match(exit, { + onFailure: (cause) => { + const error = new AcpError.AcpTransportError({ + detail: Cause.hasInterruptsOnly(cause) + ? "The ACP output writer was interrupted while closing" + : "Failed to write an outgoing ACP message", + cause, + }); + return failOutgoingWrites(error).pipe( + Effect.andThen( + Cause.hasInterruptsOnly(cause) + ? Effect.void + : handleTermination(() => Effect.succeed(error)), + ), + ); + }, + onSuccess: () => + Ref.get(outgoingWriterState).pipe( + Effect.flatMap((state) => { + // Intentional Queue.end (serverProtocol.end) completes the writer + // successfully. Do not invent a transport failure or client protocol + // error for a clean RPC shutdown. + const handled = state.intentionalEnd + ? completeOutgoingWrites() + : (() => { + const error = new AcpError.AcpTransportError({ + detail: "ACP output writer ended before the protocol closed", + cause: "Output writer ended", + }); + return failOutgoingWrites(error).pipe( + Effect.andThen(handleTermination(() => Effect.succeed(error))), + ); + })(); + return handled.pipe( + Effect.andThen( + options.testHooks?.onOutgoingWriterExit === undefined + ? Effect.void + : options.testHooks.onOutgoingWriterExit({ + intentionalEnd: state.intentionalEnd, + }), + ), + ); + }), + ), + }), + ), + Effect.forkScoped, + ); const clientProtocol = RpcClient.Protocol.of({ run: (_clientId, f) => @@ -504,8 +704,19 @@ export const makeAcpPatchedProtocol = Effect.fn("makeAcpPatchedProtocol")(functi Effect.forever, ), disconnects, - send: (_clientId, response) => offerOutgoing(response).pipe(Effect.orDie), - end: (_clientId) => Queue.end(outgoing), + send: (_clientId, response) => + offerOutgoing(response).pipe( + Effect.tapError((error) => + response._tag === "Exit" && options.onOutgoingResponseFailure !== undefined + ? options.onOutgoingResponseFailure(response.requestId, error) + : Effect.void, + ), + Effect.orDie, + ), + end: (_clientId) => + Ref.update(outgoingWriterState, (state) => ({ ...state, intentionalEnd: true })).pipe( + Effect.andThen(Queue.end(outgoing)), + ), clientIds: Effect.succeed(new Set([0])), initialMessage: Effect.succeedNone, supportsAck: true, diff --git a/packages/shared/src/orchestrationV2Timeline.test.ts b/packages/shared/src/orchestrationV2Timeline.test.ts index 829b0f6f4d5..e516cc3cccf 100644 --- a/packages/shared/src/orchestrationV2Timeline.test.ts +++ b/packages/shared/src/orchestrationV2Timeline.test.ts @@ -7,22 +7,52 @@ const runId = RunId.make("run:timeline-visibility"); const nodeId = NodeId.make("node:timeline-visibility"); describe("isOrchestrationV2TurnItemVisible", () => { - it("hides interruption results from superseded attempts", () => { + it("hides unpaired interruption results from superseded attempts", () => { expect( isOrchestrationV2TurnItemVisible({ item: { type: "run_interrupt_result", runId, nodeId }, runs: [{ id: runId, status: "running" }], attempts: [{ runId, rootNodeId: nodeId, status: "superseded" }], + items: [{ type: "run_interrupt_result", runId, nodeId }], }), ).toBe(false); }); - it("keeps interruption results from terminal attempts", () => { + it("keeps paired interruption results from superseded attempts", () => { + expect( + isOrchestrationV2TurnItemVisible({ + item: { type: "run_interrupt_result", runId, nodeId }, + runs: [{ id: runId, status: "running" }], + attempts: [{ runId, rootNodeId: nodeId, status: "superseded" }], + items: [ + { type: "run_interrupt_request", runId, nodeId }, + { type: "run_interrupt_result", runId, nodeId }, + ], + }), + ).toBe(true); + }); + + it("keeps interruption results from terminal attempts without a request", () => { expect( isOrchestrationV2TurnItemVisible({ item: { type: "run_interrupt_result", runId, nodeId }, runs: [{ id: runId, status: "interrupted" }], attempts: [{ runId, rootNodeId: nodeId, status: "interrupted" }], + items: [{ type: "run_interrupt_result", runId, nodeId }], + }), + ).toBe(true); + }); + + it("keeps interruption results from terminal attempts with a request", () => { + expect( + isOrchestrationV2TurnItemVisible({ + item: { type: "run_interrupt_result", runId, nodeId }, + runs: [{ id: runId, status: "interrupted" }], + attempts: [{ runId, rootNodeId: nodeId, status: "interrupted" }], + items: [ + { type: "run_interrupt_request", runId, nodeId }, + { type: "run_interrupt_result", runId, nodeId }, + ], }), ).toBe(true); }); @@ -40,6 +70,7 @@ describe("isOrchestrationV2TurnItemVisible", () => { }, { runId, rootNodeId: nodeId, status: "interrupted" }, ], + items: [{ type: "run_interrupt_result", runId, nodeId }], }), ).toBe(true); }); diff --git a/packages/shared/src/orchestrationV2Timeline.ts b/packages/shared/src/orchestrationV2Timeline.ts index 7eed78db8f2..222eae9e98e 100644 --- a/packages/shared/src/orchestrationV2Timeline.ts +++ b/packages/shared/src/orchestrationV2Timeline.ts @@ -11,24 +11,36 @@ type TimelineTurnItem = Pick; + readonly items: ReadonlyArray; }): boolean { const { item } = input; if (item.type !== "run_interrupt_result" || item.runId === null || item.nodeId === null) { return false; } - return input.attempts.some( + const isSuperseded = input.attempts.some( (attempt) => attempt.runId === item.runId && attempt.rootNodeId === item.nodeId && attempt.status === "superseded", ); + if (!isSuperseded) { + return false; + } + + // Paired stop-then-steer results have a matching request on the same run and + // must stay visible. Legacy plain-steer results have no request and stay hidden. + const hasMatchingRequest = input.items.some( + (candidate) => candidate.type === "run_interrupt_request" && candidate.runId === item.runId, + ); + return !hasMatchingRequest; } export function isOrchestrationV2TurnItemVisible(input: { readonly item: TimelineTurnItem; readonly runs: ReadonlyArray; readonly attempts: ReadonlyArray; + readonly items: ReadonlyArray; }): boolean { const { item } = input; if ( @@ -38,5 +50,9 @@ export function isOrchestrationV2TurnItemVisible(input: { return false; } - return !isOrchestrationV2SupersededInterrupt({ item, attempts: input.attempts }); + return !isOrchestrationV2SupersededInterrupt({ + item, + attempts: input.attempts, + items: input.items, + }); }