diff --git a/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx index c581cbc688142..5f83979197bbc 100644 --- a/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx @@ -294,7 +294,10 @@ function EnvironmentDetail({ environmentId }: { readonly environmentId: Environm void run("refresh", async () => { if (!readEnvironmentScope(environmentId, AuthOrchestrationReadScope)) return; - const result = await refreshProviders({ environmentId, input: {} }); + const result = await refreshProviders({ + environmentId, + input: { reloadSecrets: true }, + }); if (AsyncResult.isFailure(result)) throw squashAtomCommandFailure(result); setNotice("Provider status refreshed."); }); diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index 27655f503bd65..6695e7963eff0 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -46,6 +46,7 @@ import type { ScopedThreadRef, ServerConfig as T3ServerConfig, UsageLimitsReport, + RunId, } from "@t3tools/contracts"; import { appendCodexArtifactTemplateUsePrompt, @@ -156,8 +157,10 @@ import { import { appAtomRegistry } from "../../state/atom-registry"; import { threadStreamHaptics } from "./threadStreamHaptics"; import { useAtomCommand } from "../../state/use-atom-command"; +import { orchestrationEnvironment } from "../../state/orchestration"; import { useDelayedStatus } from "../../lib/useDelayedStatus"; import type { PendingUserInputDraftAnswer } from "@t3tools/client-runtime/state/pending-user-input"; +import type { RunStopPresentation } from "@t3tools/client-runtime/state/thread-execution"; import type { PendingApproval, PendingUserInput, @@ -339,6 +342,9 @@ const threadVisitTracker = createThreadVisitTracker({ // Past the composer's 180 ms height animation. const COMPOSER_HEIGHT_SETTLE_MS = 220; +const NO_RUN_STOP_ATOM = Atom.make(null).pipe( + Atom.withLabel("mobile-thread-run-stop:empty"), +); // Anchors, the submitted message, follow state and pending scrolls belong to // one thread, so the body remounts when the thread changes. @@ -528,6 +534,50 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) { const unsettleThread = useAtomCommand(threadEnvironment.unsettle, { reportFailure: false }); const updateMetadata = useAtomCommand(threadEnvironment.updateMetadata, { reportFailure: false }); const visitThread = useAtomCommand(threadEnvironment.visit, { reportFailure: false }); + const anyRunStop = useAtomValue( + props.selectedThreadDetailRef === null + ? NO_RUN_STOP_ATOM + : environmentThreadDetails.runStopAtom(props.selectedThreadDetailRef), + ); + // A background Stop matters only while that work is still pending. + const runStop = + anyRunStop?.background && props.selectedThread.pendingBackgroundTasks.length === 0 + ? null + : anyRunStop; + const forceStopThread = useAtomCommand(orchestrationEnvironment.forceStopThread, { + reportFailure: false, + }); + const canForceStop = + useAtomValue(orchestrationEnvironment.forceStopThread.permissionAtom(props.environmentId)) && + props.serverConfig?.environment.capabilities.threadForceStop === true; + const [forceStoppingKey, setForceStoppingKey] = useState(null); + // The run that ignored Stop, so a confirmation left open cannot hit a newer run. + const forceStop = async (runId: RunId | undefined) => { + const threadKey = selectedThreadKey; + setForceStoppingKey(threadKey); + const result = await forceStopThread({ + environmentId: props.environmentId, + input: + runId === undefined + ? { threadId: props.selectedThread.id } + : { threadId: props.selectedThread.id, runId }, + }); + setForceStoppingKey((current) => (current === threadKey ? null : current)); + if (result._tag === "Failure") { + if (!isAtomCommandInterrupted(result)) Dialog.alert("Could not force stop the agent"); + return; + } + const others = result.value.sharedWithThreadIds.length; + if (result.value.outcome === "releasing") { + Dialog.alert("Turn stopped; the agent is still shutting down"); + } + if (result.value.outcome === "shared") { + Dialog.alert( + "Turn stopped; the agent process kept running", + `It is shared with ${others === 0 ? "other" : others} working ${others === 1 ? "thread" : "threads"}. Stop those to free it.`, + ); + } + }; const [statusRequest, setStatusRequest] = useState<{ threadKey: string } | null>(null); const statusPending = statusRequest?.threadKey === selectedThreadKey; const [lifecycleNow, setLifecycleNow] = useState(() => new Date().toISOString()); @@ -619,6 +669,32 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) { if (connectionStatus !== null && !props.draftMode) { return connectionStatus; } + // Before the request check: an agent stuck on an approval may also ignore Stop. + if (runStop !== null && contentPresentationKind === "ready") { + const unresponsive = runStop.state === "unresponsive"; + return { + kind: "stop", + unresponsive, + label: unresponsive ? "Not responding to Stop" : "Stopping…", + action: + unresponsive && canForceStop + ? { + label: forceStoppingKey === selectedThreadKey ? "Force stopping..." : "Force stop", + accessibilityLabel: "Force stop the agent", + disabled: forceStoppingKey === selectedThreadKey, + onPress: () => + Dialog.confirm({ + title: "Force stop the agent?", + message: + "It has not answered Stop. Force stop ends the turn and kills the agent's processes, unless other working threads share them.", + confirmText: "Force stop", + destructive: true, + onConfirm: () => void forceStop(runStop.runId), + }), + } + : null, + }; + } if (props.activePendingApproval !== null || props.activePendingUserInput !== null) { return null; } diff --git a/apps/mobile/src/features/threads/floating-working-control.tsx b/apps/mobile/src/features/threads/floating-working-control.tsx index 3709cf2affa86..c1440535e9cad 100644 --- a/apps/mobile/src/features/threads/floating-working-control.tsx +++ b/apps/mobile/src/features/threads/floating-working-control.tsx @@ -83,7 +83,9 @@ export function FloatingWorkingControl(props: { const [queueWidth, setQueueWidth] = useState(0); const [actionWidth, setActionWidth] = useState(0); const action = - props.status?.kind === "background" || props.status?.kind === "lifecycle" + props.status?.kind === "background" || + props.status?.kind === "lifecycle" || + props.status?.kind === "stop" ? props.status.action : null; const [agentsWidth, setAgentsWidth] = useState(0); @@ -471,6 +473,30 @@ function FloatingStatusLabel(props: { ); } + if (props.status.kind === "stop") { + return ( + + {props.status.unresponsive ? ( + + ) : ( + + )} + + {props.status.label} + + + ); + } if (props.status.kind === "goal") { return ( { expect(collapsed[1]).toMatchObject({ type: "run-fold", createdAt: "2026-06-20T00:00:01.500Z", - label: "Worked for 2.0s", + label: "Worked for 2s", }); const expanded = deriveThreadFeedPresentation(feed, latestRun, new Set([runId])); @@ -1100,10 +1100,10 @@ describe("buildThreadFeed", () => { const settled = deriveThreadFeedPresentation(feed(false), null, new Set()); expect(shape(settled)).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:launch-answer", "user:resume", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:resume-answer", ]); const launchFold = settled.find((entry) => entry.type === "run-fold"); @@ -1112,11 +1112,11 @@ describe("buildThreadFeed", () => { shape(deriveThreadFeedPresentation(feed(false), null, new Set([launchFold.runId]))), ).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "activity-group", "assistant:launch-answer", "user:resume", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:resume-answer", ]); @@ -1133,7 +1133,7 @@ describe("buildThreadFeed", () => { ), ).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:launch-answer", "user:resume", "activity-group", diff --git a/apps/server/package.json b/apps/server/package.json index b32f331730963..30ab11d73480e 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -80,6 +80,9 @@ "@opencode-ai/sdk": "^1.3.15", "@t3tools/contracts": "workspace:*", "@t3tools/provider-testing": "workspace:*", + "@t3tools/secret-provider-core": "workspace:*", + "@t3tools/secret-provider-definitions": "workspace:*", + "@t3tools/secret-provider-onepassword": "workspace:*", "@t3tools/shared": "workspace:*", "@t3tools/source-control-testing": "workspace:*", "@t3tools/ssh": "workspace:*", diff --git a/apps/server/src/auth/RpcAuthorization.test.ts b/apps/server/src/auth/RpcAuthorization.test.ts index 24c22280175cc..8044740a7e2ee 100644 --- a/apps/server/src/auth/RpcAuthorization.test.ts +++ b/apps/server/src/auth/RpcAuthorization.test.ts @@ -5,6 +5,7 @@ import { AuthProvidersManageScope, AuthSettingsWriteScope, DEFAULT_SERVER_SETTINGS, + ProviderDriverKind, ProviderInstanceId, SecretProviderId, ThreadId, @@ -164,13 +165,34 @@ describe("settings mutation authorization", () => { ); yield* client[WS_METHODS.serverUpdateSettings]({ patch: {}, providerInstanceMutation }); expect(handled).toBe(1); + yield* client[WS_METHODS.serverUpdateSettings]({ + patch: {}, + providerInstanceMutation: { + operation: "upsert", + instanceId: ProviderInstanceId.make("codex_work"), + instance: { + driver: ProviderDriverKind.make("codex"), + environment: [ + { + name: "OPENAI_API_KEY", + sensitive: true, + value: { + provider: SecretProviderId.make("personal"), + reference: "op://Private/OpenAI/credential", + }, + }, + ], + }, + }, + }); + expect(handled).toBe(2); expect( yield* client[WS_METHODS.serverUpdateSettings]({ patch: { defaultRuntimeMode: "full-access" }, providerInstanceMutation, }).pipe(Effect.flip), ).toMatchObject({ requiredPermission: AuthSettingsWriteScope }); - expect(handled).toBe(1); + expect(handled).toBe(2); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index c852e469933ec..a550120691767 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -44,6 +44,7 @@ export const RPC_REQUIRED_SCOPES = { [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: AuthOrchestrationOperateScope, [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: AuthOrchestrationOperateScope, + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: AuthOrchestrationOperateScope, [ORCHESTRATION_V2_WS_METHODS.getTurnDiff]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.getFullThreadDiff]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.searchThreads]: AuthOrchestrationReadScope, @@ -232,7 +233,8 @@ const SettingsUpdate = Schema.Struct({ patch: ServerSettingsPatch, providerInstanceMutation: Schema.optionalKey(ProviderInstanceMutation), }); -const decodeSettingsUpdate = Schema.decodeUnknownSync(SettingsUpdate); +// Middleware receives the payload already decoded, so it is checked against the decoded shape. +const decodeSettingsUpdate = Schema.decodeUnknownSync(Schema.toType(SettingsUpdate)); const requiredScopesForSettingsUpdate = (payload: unknown) => { const input = decodeSettingsUpdate(payload); diff --git a/apps/server/src/cli/trace.ts b/apps/server/src/cli/trace.ts index 244636c514061..3e94d0d9957f7 100644 --- a/apps/server/src/cli/trace.ts +++ b/apps/server/src/cli/trace.ts @@ -100,7 +100,7 @@ export function makeTraceSpanSummary(sinceMs = -Infinity) { } const formatMs = (ms: number) => - ms < 1_000 ? `${Math.round(ms)}ms` : `${(ms / 1_000).toFixed(1)}s`; + ms < 1_000 ? `${Math.round(ms)}ms` : `${Math.round(ms / 100) / 10}s`; function formatTraceSummary( summary: ReturnType["finish"]>, @@ -127,7 +127,7 @@ function formatTraceSummary( const widths = header.map((_, column) => Math.max(...table.map((row) => row[column]!.length))); const formatIso = (ms: number) => DateTime.formatIso(DateTime.makeUnsafe(ms)); return [ - `${summary.spanCount} spans ended from ${formatIso(summary.firstEndMs)} to ${formatIso(summary.lastEndMs)} (${summary.minutes.toFixed(1)} min).`, + `${summary.spanCount} spans ended from ${formatIso(summary.firstEndMs)} to ${formatIso(summary.lastEndMs)} (${summary.minutes.toFixed(1).replace(/\.0+$/, "")} min).`, ...(summary.skippedLineCount > 0 ? [`Skipped ${summary.skippedLineCount} lines that are not spans.`] : []), diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index c37b0d05cbbae..b1b05a5a256c7 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -240,6 +240,7 @@ export const make = Effect.gen(function* () { projectSettingsOverrides: true, projectProviderInstanceOverrides: true, threadSnooze: true, + threadForceStop: true, environmentThemes: true, usageLimitSources: true, providerWorkspaceSnapshotsOnDemand: true, diff --git a/apps/server/src/guest/GuestShareService.test.ts b/apps/server/src/guest/GuestShareService.test.ts index 1ef25744c8c55..ac5ffc76aae77 100644 --- a/apps/server/src/guest/GuestShareService.test.ts +++ b/apps/server/src/guest/GuestShareService.test.ts @@ -7,6 +7,7 @@ import { GuestShareId, type GuestStreamEvent, MessageId, + type OrchestrationV2ThreadHistoryPage, type OrchestrationV2TurnItem, ProjectId, ProviderDriverKind, @@ -529,6 +530,90 @@ it.effect("gives the guest a command's detail and output, but no file diffs", () }).pipe(Effect.provide(layerTest)), ); +it.effect("pages the guest back through turns older than the stream's snapshot", () => + Effect.gen(function* () { + const { guestShares, credential } = yield* shareThread; + const { token: session } = yield* guestShares.redeem({ credential, name: "Sam" }); + const runId = yield* ownerRun; + const fileSystem = yield* FileSystem.FileSystem; + const directory = yield* fileSystem.makeTempDirectoryScoped(); + const shot = `${directory}/old.png`; + yield* fileSystem.writeFile(shot, PNG); + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const base = { threadId, runId, nodeId: null, providerThreadId: null, providerTurnId: null }; + const turnItem = (id: string, ordinal: number, payload: object) => ({ + id: EventId.make(`event:${id}`), + type: "turn-item.updated" as const, + threadId, + driver: ProviderDriverKind.make("codex"), + providerInstanceId: instanceId, + occurredAt: now, + payload: { + ...base, + id: TurnItemId.make(id), + nativeItemRef: null, + parentItemId: null, + ordinal, + title: null, + status: "completed" as const, + startedAt: now, + completedAt: now, + updatedAt: now, + messageId: MessageId.make(id), + ...payload, + } as OrchestrationV2TurnItem, + }); + yield* eventSink.write({ + events: [ + turnItem("item:old-image", 1, { + type: "assistant_message", + text: `![Old](${shot})`, + streaming: false, + }), + ...Array.from({ length: 30 }, (_, index) => + turnItem(`item:ask-${index}`, index + 2, { + type: "user_message", + text: `Ask ${index}`, + createdBy: "user", + creationSource: "web", + inputIntent: "turn_start", + attachments: [], + }), + ), + ], + }); + + const [, opening] = Array.from( + yield* Stream.runCollect(Stream.take(yield* guestShares.stream(session), 2)), + ); + if (opening?.type !== "thread" || opening.item.kind !== "snapshot") { + return assert.fail("Expected the thread snapshot second."); + } + const ids = (rows: ReadonlyArray<{ readonly sourceItemId: string }>) => + rows.map((row) => row.sourceItemId); + assert.notInclude(ids(opening.item.projection.visibleTurnItems), "item:old-image"); + let cursor = opening.item.historyCursor ?? null; + assert.isNotNull(cursor); + // The page has not shown the old reply yet, so its image does not load. + assert.equal((yield* Effect.flip(guestShares.media(session, shot))).reason, "unavailable"); + // Only a session pages. + assert.isTrue(Exit.isFailure(yield* Effect.exit(guestShares.history(credential, cursor!)))); + + const older: Array = []; + while (cursor !== null) { + const page: OrchestrationV2ThreadHistoryPage = yield* guestShares.history(session, cursor); + older.push(...ids(page.items)); + cursor = page.hasMoreHistory ? page.nextCursor : null; + } + assert.include(older, "item:old-image"); + assert.deepStrictEqual(yield* guestShares.media(session, shot), { + path: shot, + mimeType: "image/png", + }); + }).pipe(Effect.scoped, Effect.provide(layerTest)), +); + it.effect("ends a running command's output once the owner stops sharing", () => Effect.gen(function* () { const { orchestrator, guestShares, credential } = yield* shareThread; diff --git a/apps/server/src/guest/GuestShareService.ts b/apps/server/src/guest/GuestShareService.ts index 347bdc11ece5b..2a1bf853507ba 100644 --- a/apps/server/src/guest/GuestShareService.ts +++ b/apps/server/src/guest/GuestShareService.ts @@ -15,6 +15,7 @@ import { type OrchestrationV2CommandOutputFrame, type OrchestrationV2DomainEvent, type OrchestrationV2GetTurnItemResult, + type OrchestrationV2ThreadHistoryPage, type OrchestrationV2TurnItem, ThreadId, TurnItemId, @@ -185,6 +186,11 @@ export class GuestShareService extends Context.Service< session: string, itemId: string, ) => Effect.Effect; + /** Older turns of the shared thread, as the owner's client pages them. */ + readonly history: ( + session: string, + cursor: string, + ) => Effect.Effect; /** A command's output frames, as the owner's expanded row gets them, until the link ends. */ readonly commandOutput: ( session: string, @@ -665,6 +671,21 @@ const make = Effect.gen(function* () { }; }); + const history = Effect.fn("GuestShareService.history")(function* ( + session: string, + cursor: string, + ) { + const { thread, share } = yield* authorize(session, "session"); + const page = yield* orchestrator + .getThreadHistoryPage(thread.id, cursor) + .pipe(Effect.mapError(() => unavailable)); + rememberSent( + share.id, + page.items.map((row) => row.item), + ); + return page; + }); + const commandOutput = Effect.fn("GuestShareService.commandOutput")(function* ( session: string, itemId: string, @@ -739,6 +760,7 @@ const make = Effect.gen(function* () { stream, render, item, + history, commandOutput, attachment, media, diff --git a/apps/server/src/guest/http.ts b/apps/server/src/guest/http.ts index bf368ac884d92..b3e856200c845 100644 --- a/apps/server/src/guest/http.ts +++ b/apps/server/src/guest/http.ts @@ -1,6 +1,7 @@ import { GUEST_ATTACHMENT_ROUTE, GUEST_COMMAND_OUTPUT_ROUTE, + GUEST_HISTORY_ROUTE, GUEST_ITEM_ROUTE, GUEST_MEDIA_ROUTE, GUEST_RENDER_ROUTE, @@ -16,6 +17,7 @@ import { isProviderSendTurnSupportedImageMimeType, OrchestrationV2CommandOutputFrame, OrchestrationV2GetTurnItemResult, + OrchestrationV2ThreadHistoryPage, } from "@t3tools/contracts"; import * as ByteSize from "effect/ByteSize"; import * as Duration from "effect/Duration"; @@ -52,6 +54,7 @@ const encodeOutputFrame = Schema.encodeEffect( Schema.fromJsonString(OrchestrationV2CommandOutputFrame), ); const encodeItem = Schema.encodeEffect(Schema.toCodecJson(OrchestrationV2GetTurnItemResult)); +const encodeHistoryPage = Schema.encodeEffect(Schema.toCodecJson(OrchestrationV2ThreadHistoryPage)); const errorResponse = (error: GuestShareAccessError) => HttpServerResponse.json( @@ -292,6 +295,26 @@ const itemRoute = HttpRouter.add( }), ); +const historyRoute = HttpRouter.add( + "GET", + GUEST_HISTORY_ROUTE, + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const cursor = new URLSearchParams(request.url.split("?", 2)[1] ?? "").get("cursor"); + if (!cursor) return yield* badRequest; + const guestShares = yield* GuestShareService; + return yield* guestShares.history(yield* sessionToken, cursor).pipe( + Effect.flatMap((page) => + encodeHistoryPage(page).pipe( + Effect.mapError(() => new GuestShareAccessError({ reason: "unavailable" })), + ), + ), + Effect.map((body) => HttpServerResponse.jsonUnsafe(body, { headers: NO_STORE })), + Effect.catchTags({ GuestShareAccessError: errorResponse }), + ); + }), +); + // Ends when the command settles; the page closes its EventSource on that last frame, so the // browser does not reconnect. const commandOutputRoute = HttpRouter.add( @@ -365,6 +388,7 @@ export const layer = Layer.mergeAll( streamRoute, renderRoute, itemRoute, + historyRoute, commandOutputRoute, attachmentRoute, mediaRoute, diff --git a/apps/server/src/htmlRender/HtmlRender.test.ts b/apps/server/src/htmlRender/HtmlRender.test.ts index 80d367b2f8ac9..1916327672516 100644 --- a/apps/server/src/htmlRender/HtmlRender.test.ts +++ b/apps/server/src/htmlRender/HtmlRender.test.ts @@ -316,6 +316,58 @@ describe("HtmlRender", () => { }, ); + // The fake browser is a shebang script, which Windows cannot execute. + describe.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "when the browser cannot be signalled", + () => { + // A browser T3 cannot signal, as under AppArmor in an LXC container, must still exit. + it.live("asks the browser to close when the preview finishes instead of signalling it", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-fake-browser-", + }); + const executable = path.join(directory, "chrome-headless-shell"); + const exitRecord = path.join(directory, "exited"); + yield* fileSystem.writeFileString( + executable, + `#!${process.execPath} +const fs = require("node:fs"); +process.on("SIGTERM", () => {}); +const quit = () => { fs.writeFileSync(${JSON.stringify(exitRecord)}, "asked to close"); process.exit(0); }; +const reply = (message) => fs.writeSync(4, JSON.stringify(message) + "\\0"); +const results = { + "Target.createTarget": { targetId: "page" }, + "Target.attachToTarget": { sessionId: "session" }, + "Runtime.evaluate": { result: { value: 100 } }, + "Page.captureScreenshot": { data: "iVBORw0KGgo=" }, +}; +let buffered = ""; +fs.createReadStream(null, { fd: 3 }).on("end", quit).on("data", (chunk) => { + const messages = (buffered + chunk).split("\\0"); + buffered = messages.pop(); + for (const { id, method, sessionId } of messages.map((raw) => JSON.parse(raw))) { + if (method === "Browser.close") quit(); + reply({ id, sessionId, result: results[method] ?? {} }); + if (method === "Page.navigate") reply({ sessionId, method: "Page.loadEventFired", params: {} }); + } +}); +`, + { mode: 0o755 }, + ); + + yield* HtmlRender.HtmlRender.use((htmlRender) => + htmlRender.preview({ html: "

x

" }), + ).pipe(Effect.provide(layerHtmlRender(executable))); + + const exitedBy = yield* fileSystem.readFileString(exitRecord).pipe(Effect.option); + expect(Option.getOrElse(exitedBy, () => "killed by a signal")).toBe("asked to close"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + }, + ); + it.live("matches preview media queries and text colors to the requested appearance", (ctx) => Effect.gen(function* () { const executable = (yield* HostProcess.Environment)[TEST_BROWSER_ENV]; diff --git a/apps/server/src/htmlRender/headlessChrome.ts b/apps/server/src/htmlRender/headlessChrome.ts index aaf64bdcecc36..13ce0c6845740 100644 --- a/apps/server/src/htmlRender/headlessChrome.ts +++ b/apps/server/src/htmlRender/headlessChrome.ts @@ -247,7 +247,7 @@ const launchBrowser = Effect.fnUntraced(function* (input: { let disconnected: BrowserFailure | undefined; let nextId = 0; - const post = (method: string, params: Record, sessionId: string) => + const post = (method: string, params: Record, sessionId?: string) => Queue.offer( outgoing, new TextEncoder().encode( @@ -374,6 +374,18 @@ const launchBrowser = Effect.fnUntraced(function* (input: { Effect.forkScoped, ); + // Signals can miss a confined browser (AppArmor in LXC), so ask it to quit first. + // Runs before the spawn's release, which still signals a browser that stays up. + yield* Effect.addFinalizer(() => + disconnected + ? Effect.void + : post("Browser.close", {}).pipe( + Effect.andThen(child.exitCode), + Effect.timeout("2 seconds"), + Effect.ignore, + ), + ); + const send = Effect.fn("headlessChrome.send")(function* ( method: string, params: Record, diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts index a978dbc4d1ff9..93aee4d52f688 100644 --- a/apps/server/src/mcp/toolkits/core.test.ts +++ b/apps/server/src/mcp/toolkits/core.test.ts @@ -350,6 +350,48 @@ it.effect("records only failed calls of tools that hand off no work", () => { ); }); +it.effect("rejects an unknown organize target instead of archiving the calling thread", () => { + const dispatched: Array = []; + return Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const outcome = yield* server + .callTool({ + name: "t3_thread_organize", + arguments: { action: "archive", threadIds: ["other-thread"] }, + }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, scope), + Effect.provideService(McpSchema.McpServerClient, client), + Effect.result, + ); + expect(dispatched).toEqual([]); + expect(outcome).toMatchObject({ + _tag: "Failure", + failure: { message: expect.stringContaining("Expected no excess property") }, + }); + }).pipe( + Effect.provide( + layerThreadToolkit.pipe( + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provide(NodeCrypto.layer), + Layer.provide( + Layer.mock(ThreadManagement.ThreadManagementService)({ + getThreadShell: (id) => Effect.succeed(McpToolAccessTestkit.liveThreadShell(id)), + getProjectThreadRecords: ({ threadId: id }) => + Effect.succeed( + McpToolAccessTestkit.idleThreadProjection(McpToolAccessTestkit.liveThreadShell(id)), + ), + dispatch: (command) => { + dispatched.push(command); + return Effect.succeed({ sequence: 1 } as never); + }, + }), + ), + ), + ), + ); +}); + it.effect("keeps unexpected handler defects private through the production registration", () => Effect.gen(function* () { const server = yield* McpServer.McpServer; diff --git a/apps/server/src/mcp/toolkits/thread/tools.ts b/apps/server/src/mcp/toolkits/thread/tools.ts index 4b566c44a69d1..018acac5c715f 100644 --- a/apps/server/src/mcp/toolkits/thread/tools.ts +++ b/apps/server/src/mcp/toolkits/thread/tools.ts @@ -61,6 +61,7 @@ const ThreadOrganizeTool = Tool.make("t3_thread_organize", { ], }) .annotate(Tool.Title, "Organize a thread") + .annotate(Tool.Strict, true) .annotate(Tool.Destructive, true); const commandTool = { diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index 9794f94634504..a93b969ed203d 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -14,6 +14,7 @@ const RPC_AGGREGATES = { [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: "orchestrationV2", [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: "orchestration", + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.getTurnItem]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.getTurnDiff]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.getFullThreadDiff]: "orchestration", diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 8a68e92b89160..b924f6efb3e70 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -7498,6 +7498,96 @@ describe("AcpAdapterV2", () => { }).pipe(Effect.provide(layerTest), Effect.scoped), ); + it.live("lets a caller abandon Stop while hard teardown stalls, and finishes the teardown", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const path = yield* Path.Path; + const selfInvocation = yield* resolveSelfInvocation(); + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const teardownStarted = yield* Deferred.make(); + const releaseTeardown = yield* Deferred.make(); + let runtimeOrdinalSeen = 0; + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = yield* makeAcpAdapterV2({ + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + interruptPromptOnCancel: true, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: () => + Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseTeardown)), + ), + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = runtimeOrdinal; + return runtimeOrdinal === 1 ? { T3_ACP_HANG_PROMPT_FOREVER: "1" } : {}; + }, + protocolEvents, + }), + }, + selfInvocation, + }); + const threadId = ThreadId.make("thread-acp-abandoned-hard-teardown"); + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-abandoned-hard-teardown"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: acpScopedNativeId(instanceId, "mock-session-1:turn:1"), + }); + yield* pollProtocolMethods(protocolEvents); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true }) + .pipe(Effect.forkScoped); + yield* Deferred.await(teardownStarted); + + // The session manager's Stop timeout interrupts this call; it must not wait for teardown. + yield* Fiber.interrupt(interruptFiber); + + const startFiber = yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), + ) + .pipe(Effect.forkScoped); + yield* Deferred.succeed(releaseTeardown, undefined); + yield* Fiber.join(startFiber); + assert.equal(runtimeOrdinalSeen, 2); + const replacementMethods = yield* pollProtocolMethods(protocolEvents); + assert.equal(replacementMethods.filter((method) => method === "session/resume").length, 1); + }).pipe(Effect.provide(layerTest), Effect.scoped), + ); + it.effect( "direct Stop on a deferred subagent hold terminalizes the subagent and does not carry it forward", () => diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index fb17c34713fd8..40f33b3a1a309 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -40,7 +40,7 @@ import { OrchestrationV2ProviderThread, ProjectId, ProviderInstanceId, - type ProviderInstanceEnvironment, + type ResolvedProviderInstanceEnvironment, ProviderDriverKind, type ProviderReplayTranscript, type ProviderApprovalDecision, @@ -1189,7 +1189,7 @@ describe("ClaudeAdapterV2 approval cancellation", () => { const captureSdkExecutablePaths = Effect.fn("captureSdkExecutablePaths")(function* ( binaryPath: string, - environment: ProviderInstanceEnvironment = [], + environment: ResolvedProviderInstanceEnvironment = [], ) { const executablePaths: Array = []; const taskToolEnvironment: Array = []; @@ -9159,6 +9159,172 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + // regression: #17531 a mod that holds a tool behind a focused pane must fail the turn, not hang it + it.effect.each([ + { case: "fails a turn whose mod requests focus for a pane T3 cannot show", focus: true }, + { case: "keeps a turn running beside a passive mod pane", focus: false }, + ] as const)("$case", ({ focus }) => + Effect.scoped( + Effect.gen(function* () { + const closed = yield* Deferred.make(); + const harness = yield* makeWakeHarnessWithOptions({ + close: () => Deferred.succeed(closed, undefined), + }); + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-mod-pane-focus"), + text: "Run rm -r build", + attachments: [], + }), + ); + // Captured from Claude 2.1.303 with the blast-radius mod holding a Bash + // call: the mod waits for keyboard input in its pane and nothing else + // arrives until it gives up. + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "system", + subtype: "ui_panes", + panes: [ + { id: "blast-radius", title: "Blast Radius", plugin: "blast-radius", rows: 10 }, + ], + shown_id: "blast-radius", + focused_id: null, + focus_requested_id: focus ? "blast-radius" : null, + uuid: "00000000-0000-4000-8000-000000000795", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "system", + subtype: "ui_invalidate", + event: "ui.render", + uuid: "00000000-0000-4000-8000-000000000796", + session_id: WAKE_NATIVE_SESSION, + }), + ); + // Frames are handled in order, so a later notice proves the pane frame + // was handled without the turn ending. + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "system", + subtype: "informational", + content: "Processing barrier.", + level: "warning", + uuid: "00000000-0000-4000-8000-000000000797", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* harness.until( + () => + harness.terminalEvents().length > 0 || + harness.events.some( + (event) => + event.type === "turn_item.updated" && event.turnItem.type === "system_notice", + ), + ); + const [terminal] = harness.terminalEvents(); + if (!focus) { + assert.equal(terminal, undefined); + return; + } + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.include(terminal.failure.message, "blast-radius"); + // The held tool call is abandoned by closing the query, never answered. + yield* Deferred.await(closed); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + // regression: #17531 a follow-up sent while the failed mod turn's process closes gets a new process + it.effect("runs a follow-up in a fresh query while the mod-blocked query is closing", () => + Effect.scoped( + Effect.gen(function* () { + const releaseClose = yield* Deferred.make(); + const replacementClosedOld = yield* Deferred.make(); + let oldCloses = 0; + const harness = yield* makeWakeHarnessWithOptions({ + // The held CLI never answers an interrupt; closure must not need one. + interrupt: Effect.never, + close: (sdkMessages) => + Effect.suspend(() => + ++oldCloses === 2 ? Deferred.succeed(replacementClosedOld, undefined) : Effect.void, + ).pipe( + Effect.andThen(Deferred.await(releaseClose)), + Effect.andThen(Queue.shutdown(sdkMessages)), + ), + freshQueueOnReopen: true, + }); + // Added after the harness so it runs first: a failed assertion must not + // leave teardown waiting on the held close. + yield* Effect.addFinalizer(() => Deferred.succeed(releaseClose, undefined)); + const now = yield* DateTime.now; + const turnInput = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-mod-pane-blocked"), + text: "Run rm -r build", + attachments: [], + }); + yield* harness.runtime.startTurn(turnInput); + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "system", + subtype: "ui_panes", + panes: [ + { id: "blast-radius", title: "Blast Radius", plugin: "blast-radius", rows: 10 }, + ], + shown_id: "blast-radius", + focused_id: null, + focus_requested_id: "blast-radius", + uuid: "00000000-0000-4000-8000-000000000798", + session_id: WAKE_NATIVE_SESSION, + }), + ); + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "failed"); + + const followUp = yield* harness.runtime + .startTurn({ + ...turnInput, + attemptId: RunAttemptId.make("attempt-claude-after-mod-pane"), + providerTurnOrdinal: 2, + }) + .pipe(Effect.forkScoped); + // Reusing the closing process would offer the prompt to it instead. + yield* Effect.raceFirst( + Deferred.await(replacementClosedOld), + harness.until(() => harness.offeredMessages.length >= 2), + ); + assert.isTrue(yield* Deferred.isDone(replacementClosedOld)); + yield* Deferred.succeed(releaseClose, undefined); + yield* Fiber.join(followUp); + assert.equal(harness.processQueues.length, 2); + yield* harness.offerAndWait( + makeResultFrame({ uuid: "after-mod-pane-result", result: "Done." }), + ); + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "completed"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + // regression: #16897 a prompt whose uuid the session already holds must not hang or end early it.effect("keeps a started prompt running until its result", () => Effect.scoped( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 496a0b2e77d0f..428e44e19020c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -322,6 +322,8 @@ const WORKFLOW_MEMBER_TRANSCRIPT_FLUSH_MS = 500; // A steer waits at most this long, from its request, for its foreground // subagents to move to the background before it is sent. const BACKGROUND_MOVE_TIMEOUT_MS = 5_000; +// How long a Stop waits for the CLI to exit before the user is offered Force stop. +const CLAUDE_STOP_CONFIRM_TIMEOUT_MS = 10_000; const CLAUDE_GROUP_WATCH_SCRIPT = [ "{ trap '' TERM; read -r _ <&3; kill -TERM 0; sleep 5; kill -KILL 0; } /dev/null 2>&1 &", @@ -4213,6 +4215,20 @@ function claudeTaskString(value: unknown, key: string): string | undefined { return typeof field === "string" && field.trim().length > 0 ? field.trim() : undefined; } +// A Claude Code mod can hold a tool call until the user acts in its terminal +// pane. T3 cannot provide the requested pane interaction. +// Passive panes never request focus and keep working. +function claudeFocusRequestedPanePlugin(message: SDKMessage): string | null { + if (message.type !== "system" || claudeTaskField(message, "subtype") !== "ui_panes") return null; + const paneId = claudeTaskString(message, "focus_requested_id"); + if (paneId === undefined) return null; + const panes = claudeTaskField(message, "panes"); + const pane = Array.isArray(panes) + ? panes.find((entry) => claudeTaskField(entry, "id") === paneId) + : undefined; + return claudeTaskString(pane, "plugin") ?? paneId; +} + function claudeTaskStatus(value: unknown): OrchestrationV2PlanStep["status"] | undefined { switch (value) { case "pending": @@ -8182,6 +8198,28 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } return; } + const focusedPlugin = claudeFocusRequestedPanePlugin(message); + if (focusedPlugin !== null && context === (yield* Ref.get(activeTurn))) { + yield* Effect.logWarning("orchestration-v2.claude-mod-pane-focus-unsupported", { + providerTurnId: context.providerTurnId, + plugin: focusedPlugin, + }); + liveQuery.stopping = true; + yield* finalizeActiveTurn({ + context, + status: "failed", + completedAt: yield* DateTime.now, + failure: makeProviderFailure({ + class: "provider_error", + code: "claude_mod_ui_unsupported", + message: `The Claude Code mod "${focusedPlugin}" is waiting for input in its terminal pane, which T3 Code cannot show. Disable the mod for sessions run from T3 Code, or run the task in the Claude Code terminal.`, + }), + }); + // Closing ends the CLI process and abandons the held tool call + // without answering it. Not interrupt: the held CLI may never ack it. + yield* liveQuery.query.close.pipe(Effect.ignore); + return; + } yield* trackClaudeGoal({ nativeThreadId: liveQuery.nativeThreadId, context, message }); if (message.type === "stream_event" && !message.parent_tool_use_id) { @@ -9817,6 +9855,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if ( existing !== null && existing.nativeThreadId === nativeThreadId && + !existing.stopping && isClaudeProviderContinuationTurn(turnInput) ) { return existing; @@ -9848,6 +9887,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( existing !== null && existing.nativeThreadId === nativeThreadId && existing.queryPolicyKey === queryPolicyKey && + !existing.stopping && existing.selection.queryIdentity === compiledSelection.queryIdentity ) { // A denied ExitPlanMode leaves Claude in plan mode. Restore only from plan: @@ -10433,44 +10473,36 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( yield* existing.query.interrupt; yield* existing.query.close.pipe(Effect.ignore); const closed = yield* Deferred.await(existing.closed).pipe( - Effect.timeoutOption("10 seconds"), + Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS), ); if (Option.isSome(closed)) { return; } - - const completedAt = yield* DateTime.now; + // The turn stays open: the CLI may still be working, and the user decides + // whether to Force stop it. yield* Effect.logWarning("orchestration-v2.claude-query-interrupt-timeout", { providerSessionId: input.providerSessionId, providerThreadId: turnInput.providerThread.id, providerTurnId: turnInput.providerTurnId, }); - const ownedLiveQuery = yield* Ref.modify(queryContext, (current) => - current?.query === existing.query ? [true, null] : [false, current], - ); - if (ownedLiveQuery) { - existing.stopping = true; - yield* finalizeActiveTurnAfterQueryExit(existing); - yield* endSubagentCallsOfClosedQuery(existing.nativeThreadId, "interrupted"); - } else { - yield* finalizeActiveTurn({ - context: currentTurn, - status: "interrupted", - completedAt, - }); - } - yield* Deferred.succeed(existing.closed, undefined); + return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + timeoutMs: CLAUDE_STOP_CONFIRM_TIMEOUT_MS, + }); }, (effect, turnInput) => effect.pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterInterruptError({ - driver: CLAUDE_PROVIDER, - providerThreadId: turnInput.providerThread.id, - providerTurnId: turnInput.providerTurnId, - cause, - }), + Effect.mapError((cause) => + cause._tag === "ProviderAdapterInterruptTimeoutError" + ? cause + : new ProviderAdapter.ProviderAdapterInterruptError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + cause, + }), ), ), ); diff --git a/apps/server/src/orchestration-v2/Adapters/PiAdapterV2.testkit.ts b/apps/server/src/orchestration-v2/Adapters/PiAdapterV2.testkit.ts index c894442409921..49eb6c9b3b999 100644 --- a/apps/server/src/orchestration-v2/Adapters/PiAdapterV2.testkit.ts +++ b/apps/server/src/orchestration-v2/Adapters/PiAdapterV2.testkit.ts @@ -2,7 +2,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { ProviderReplayEntry, ProviderInstanceId, - type ProviderInstanceEnvironment, + type ResolvedProviderInstanceEnvironment, type ProviderReplayTranscript, type ProviderReplayEntry as ProviderReplayEntryType, } from "@t3tools/contracts"; @@ -363,7 +363,7 @@ export function layer(input: { readonly spawner: Layer.Layer; readonly binaryPath: string; readonly launchArgs: string; - readonly environment?: ProviderInstanceEnvironment; + readonly environment?: ResolvedProviderInstanceEnvironment; }) { return ProviderAdapterRegistry.layerFromAdaptersEffect( PiAdapterV2Driver.create({ diff --git a/apps/server/src/orchestration-v2/EffectWorker.test.ts b/apps/server/src/orchestration-v2/EffectWorker.test.ts index cb130930372c8..2329f2aa20cf0 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.test.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.test.ts @@ -111,6 +111,8 @@ function layerExecutorFor(input: { closeInstance: () => Effect.void, release: () => record("release"), detach: () => record("detach"), + otherWorkingThreads: () => Effect.succeed([]), + releaseUnlessBusy: () => Effect.succeed(true), }), ), Layer.succeed( @@ -167,7 +169,18 @@ function layerExecutorFor(input: { ); } -it.effect("settles a stopped run when its adapter has already lost the native turn", () => +it.effect.each([ + { + name: "settles a stopped run when its adapter has already lost the native turn", + unresponsive: false, + dispatched: ["thread.background-work.settle"], + }, + { + name: "leaves the run running and flags the Stop when the provider never confirms it", + unresponsive: true, + dispatched: ["thread.interrupt.unresponsive"], + }, +])("$name", ({ unresponsive, dispatched }) => Effect.gen(function* () { const now = yield* DateTime.now; const events = yield* Ref.make>([]); @@ -178,7 +191,8 @@ it.effect("settles a stopped run when its adapter has already lost the native tu threadId, operation: "interrupt", providerTurnId, - cause: "Provider turn is not active.", + ...(unresponsive ? { unresponsive } : {}), + cause: unresponsive ? "Timed out." : "Provider turn is not active.", }), threads: { dispatch: (command) => @@ -199,7 +213,7 @@ it.effect("settles a stopped run when its adapter has already lost the native tu }, }); }).pipe(Effect.provide(layer)); - assert.deepEqual(yield* Ref.get(events), ["thread.background-work.settle"]); + assert.deepEqual(yield* Ref.get(events), dispatched); }), ); diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index d174db2fb3a29..534043aeb4eb8 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -177,6 +177,13 @@ export const layerExecutor: Layer.Layer< cause, }), ), + Effect.map((outcome) => ({ unresponsive: false, outcome }) as const), + // Settling here would end the run while the agent may still be running; the + // user decides with Force stop instead. + Effect.catchIf( + (cause) => request.subagent === undefined && cause.unresponsive === true, + () => Effect.succeed({ unresponsive: true, outcome: undefined } as const), + ), Effect.catchIf( (cause) => request.subagent === undefined && @@ -184,27 +191,35 @@ export const layerExecutor: Layer.Layer< request.type, Cause.pretty(Cause.fail(cause)), ), - () => Effect.void, + () => Effect.succeed({ unresponsive: false, outcome: undefined } as const), ), - Effect.andThen((outcome) => - request.subagent !== undefined && outcome === undefined - ? Effect.void - : threads.dispatch({ - type: "thread.background-work.settle", - commandId: CommandId.make(`${effect.id}:background-work-settled`), - ...(request.subagent === undefined || outcome === undefined - ? {} - : { - subagent: { - id: request.subagent.id, - stoppedAt: DateTime.makeUnsafe(effect.createdAt), - outcome, - }, - }), + Effect.andThen(({ unresponsive, outcome }) => + unresponsive + ? threads.dispatch({ + type: "thread.interrupt.unresponsive", + commandId: CommandId.make(`${effect.id}:unresponsive`), threadId: effect.threadId, providerThreadId: request.providerThreadId, providerTurnId: request.providerTurnId, - }), + }) + : request.subagent !== undefined && outcome === undefined + ? Effect.void + : threads.dispatch({ + type: "thread.background-work.settle", + commandId: CommandId.make(`${effect.id}:background-work-settled`), + ...(request.subagent === undefined || outcome === undefined + ? {} + : { + subagent: { + id: request.subagent.id, + stoppedAt: DateTime.makeUnsafe(effect.createdAt), + outcome, + }, + }), + threadId: effect.threadId, + providerThreadId: request.providerThreadId, + providerTurnId: request.providerTurnId, + }), ), Effect.mapError( (cause) => diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index c65005deefb48..50f9a2acc0f39 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -536,6 +536,7 @@ function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId { case "checkpoint.rollback": case "checkpoint.rollback.fail": case "thread.background-work.settle": + case "thread.interrupt.unresponsive": case "thread.stop": case "provider.switch": return command.threadId; @@ -10278,7 +10279,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio run, rootNode, providerThread, - reason: yield* runInterruptReason(input.command, input.events, run.id), + reason: + (input.command.type === "thread.background-work.settle" + ? input.command.reason + : undefined) ?? (yield* runInterruptReason(input.command, input.events, run.id)), completedAt: input.now, }), }); @@ -10525,6 +10529,55 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio yield* closePendingLinkRequests({ ...input, threadId: thread.id }); }); + const dispatchInterruptUnresponsive = Effect.fn("Orchestrator.dispatchInterruptUnresponsive")( + function* ( + command: Extract< + OrchestrationV2InternalCommand, + { readonly type: "thread.interrupt.unresponsive" } + >, + events: Ref.Ref>, + ) { + const { providerTurn } = yield* projectionStore + .getProviderControlContext(command.threadId, { + providerThreadId: command.providerThreadId, + providerTurnId: command.providerTurnId, + }) + .pipe(mapDispatchError(command)); + const { runs } = yield* projectionStore + .getThreadRecords(command.threadId, ["runs"]) + .pipe(mapDispatchError(command)); + const run = runs.find( + (candidate) => + providerTurn !== undefined && candidate.activeAttemptId === providerTurn.runAttemptId, + ); + // A settled run still counts: Stop may have targeted its background work. + if (run === undefined) return; + const { turnItems } = yield* projectionStore + .getThreadRecords(command.threadId, ["turnItems"], { + turnItemTypes: ["run_interrupt_request", "run_interrupt_result"], + turnItemRunIds: [run.id], + }) + .pipe(mapDispatchError(command)); + // A provider that stopped late already wrote the result; it needs no Force stop. + if (turnItems.some((item) => item.type === "run_interrupt_result")) return; + const request = turnItems.find((item) => item.type === "run_interrupt_request"); + if (request?.type !== "run_interrupt_request" || request.unresponsiveAt !== undefined) return; + const now = yield* DateTime.now; + yield* emit( + events, + command, + )({ + type: "turn-item.updated", + threadId: command.threadId, + runId: run.id, + ...(request.nodeId === null ? {} : { nodeId: request.nodeId }), + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...request, unresponsiveAt: now, updatedAt: now }, + }); + }, + ); + const dispatchBackgroundWorkSettle = Effect.fn("Orchestrator.dispatchBackgroundWorkSettle")( function* ( command: Extract< @@ -11062,6 +11115,45 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio occurredAt: now, payload: { ...run, status: "interrupted", completedAt: now }, }); + // Stop cancels a pending restart, so it must end the turn that restart could not stop. + const supersededTurn = projection.providerTurns.findLast( + (candidate) => + candidate.status === "running" && + projection.attempts.some( + (attempt) => attempt.id === candidate.runAttemptId && attempt.runId === run.id, + ), + ); + const supersededOwner = + supersededTurn === undefined + ? undefined + : projection.providerThreads.find( + (candidate) => candidate.id === supersededTurn.providerThreadId, + ); + if ( + supersededTurn !== undefined && + supersededOwner !== undefined && + supersededOwner.providerSessionId !== null && + Option.isSome( + yield* providerSessions + .get(supersededOwner.providerSessionId) + .pipe(Effect.orElseSucceed(() => Option.none())), + ) + ) { + const id = `effect:${command.commandId}:provider-turn.interrupt:${supersededTurn.id}`; + if (!otherProviderInterrupts.some((effect) => effect.id === id)) { + otherProviderInterrupts.push({ + id, + commandId: command.commandId, + threadId: command.threadId, + request: { + type: "provider-turn.interrupt", + providerSessionId: supersededOwner.providerSessionId, + providerThreadId: supersededOwner.id, + providerTurnId: supersededTurn.id, + }, + }); + } + } yield* stopRemainingWork(); yield* Ref.update(effects, (existing) => [...existing, ...otherProviderInterrupts]); return { @@ -12863,6 +12955,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio case "thread.background-work.settle": yield* dispatchBackgroundWorkSettle(command, events, effects); break; + case "thread.interrupt.unresponsive": + yield* dispatchInterruptUnresponsive(command, events); + break; case "thread.fork": yield* dispatchThreadFork(command, events); break; @@ -12993,6 +13088,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (planned) => planned.events.length > 0 || command.type === "thread.background-work.settle" || + command.type === "thread.interrupt.unresponsive" || command.type === "thread.stop" || command.type === "subagent.stop" || // Closing a share again, or after it ended, is not an error. diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index cac8e4fbbb8e0..5bbb3926e2b66 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -338,6 +338,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( readonly backgroundTasksEndWithProcess?: boolean; readonly hangSessionScopeClose?: boolean; readonly startTurn?: Effect.Effect; + readonly interruptTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; readonly spawnBeforeOpen?: boolean; readonly scopeCloseReached?: Deferred.Deferred; @@ -437,7 +438,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( Ref.update(state, (current) => ({ ...current, interruptCount: current.interruptCount + 1, - })), + })).pipe(Effect.andThen(options.interruptTurn ?? Effect.void)), unloadThread: ({ providerThread }) => (options.beforeUnload ?? Effect.void).pipe( Effect.andThen( @@ -485,6 +486,7 @@ function layerTest(input: { readonly backgroundTasksEndWithProcess?: boolean; readonly hangSessionScopeClose?: boolean; readonly startTurn?: Effect.Effect; + readonly interruptTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; readonly spawnBeforeOpen?: boolean; readonly scopeCloseReached?: Deferred.Deferred; @@ -520,6 +522,7 @@ function layerTest(input: { ? {} : { hangSessionScopeClose: input.hangSessionScopeClose }), ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), + ...(input.interruptTurn === undefined ? {} : { interruptTurn: input.interruptTurn }), ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }), ...(input.scopeCloseReached === undefined @@ -1367,6 +1370,159 @@ it.effect( }), ); +it.effect("ProviderSessionManagerV2 gives up on a provider that never confirms a Stop", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-provider-session-manager-unresponsive-stop"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const stopping = yield* runtime + .interruptTurn({ + providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }), + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-unresponsive", + }), + }) + .pipe(Effect.flip, Effect.forkChild); + yield* TestClock.adjust(ProviderSessionManager.PROVIDER_INTERRUPT_TIMEOUT_MS); + assert.equal((yield* Fiber.join(stopping))._tag, "ProviderAdapterInterruptTimeoutError"); + }).pipe( + Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, interruptTurn: Effect.never })), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps a shared session a sibling is busy on when releasing", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const stopping = ThreadId.make("thread-provider-session-manager-release-stopping"); + const sibling = ThreadId.make("thread-provider-session-manager-release-sibling"); + const providerSessionId = idAllocator.derive.providerSession({ + providerInstanceId: modelSelection.instanceId, + }); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId: stopping, now }), + yield* makeThreadCreatedEvent({ idAllocator, threadId: sibling, now }), + ], + }); + const runtime = yield* manager.open({ + threadId: stopping, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* manager.open({ + threadId: sibling, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThreadOf = (threadId: ThreadId) => + makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + nativeThreadId: `native-${threadId}`, + }); + const runId = idAllocator.derive.run({ threadId: sibling, ordinal: 1 }); + yield* runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(sibling)).thread, + threadId: sibling, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread: providerThreadOf(sibling), + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId: sibling, ordinal: 1 }), + text: "turn", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + const release = (own: ThreadId) => + manager.releaseUnlessBusy({ + providerSessionId, + ownProviderThreadIds: [providerThreadOf(own).id], + reason: "manual_shutdown", + }); + + assert.isFalse(yield* release(stopping)); + assert.isTrue(yield* manager.isLive(providerSessionId)); + assert.isTrue(yield* release(sibling)); + assert.isFalse(yield* manager.isLive(providerSessionId)); + + // A turn that starts on a runtime released meanwhile fails instead of running on it. + const startOnReleasedRuntime = Effect.gen(function* () { + const lateRunId = idAllocator.derive.run({ threadId: stopping, ordinal: 1 }); + return yield* Effect.flip( + runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(stopping)).thread, + threadId: stopping, + runId: lateRunId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId: lateRunId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId: lateRunId }), + providerThread: providerThreadOf(stopping), + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId: stopping, ordinal: 1 }), + text: "turn", + attachments: [], + }, + modelSelection, + runtimePolicy, + }), + ); + }); + assert.equal((yield* startOnReleasedRuntime)._tag, "ProviderAdapterTurnStartError"); + + // Nor may it leave its turn busy on a replacement opened under the same id. + yield* manager.open({ + threadId: sibling, + providerSessionId, + modelSelection, + runtimePolicy, + }); + assert.equal((yield* startOnReleasedRuntime)._tag, "ProviderAdapterTurnStartError"); + assert.isTrue(yield* release(sibling)); + }).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60 * 60_000 }))); + }), +); + it.effect("ProviderSessionManagerV2 stops a session still opening when its layer shuts down", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); @@ -2551,6 +2707,10 @@ it.effect("ProviderSessionManagerV2 releases sessions when provider event stream assert.isTrue(Option.isNone(liveSession)); assert.equal(runtimeState.closeCount, 1); assert.equal(projection.providerSessions.at(-1)?.status, "error"); + assert.equal( + projection.providerSessions.at(-1)?.lastError, + "The provider event stream closed unexpectedly. Retry the turn; if it keeps failing, check the provider and server logs.", + ); }); yield* effect.pipe( diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index df2ee630df38b..03a7278774a64 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -50,17 +50,26 @@ import * as CommandOutputHub from "./CommandOutputHub.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000; +/** Past adapters' own 10 s stop fallbacks, so they settle first when the provider answers at all. */ +export const PROVIDER_INTERRUPT_TIMEOUT_MS = 15 * 1000; const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThreadId}#`; const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) => `${busyTurnPrefix(providerThreadId)}${runOrdinal}`; const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000; +const WORKING_RUN_STATUSES: ReadonlySet = new Set([ + "preparing", + "starting", + "running", + "waiting", +]); export const ProviderSessionReleaseReason = Schema.Literals([ "idle_timeout", @@ -180,6 +189,25 @@ export interface ProviderSessionManagerV2Shape { readonly detail?: string; readonly revokeMcpCredential?: boolean; }) => Effect.Effect; + /** + * Releases the session unless, at the moment of removal, a turn of a provider thread outside + * `ownProviderThreadIds` is busy on it. Checked atomically with turn starts. Succeeds with + * whether it was released. + */ + readonly releaseUnlessBusy: (input: { + readonly providerSessionId: ProviderSessionId; + readonly ownProviderThreadIds: ReadonlyArray; + readonly reason: ProviderSessionReleaseReason; + readonly detail?: string; + }) => Effect.Effect; + /** + * Other threads with a run or background work on this session's shared process; empty when + * it serves one thread. + */ + readonly otherWorkingThreads: (input: { + readonly providerSessionId: ProviderSessionId; + readonly threadId: ThreadId; + }) => Effect.Effect>; } export class ProviderSessionManagerV2 extends Context.Service< @@ -894,6 +922,7 @@ export const layerWithOptions = ( function* (input: { readonly providerSessionId: ProviderSessionId; readonly onlyIfIdleGeneration?: number; + readonly keepIfBusyOtherThan?: ReadonlyArray; }): Effect.fn.Return, DateTime.Utc]> { const key = sessionKey(input.providerSessionId); const candidate = (yield* Ref.get(sessions)).get(key); @@ -913,6 +942,15 @@ export const layerWithOptions = ( ) { return ["kept", current] as const; } + const own = input.keepIfBusyOtherThan; + if ( + own !== undefined && + [...existing.busyTurns].some( + (turnKey) => !own.some((id) => turnKey.startsWith(busyTurnPrefix(id))), + ) + ) { + return ["kept", current] as const; + } const updated = new Map(current); updated.delete(key); return ["removed", updated] as const; @@ -971,6 +1009,7 @@ export const layerWithOptions = ( readonly detachedThreadId?: ThreadId; readonly cancelIdleFiber?: boolean; readonly onlyIfIdleGeneration?: number; + readonly keepIfBusyOtherThan?: ReadonlyArray; readonly gracefulSubscribers?: boolean; }) => Effect.acquireUseRelease( @@ -1379,15 +1418,21 @@ export const layerWithOptions = ( .pipe(Effect.ensuring(Effect.sync(dropReservation))); }); - const markBusy = (providerSessionId: ProviderSessionId, turnKey: string) => + // Marks only the entry this runtime belongs to: a released or replaced session must not + // gain a busy turn that never ends. Succeeds with whether the turn was marked. + const markBusy = ( + providerSessionId: ProviderSessionId, + turnKey: string, + runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime, + ) => withActivityError( providerSessionId, Effect.gen(function* () { const key = sessionKey(providerSessionId); - const idleFiber = yield* Ref.modify(sessions, (current) => { + const marked = yield* Ref.modify(sessions, (current) => { const entry = current.get(key); - if (entry === undefined) { - return [null, current] as const; + if (entry === undefined || entry.runtime !== runtime) { + return [undefined, current] as const; } const updated = new Map(current); updated.set(key, { @@ -1396,9 +1441,11 @@ export const layerWithOptions = ( idleFiber: null, pinnedSinceMs: null, }); - return [entry.idleFiber, updated] as const; + return [{ idleFiber: entry.idleFiber }, updated] as const; }); - yield* cancelIdleFiber(idleFiber); + if (marked === undefined) return false; + yield* cancelIdleFiber(marked.idleFiber); + return true; }), ); @@ -1784,11 +1831,31 @@ export const layerWithOptions = ( ), Effect.andThen( Effect.acquireUseRelease( - observeActivity( + markBusy( providerSessionId, - markBusy( - providerSessionId, - busyTurnKey(input.providerThread.id, input.runOrdinal), + busyTurnKey(input.providerThread.id, input.runOrdinal), + runtime, + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.activity-failed", { + providerSessionId, + cause, + }).pipe(Effect.as(true)), + ), + // A release between attach and markBusy leaves nothing to mark: fail the + // start instead of running it on a runtime being torn down. + Effect.flatMap((marked) => + marked + ? Effect.void + : Effect.fail( + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: runtime.driver, + threadId: input.threadId, + providerThreadId: input.providerThread.id, + runId: input.runId, + cause: "The provider session was released while the turn started.", + }), + ), ), ), () => @@ -1807,9 +1874,26 @@ export const layerWithOptions = ( observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( Effect.andThen(runtime.steerTurn(input).pipe(turnMetrics("steer"))), ), + // A provider starved of memory can stop answering, so no caller may wait on it unbounded. interruptTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( - Effect.andThen(runtime.interruptTurn(input).pipe(turnMetrics("interrupt"))), + Effect.andThen( + runtime.interruptTurn(input).pipe( + Effect.timeoutOrElse({ + duration: PROVIDER_INTERRUPT_TIMEOUT_MS, + orElse: () => + Effect.fail( + new ProviderAdapter.ProviderAdapterInterruptTimeoutError({ + driver: runtime.driver, + providerThreadId: input.providerThread.id, + providerTurnId: input.providerTurnId, + timeoutMs: PROVIDER_INTERRUPT_TIMEOUT_MS, + }), + ), + }), + turnMetrics("interrupt"), + ), + ), ), respondToRuntimeRequest: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( @@ -1987,10 +2071,16 @@ export const layerWithOptions = ( cause, }); yield* Ref.set(entry.eventSubscribers, new Map()); + // lastError is the thread banner, so it gets the same bounded + // message as a failed turn; the raw cause survives only in the log. + yield* Effect.logWarning("orchestration-v2.provider-session-runtime-failed", { + providerSessionId: entry.runtime.providerSessionId, + cause, + }); yield* releaseEntry({ providerSessionId: entry.runtime.providerSessionId, reason: "runtime_error", - detail: Cause.pretty(cause), + detail: makeProviderFailure({ cause }).message, }).pipe(logReleaseFailure(entry.runtime.providerSessionId)); }), ), @@ -2269,6 +2359,40 @@ export const layerWithOptions = ( ), ), release: releaseEntry, + releaseUnlessBusy: Effect.fn("ProviderSessionManager.releaseUnlessBusy")(function* (input) { + const key = sessionKey(input.providerSessionId); + const before = (yield* Ref.get(sessions)).get(key); + yield* releaseEntry({ + providerSessionId: input.providerSessionId, + reason: input.reason, + ...(input.detail === undefined ? {} : { detail: input.detail }), + keepIfBusyOtherThan: input.ownProviderThreadIds, + }); + // Identity, not presence: open can install a replacement under the same id meanwhile. + return before !== undefined && (yield* Ref.get(sessions)).get(key) !== before; + }), + otherWorkingThreads: Effect.fn("ProviderSessionManager.otherWorkingThreads")( + function* (input) { + const entry = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId)); + if (entry === undefined || !entry.supportsMultipleProviderThreads) return []; + return yield* Effect.filter( + [...entry.attachedThreadIds].filter((threadId) => threadId !== input.threadId), + (threadId) => + // Runs, not the shell's activeRunId: the shell omits silent runs such as idle + // compaction. Background work outlives its run and still needs the process. + Effect.zipWith( + projectionStore.getThreadRecords(threadId, ["runs"]), + projectionStore.getThreadShell(threadId), + ({ runs }, shell) => + runs.some((run) => WORKING_RUN_STATUSES.has(run.status)) || + (shell?.pendingBackgroundTasks?.length ?? 0) > 0, + ).pipe( + // An unreadable thread may still be working; never kill it unseen. + Effect.orElseSucceed(() => true), + ), + ); + }, + ), detach: Effect.fn("ProviderSessionManager.detach")( function* (input) { const key = sessionKey(input.providerSessionId); diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index bab115f6e24cb..65da8b23676b0 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -284,6 +284,8 @@ it.effect( closeInstance: () => Effect.void, release: () => Effect.void, detach: () => Effect.void, + otherWorkingThreads: () => Effect.succeed([]), + releaseUnlessBusy: () => Effect.succeed(true), }), ); const layerControl = ProviderTurnControlService.layer.pipe( diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts index e1dbac21bc52e..bd225bb41773b 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts @@ -38,6 +38,8 @@ export class ProviderTurnControlError extends Schema.TaggedError + Effect.fail( + new ProviderTurnControlError({ + threadId: input.threadId, + operation: "interrupt", + providerTurnId: input.providerTurnId, + unresponsive: true, + cause, + }), + ), + }), + ); // Use real time; ingestion fibers do not advance TestClock and would leave Stop blocked. yield* Effect.gen(function* () { const deadline = (yield* Clock.currentTimeMillis) + 2_000; diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index 03cdf5e85a642..d059c4c20e9d7 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -438,6 +438,118 @@ it.live("retries a failed selection restart session open before completing", () ), ); +it.live("Stop during a restart interrupts the turn the restart could not stop", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("selection-restart-stop"); + const threadId = ThreadId.make("thread:selection-restart-stop"); + const state = yield* Ref.make({ + activeTurn: null, + opened: [], + started: [], + closedSessionCount: 0, + failedReplacementOpen: false, + }); + const base = makeRestartAdapter(state); + // A wedged provider ignores the restart's soft cancel; only Stop ends its turn. + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + ...base, + openSession: (input) => + base.openSession(input).pipe( + Effect.map((session) => ({ + ...session, + interruptTurn: (interrupt: ProviderAdapter.ProviderAdapterV2InterruptInput) => + interrupt.requestRuntimeRestart === true + ? session.interruptTurn(interrupt) + : Effect.never, + })), + ), + }; + + const result = yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const running = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => event.type === "provider-turn.updated" && event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped({ startImmediately: true }), + ); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:selection-restart-stop:create"), + threadId, + projectId: ProjectId.make("project:selection-restart-stop"), + title: "Stop during restart", + modelSelection: initialSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:selection-restart-stop:first"), + threadId, + messageId: MessageId.make("message:selection-restart-stop:first"), + text: "first", + attachments: [], + modelSelection: initialSelection, + dispatchMode: { type: "start_immediately" }, + }); + yield* worker.drain(); + yield* Fiber.join(running); + const runId = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!.id; + + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:selection-restart-stop:steer"), + threadId, + messageId: MessageId.make("message:selection-restart-stop:steer"), + text: "Are you still going?", + attachments: [], + modelSelection: initialSelection, + dispatchMode: { type: "restart_active", targetRunId: runId }, + }); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("command:selection-restart-stop:stop"), + threadId, + runId, + }); + yield* worker.drain(); + return { + projection: yield* orchestrator.getThreadProjection(threadId), + captured: yield* Ref.get(state), + }; + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "selection-restart-stop" }, + ProviderAdapterRegistry.layerSingle(adapter), + ), + ), + ); + + assert.isNull(result.captured.activeTurn); + assert.lengthOf(result.captured.started, 1); + assert.equal(result.projection.runs[0]?.status, "interrupted"); + assert.deepEqual( + result.projection.providerTurns.map((turn) => turn.status), + ["interrupted"], + ); + }), + ), +); + it.live( "restarts the live session on a model change when a newer stopped session record exists", () => diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts index b2daad62681e9..9285f72d46f56 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts @@ -27,6 +27,8 @@ import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; +import { AgentScope } from "@t3tools/shared/AgentScope"; + import * as ServerSettings from "../serverSettings.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; @@ -596,3 +598,174 @@ it.effect.each([ expect(dispatched).toEqual(settles ? ["thread.settle"] : []); }), ); + +it.effect.each([ + { + name: "kills the thread's own agent and ends its turn", + others: [] as ReadonlyArray, + killed: true, + expected: { outcome: "killed", sharedWithThreadIds: [] }, + teardown: ["release"], + }, + { + name: "keeps a process another working thread shares and only ends this turn", + others: ["thread:thread-management:force-stop:other"], + killed: false, + expected: { + outcome: "shared", + sharedWithThreadIds: ["thread:thread-management:force-stop:other"], + }, + teardown: ["detach"], + }, + { + name: "still settles a finished run's background work in a shared process", + others: ["thread:thread-management:force-stop:other"], + killed: false, + settledRun: true, + expected: { + outcome: "shared", + sharedWithThreadIds: ["thread:thread-management:force-stop:other"], + }, + teardown: ["detach"], + }, + { + name: "detaches when a sibling starts a turn just before the session is released", + others: [] as ReadonlyArray, + killed: false, + siblingBusy: true, + expected: { outcome: "shared", sharedWithThreadIds: [] }, + teardown: ["release", "detach"], + }, + { + name: "reports a teardown that has not finished in time as still releasing", + others: [] as ReadonlyArray, + killed: false, + releaseHangs: true, + expected: { outcome: "releasing", sharedWithThreadIds: [] }, + teardown: ["release"], + }, +])("Force stop $name", ({ others, killed, expected, teardown, ...row }) => + Effect.gen(function* () { + const threadId = ThreadId.make("thread:thread-management:force-stop"); + const providerSessionId = ProviderSessionId.make("provider-session:force-stop"); + const providerThreadId = ProviderThreadId.make("provider-thread:force-stop"); + const runId = RunId.make("run:thread-management:force-stop"); + const dispatched: Array = []; + const teardowns: Array = []; + const tornDown = yield* Deferred.make(); + const layerTest = ThreadManagementService.layer.pipe( + Layer.provide( + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + isLive: () => Effect.succeed(true), + otherWorkingThreads: () => Effect.succeed(others.map((id) => ThreadId.make(id))), + releaseUnlessBusy: () => + Effect.sync(() => teardowns.push("release")).pipe( + Effect.andThen( + "releaseHangs" in row ? Effect.never : Effect.succeed(!("siblingBusy" in row)), + ), + ), + detach: () => + Effect.sync(() => teardowns.push("detach")).pipe( + Effect.andThen(Deferred.succeed(tornDown, undefined)), + Effect.asVoid, + ), + }), + ), + Layer.provide( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadRecords: () => + Effect.succeed({ + thread: { id: threadId, deletedAt: null }, + runs: [ + { + id: runId, + ordinal: 1, + status: "settledRun" in row ? "completed" : "running", + activeAttemptId: "attempt", + }, + // A queued follow-up neither supersedes the request nor becomes the target. + { id: RunId.make("run:queued-follow-up"), ordinal: 2, status: "queued" }, + ], + providerThreads: [{ id: providerThreadId, appThreadId: threadId, providerSessionId }], + providerTurns: [ + { + id: "provider-turn:force-stop", + providerThreadId, + runAttemptId: "attempt", + status: "settledRun" in row ? "completed" : "running", + }, + ], + } as unknown as OrchestrationV2ThreadProjection), + dispatch: (command) => + Effect.sync(() => { + dispatched.push(command); + return { sequence: 1, storedEvents: [] } as never; + }), + }), + ), + Layer.provide( + Layer.succeed(AgentScope, { + ...AgentScope.defaultValue(), + kill: () => Effect.succeed(killed), + }), + ), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerReceiptsTest), + ); + yield* Effect.gen(function* () { + const service = yield* ThreadManagementService.ThreadManagementService; + const stopping = yield* service.forceStopThread({ threadId, runId }).pipe(Effect.forkChild); + yield* TestClock.adjust("10 seconds"); + expect(yield* Fiber.join(stopping)).toEqual(expected); + expect(dispatched).toMatchObject([ + { + type: "thread.background-work.settle", + providerThreadId, + reason: "Force stopped by user", + }, + ]); + if (teardown.at(-1) === "detach") yield* Deferred.await(tornDown); + expect(teardowns).toEqual(teardown); + }).pipe(Effect.provide(layerTest)); + }), +); + +it.effect("Force stop refuses once a newer run has started, before killing anything", () => + Effect.gen(function* () { + const threadId = ThreadId.make("thread:thread-management:force-stop-superseded"); + let killed = false; + const layerTest = ThreadManagementService.layer.pipe( + Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), + Layer.provide( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadRecords: () => + Effect.succeed({ + thread: { id: threadId, deletedAt: null }, + runs: [ + { id: RunId.make("run:stopped"), ordinal: 1, status: "running" }, + { id: RunId.make("run:newer"), ordinal: 2, status: "running" }, + ], + providerThreads: [], + providerTurns: [], + } as unknown as OrchestrationV2ThreadProjection), + }), + ), + Layer.provide( + Layer.succeed(AgentScope, { + ...AgentScope.defaultValue(), + kill: () => Effect.sync(() => (killed = true)), + }), + ), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerReceiptsTest), + ); + const error = yield* Effect.gen(function* () { + const service = yield* ThreadManagementService.ThreadManagementService; + return yield* Effect.flip( + service.forceStopThread({ threadId, runId: RunId.make("run:stopped") }), + ); + }).pipe(Effect.provide(layerTest)); + expect(error.reason).toBe("superseded"); + expect(killed).toBe(false); + }), +); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 7b79b21e88153..192ed4007fc17 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -14,6 +14,9 @@ import { MessageId, OrchestrationV2StopWorkflowError, type OrchestrationV2StopWorkflowInput, + OrchestrationV2ForceStopThreadError, + type OrchestrationV2ForceStopThreadInput, + type OrchestrationV2ForceStopThreadResult, type ModelSelection, type OrchestrationV2Actor, type OrchestrationV2Command, @@ -30,6 +33,7 @@ import { ProjectId, PROVIDER_SEND_TURN_MAX_FILE_BYTES, ProviderInstanceId, + type ProviderSessionId, resolveProjectProviderInstanceEnabled, RunId, type ScheduledTaskId, @@ -38,10 +42,13 @@ import { } from "@t3tools/contracts"; import { delegatedTaskProgress } from "@t3tools/provider-core/server/subagentProjection"; import { threadTranscriptHeader } from "@t3tools/shared/threadTranscript"; +import { AgentScope } from "@t3tools/shared/AgentScope"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; @@ -454,6 +461,13 @@ export interface ThreadManagementServiceShape { readonly stopWorkflow: ( input: OrchestrationV2StopWorkflowInput, ) => Effect.Effect; + /** + * Kills the thread's agent for a provider that no longer responds, then ends + * its turn as interrupted. Never kills a process other working threads share. + */ + readonly forceStopThread: ( + input: OrchestrationV2ForceStopThreadInput, + ) => Effect.Effect; readonly awaitBackgroundTasks: (input: { readonly threadId: ThreadId; readonly taskIds: ReadonlyArray; @@ -547,12 +561,16 @@ function latestSteerableRun( } const SETTLE_AFTER_RUN_WAIT_MS = 24 * 60 * 60 * 1_000; +const FORCE_STOP_REASON = "Force stopped by user"; +const FORCE_STOP_RELEASE_WAIT = "10 seconds"; +const isForceStopThreadError = Schema.is(OrchestrationV2ForceStopThreadError); const make = Effect.gen(function* () { const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor; const orchestrator = yield* Orchestrator.OrchestratorV2; const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; const layerScope = yield* Effect.scope; + const agentScope = yield* AgentScope; const legacyImporter = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; const serverSettings = yield* ServerSettings.ServerSettingsService; const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2; @@ -1072,6 +1090,153 @@ const make = Effect.gen(function* () { return { type: "interrupt_requested", run: interruptibleRun, dispatch } as const; }); + const forceStopThread: ThreadManagementServiceShape["forceStopThread"] = Effect.fn( + "ThreadManagementService.forceStopThread", + )( + function* (input) { + // Under the thread lock no new run can be created between the check and the kill. The + // settlement below dispatches, which takes that lock itself, so it runs after release. + const { records, target, killed } = yield* threadDispatch.withLock( + input.threadId, + Effect.gen(function* () { + const records = yield* orchestrator.getThreadRecords(input.threadId, [ + "runs", + "providerThreads", + "providerTurns", + ]); + // The run Force stop acts on: the active one, else the latest that started. Queued + // follow-ups never ran, so they neither count as targets nor supersede the request. + const target = + latestActiveRun(records) ?? + records.runs + .filter((run) => run.status !== "queued") + .reduce( + (current, run) => + current === undefined || run.ordinal > current.ordinal ? run : current, + undefined, + ); + if (input.runId !== undefined && input.runId !== target?.id) { + return yield* new OrchestrationV2ForceStopThreadError({ + threadId: input.threadId, + reason: "superseded", + }); + } + // Shared scopes are never tracked per thread, so this only kills the thread's own agent. + const killed = yield* agentScope.kill(input.threadId); + return { records, target, killed }; + }), + ); + const sessionIds = new Set( + records.providerThreads.flatMap((providerThread) => + providerThread.appThreadId === input.threadId && providerThread.providerSessionId !== null + ? [providerThread.providerSessionId] + : [], + ), + ); + // A run that already ended may still hold background work; settling it is a no-op otherwise. + const run = target; + if (run !== undefined) { + const active = isActiveRun(run); + const providerTurn = records.providerTurns.findLast( + (turn) => + turn.runAttemptId === run.activeAttemptId && (!active || turn.status === "running"), + ); + const commandId = CommandId.make( + `server:force-stop:${run.id}:${yield* Clock.currentTimeMillis}`, + ); + yield* providerTurn === undefined && !active + ? Effect.void + : providerTurn === undefined + ? orchestrator.dispatch({ + type: "run.interrupt", + commandId, + threadId: input.threadId, + runId: run.id, + reason: FORCE_STOP_REASON, + }) + : orchestrator.dispatch({ + type: "thread.background-work.settle", + commandId, + threadId: input.threadId, + providerThreadId: providerTurn.providerThreadId, + providerTurnId: providerTurn.id, + reason: FORCE_STOP_REASON, + }); + } + const ownProviderThreadIds = records.providerThreads.flatMap((providerThread) => + providerThread.appThreadId === input.threadId ? [providerThread.id] : [], + ); + const detach = (providerSessionId: ProviderSessionId) => + sessions + .detach({ providerSessionId, threadId: input.threadId, detail: FORCE_STOP_REASON }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.force-stop-detach-failed", { + threadId: input.threadId, + cause, + }), + ), + Effect.forkIn(layerScope), + ); + let shared = false; + const sharedWith = new Set(); + const releases: Array<{ + readonly providerSessionId: ProviderSessionId; + readonly fiber: Fiber.Fiber; + }> = []; + for (const providerSessionId of sessionIds) { + if (!(yield* sessions.isLive(providerSessionId))) continue; + // Decided just before teardown, so the reply matches what happens to the process. + const others = yield* sessions.otherWorkingThreads({ + providerSessionId, + threadId: input.threadId, + }); + for (const threadId of others) sharedWith.add(threadId); + if (others.length > 0) { + shared = true; + yield* detach(providerSessionId); + continue; + } + // The session manager re-checks atomically with turn starts and keeps a session a + // sibling began using in the meantime. + const fiber = yield* sessions + .releaseUnlessBusy({ + providerSessionId, + ownProviderThreadIds, + reason: "manual_shutdown", + detail: FORCE_STOP_REASON, + }) + .pipe(Effect.forkIn(layerScope)); + releases.push({ providerSessionId, fiber }); + } + // Teardown waits on the provider's own exit paths; past this bound it keeps going on its own. + let releasing = false; + for (const release of releases) { + const released = yield* Fiber.join(release.fiber).pipe( + Effect.timeoutOption(FORCE_STOP_RELEASE_WAIT), + ); + if (Option.isNone(released)) { + releasing = true; + } else if (!released.value) { + shared = true; + yield* detach(release.providerSessionId); + } + } + return { + outcome: killed ? "killed" : shared ? "shared" : releasing ? "releasing" : "released", + sharedWithThreadIds: [...sharedWith], + } as const; + }, + (effect, input) => + effect.pipe( + Effect.mapError((cause) => + isForceStopThreadError(cause) + ? cause + : new OrchestrationV2ForceStopThreadError({ threadId: input.threadId, cause }), + ), + ), + ); + const stopWorkflow: ThreadManagementServiceShape["stopWorkflow"] = Effect.fn( "ThreadManagementService.stopWorkflow", )( @@ -1354,6 +1519,7 @@ const make = Effect.gen(function* () { interruptThread, stopDelegatedTasks, stopWorkflow, + forceStopThread, awaitBackgroundTasks, getThreadEventSequence: orchestrator.getThreadEventSequence, recoverDelegatedTask: orchestrator.recoverDelegatedTask, diff --git a/apps/server/src/orchestration-v2/ThreadStop.test.ts b/apps/server/src/orchestration-v2/ThreadStop.test.ts index 23a52f39d33d3..16dcedaf8ac9c 100644 --- a/apps/server/src/orchestration-v2/ThreadStop.test.ts +++ b/apps/server/src/orchestration-v2/ThreadStop.test.ts @@ -1552,3 +1552,115 @@ it.effect.each(["missing session", "completed", "failed", "cancelled"] as const) ); }).pipe(Effect.provide(layerTest)), ); + +it.effect("an unanswered Stop is flagged until Force stop ends the run with its own reason", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread:force-stop"); + yield* createWatchingThread(threadId, 6); + yield* send(threadId, "work", "start_immediately"); + const now = yield* DateTime.now; + const started = yield* orchestrator.getThreadProjection(threadId); + const run = started.runs[0]!; + const providerThreadId = run.providerThreadId!; + const providerTurnId = ProviderTurnId.make("provider-turn:force-stop"); + const apply = (event: Omit) => + projections.apply({ + ...event, + id: EventId.make(`event:force-stop:${event.type}`), + threadId, + } as OrchestrationV2DomainEvent); + const attempt = started.attempts[0]!; + const attemptId = attempt.id; + yield* apply({ + type: "run.updated", + runId: run.id, + occurredAt: now, + payload: { ...run, status: "running", startedAt: now, activeAttemptId: attemptId }, + }); + yield* apply({ + type: "run-attempt.updated", + occurredAt: now, + payload: { ...attempt, status: "running", startedAt: now }, + }); + yield* apply({ + type: "provider-turn.updated", + occurredAt: now, + payload: { + id: providerTurnId, + providerThreadId, + nodeId: run.rootNodeId!, + runAttemptId: attemptId, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + yield* apply({ + type: "turn-item.updated", + runId: run.id, + occurredAt: now, + payload: { + id: TurnItemId.make("turn-item:force-stop:interrupt-request"), + threadId, + runId: run.id, + nodeId: run.rootNodeId!, + providerThreadId, + providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: 1_000, + status: "completed", + title: "Interrupt requested", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "run_interrupt_request", + message: "Stopped by user", + }, + }); + const target = { threadId, providerThreadId, providerTurnId }; + const request = Effect.map(orchestrator.getThreadProjection(threadId), (projection) => + projection.turnItems.find((item) => item.type === "run_interrupt_request"), + ); + + yield* orchestrator.dispatch({ + ...target, + type: "thread.interrupt.unresponsive", + commandId: CommandId.make("force-stop:unresponsive"), + }); + const flagged = yield* request; + assert.isTrue( + flagged?.type === "run_interrupt_request" && flagged.unresponsiveAt !== undefined, + ); + assert.deepEqual((yield* threadState(threadId)).runs, ["running"]); + + yield* orchestrator.dispatch({ + ...target, + type: "thread.background-work.settle", + commandId: CommandId.make("force-stop:settle"), + reason: "Force stopped by user", + }); + const settled = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual( + settled.runs.map((candidate) => candidate.status), + ["interrupted"], + ); + const result = settled.turnItems.find((item) => item.type === "run_interrupt_result"); + assert.equal( + result?.type === "run_interrupt_result" ? result.message : undefined, + "Force stopped by user", + ); + + // A provider confirming late changes nothing once the run has ended. + const late = yield* orchestrator.dispatch({ + ...target, + type: "thread.interrupt.unresponsive", + commandId: CommandId.make("force-stop:late-unresponsive"), + }); + assert.lengthOf(late.storedEvents, 0); + }).pipe(Effect.provide(layerTest)), +); diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 286452a5c5781..2a9998f4cfbf6 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -110,6 +110,8 @@ const withFakeSystemd = ( ...fileSystem, readFileString: (path, encoding) => fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding), + writeFileString: (path, data, options) => + fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options), }), ); const { stateDir } = yield* ServerConfig.ServerConfig; @@ -213,6 +215,36 @@ describe("AgentScope service", () => { ), ); + it.effect.each([{ oomPolicy: true }, { oomPolicy: false }])( + "Force stop kills a thread's own scope but never a shared Codex scope (OOM policy $oomPolicy)", + ({ oomPolicy }) => + withFakeSystemd( + ({ scope, units, calls, cgroupRoot }) => + Effect.gen(function* () { + const agent = yield* scope.wrap({ + command: "agent", + args: [], + name: "t", + threadId: "a", + }); + exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 0 }); + const codex = yield* launchCodex(scope, "b"); + exitedAgent(units, cgroupRoot, unitOf(codex), { populated: true, oomKills: 0 }); + + expect(yield* scope.kill("a")).toBe(true); + expect( + NodeFS.readFileSync(NodePath.join(cgroupRoot, unitOf(agent), "cgroup.kill"), "utf8"), + ).toBe("1"); + expect(yield* scope.kill("b")).toBe(false); + expect(NodeFS.existsSync(NodePath.join(cgroupRoot, unitOf(codex), "cgroup.kill"))).toBe( + false, + ); + expect(calls.filter(([, verb]) => verb === "kill")).toEqual([]); + }), + { oomPolicy }, + ), + ); + it.effect("keeps a running scope while hundreds of other threads launch", () => withFakeSystemd(({ scope, units }) => Effect.gen(function* () { diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index c72e4eed88a9c..b9b6b56acb140 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -215,7 +215,8 @@ export const make = Effect.gen(function* () { }), ); - const scopeByThread = new Map(); + // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd. + const scopeByThread = new Map(); const readScope = Effect.fn("agentScope.readScope")(function* (systemctl: string, unit: string) { const shown = yield* run(systemctl, [ @@ -277,10 +278,11 @@ export const make = Effect.gen(function* () { systemctl: string, threadId: string, unit: string | undefined, + shared = false, ) { const previous = scopeByThread.get(threadId); scopeByThread.delete(threadId); - if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false }); + if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false, shared }); if (previous !== undefined) yield* forget(systemctl, previous.unit); if (scopeByThread.size > MAX_TRACKED_THREADS) yield* evictOldest(systemctl); }); @@ -340,6 +342,40 @@ export const make = Effect.gen(function* () { } }); + const kill: AgentScopeShape["kill"] = Effect.fn("AgentScope.kill")(function* (threadId) { + const tracked = scopeByThread.get(threadId); + if (tracked === undefined || tracked.shared) return false; + const scope = yield* systemd; + if (scope === undefined) return false; + const shown = yield* run(scope.systemctl, [ + "--user", + "show", + tracked.unit, + "--property=ActiveState,ControlGroup", + ]); + if (shown._tag === "None" || shown.value.code !== 0) return false; + const values = parseKeyValues(shown.value.stdout); + if (values.get("ActiveState") !== "active") return false; + const controlGroup = values.get("ControlGroup") ?? ""; + // cgroup.kill (Linux 5.14+) kills the whole tree at once; systemd's kill is the fallback. + const killedByCgroup = + controlGroup.length > 0 && + (yield* fileSystem + .writeFileString(path.join("/sys/fs/cgroup", controlGroup, "cgroup.kill"), "1") + .pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + )); + if (killedByCgroup) return true; + const killed = yield* run(scope.systemctl, [ + "--user", + "kill", + "--signal=SIGKILL", + tracked.unit, + ]); + return killed._tag === "Some" && killed.value.code === 0; + }); + const clear: AgentScopeShape["clear"] = Effect.fn("agentScope.clear")(function* (threadId) { if (!scopeByThread.has(threadId)) return; const scope = yield* systemd; @@ -365,7 +401,8 @@ export const make = Effect.gen(function* () { // Clear the launch thread's previous scope instead. const continuing = shared === true && scope.supportsOomPolicy; const owned = threadId !== undefined && !continuing; - if (threadId !== undefined) yield* track(scope.systemctl, threadId, owned ? unit : undefined); + if (threadId !== undefined) + yield* track(scope.systemctl, threadId, owned ? unit : undefined, shared === true); const systemdScope: AgentSystemdScope = { systemdRun: scope.systemdRun, runtimeDir: scope.runtimeDir, @@ -385,6 +422,7 @@ export const make = Effect.gen(function* () { }), oomKilled, stop, + kill, clear, } satisfies AgentScopeShape; }); diff --git a/apps/server/src/project/AgentSessionScanner.ts b/apps/server/src/project/AgentSessionScanner.ts index 9c26b187a55bf..7ec79c38a38a9 100644 --- a/apps/server/src/project/AgentSessionScanner.ts +++ b/apps/server/src/project/AgentSessionScanner.ts @@ -1112,9 +1112,11 @@ export const make = Effect.gen(function* () { const seenHomes = new Set(); for (const { instanceId, config: instance } of instances) { const homeVariable = source === "claudeAgent" ? "CLAUDE_CONFIG_DIR" : "CODEX_HOME"; + const configuredHome = instance.environment?.findLast( + (variable) => variable.name === homeVariable, + )?.value; const environmentHome = - instance.environment?.findLast((variable) => variable.name === homeVariable)?.value ?? - hostEnvironment[homeVariable]; + typeof configuredHome === "string" ? configuredHome : hostEnvironment[homeVariable]; let homePath: string; if (source === "claudeAgent") { diff --git a/apps/server/src/provider/Drivers/claudeUsage.test.ts b/apps/server/src/provider/Drivers/claudeUsage.test.ts index a9ebb05a2e1e7..6f457f4f4cf2f 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.test.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.test.ts @@ -32,9 +32,9 @@ function claudeLine(overrides: { describe("parseClaudeLine", () => { it("extracts token totals and a dedupe key", () => { - const record = parseClaudeLine(claudeLine({ messageId: "msg_1", contentType: "text" })); + const [record] = parseClaudeLine(claudeLine({ messageId: "msg_1", contentType: "text" })); - expect(record).not.toBeNull(); + expect(record).toBeDefined(); expect(record?.provider).toBe("claude"); expect(record?.model).toBe("claude-fable-5"); expect(record?.totals).toEqual({ @@ -50,8 +50,8 @@ describe("parseClaudeLine", () => { it("gives every content block of one message the same dedupe key", () => { // Content-block records repeat full parent usage; summing them overcounts. - const text = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "text" })); - const toolUse = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "tool_use" })); + const [text] = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "text" })); + const [toolUse] = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "tool_use" })); expect(text?.dedupeKey).toBe(toolUse?.dedupeKey); expect(text?.totals).toEqual(toolUse?.totals); @@ -86,3 +86,133 @@ describe("parseClaudeLine", () => { expect(claudeUsageFormat.parseProjected(synthetic, undefined)).toEqual([]); }); }); + +describe("Claude advisor iterations", () => { + const advisorTurn = { + type: "assistant", + timestamp: "2026-10-22T01:31:11.984Z", + sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + requestId: "req_1", + message: { + id: "msg_1", + model: "claude-opus-5-5", + usage: { + input_tokens: 4, + cache_read_input_tokens: 118320, + cache_creation_input_tokens: 2676, + output_tokens: 4506, + speed: "standard", + iterations: [ + { + type: "message", + input_tokens: 2, + cache_read_input_tokens: 58352, + cache_creation_input_tokens: 1616, + output_tokens: 141, + }, + { + type: "advisor_message", + model: "claude-fable-5-1", + input_tokens: 54204, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + output_tokens: 4229, + }, + { + type: "message", + input_tokens: 2, + cache_read_input_tokens: 59968, + cache_creation_input_tokens: 1060, + output_tokens: 4365, + }, + ], + }, + }, + }; + const withIterations = (iterations: readonly unknown[]) => ({ + ...advisorTurn, + message: { ...advisorTurn.message, usage: { ...advisorTurn.message.usage, iterations } }, + }); + const parse = (turn: unknown) => claudeUsageFormat.parseLine(JSON.stringify(turn), undefined); + + it("reports the advisor model's tokens as their own record beside the unchanged executor record", () => { + const executor = { + provider: "claude", + timestampMs: Date.parse("2026-10-22T01:31:11.984Z"), + model: "claude-opus-5-5", + sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + totals: { + uncachedInputTokens: 4, + cachedInputTokens: 118320, + cacheCreationTokens: 2676, + outputTokens: 4506, + reasoningTokens: 0, + }, + reportedCostUsd: null, + speed: "standard", + dedupeKey: "msg_1:req_1", + }; + const expected = [ + executor, + { + ...executor, + model: "claude-fable-5-1", + totals: { + uncachedInputTokens: 54204, + cachedInputTokens: 0, + cacheCreationTokens: 0, + outputTokens: 4229, + reasoningTokens: 0, + }, + dedupeKey: expect.any(String), + }, + ]; + + expect(parse(advisorTurn)).toEqual(expected); + expect(claudeUsageFormat.parseProjected(advisorTurn, undefined)).toEqual(expected); + }); + + it("keys the advisor record apart from the executor and the same across a message's content blocks", () => { + const block = (type: string) => + parse({ ...advisorTurn, message: { ...advisorTurn.message, content: [{ type }] } }); + const [executorKey, advisorKey] = block("text").map((record) => record.dedupeKey); + + expect(advisorKey).toEqual(expect.any(String)); + expect(advisorKey).not.toBe(executorKey); + expect(block("tool_use").map((record) => record.dedupeKey)).toEqual([executorKey, advisorKey]); + + const unkeyed = parse({ + ...advisorTurn, + requestId: undefined, + message: { ...advisorTurn.message, id: undefined }, + }); + expect(unkeyed.map((record) => record.dedupeKey)).toEqual([null, null]); + }); + + it("emits only the executor record without an advisor iteration", () => { + const executorIterations = advisorTurn.message.usage.iterations.filter( + (iteration) => iteration.type === "message", + ); + + expect(parse(withIterations(executorIterations))).toHaveLength(1); + expect(claudeUsageFormat.parseProjected(withIterations([]), undefined)).toHaveLength(1); + expect( + claudeUsageFormat.parseLine( + claudeLine({ messageId: "msg_3", contentType: "text" }), + undefined, + ), + ).toHaveLength(1); + }); + + it("ignores advisor iterations without a usable model and malformed entries", () => { + const unusable = withIterations([ + null, + "advisor_message", + { type: "advisor_message", input_tokens: 5 }, + { type: "advisor_message", model: "", input_tokens: 5 }, + { type: "advisor_message", model: "", input_tokens: 5 }, + ]); + + expect(parse(unusable).map((record) => record.model)).toEqual(["claude-opus-5-5"]); + }); +}); diff --git a/apps/server/src/provider/Drivers/claudeUsage.ts b/apps/server/src/provider/Drivers/claudeUsage.ts index c5ff71a5c839e..aca7696d31df5 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.ts @@ -20,35 +20,35 @@ export const CLAUDE_SYNTHETIC_MODEL = ""; * the same complete `usage` object for the parent message. Summing them overcounts by roughly * 2.4x on a real workload, so the caller must drop repeats by `dedupeKey` and keep the first. */ -export function parseClaudeLine(line: string): UsageRecord | null { +export function parseClaudeLine(line: string): readonly UsageRecord[] { let parsed: unknown; try { parsed = JSON.parse(line); } catch { - return null; + return []; } return parseClaudeRecord(parsed); } -function parseClaudeRecord(parsed: unknown): UsageRecord | null { - if (typeof parsed !== "object" || parsed === null) return null; +function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { + if (typeof parsed !== "object" || parsed === null) return []; const record = parsed as Record; - if (record["type"] !== "assistant") return null; + if (record["type"] !== "assistant") return []; const message = record["message"]; - if (typeof message !== "object" || message === null) return null; + if (typeof message !== "object" || message === null) return []; const messageRecord = message as Record; const usage = messageRecord["usage"]; - if (typeof usage !== "object" || usage === null) return null; + if (typeof usage !== "object" || usage === null) return []; const usageRecord = usage as Record; const timestampMs = parseTimestampMs(record["timestamp"]); - if (timestampMs === null) return null; + if (timestampMs === null) return []; const model = typeof messageRecord["model"] === "string" ? messageRecord["model"] : ""; - if (model.length === 0 || model === CLAUDE_SYNTHETIC_MODEL) return null; + if (model.length === 0 || model === CLAUDE_SYNTHETIC_MODEL) return []; const messageId = typeof messageRecord["id"] === "string" ? messageRecord["id"] : null; const requestId = typeof record["requestId"] === "string" ? record["requestId"] : null; @@ -57,7 +57,7 @@ function parseClaudeRecord(parsed: unknown): UsageRecord | null { const cost = record["costUSD"]; - return { + const executor: UsageRecord = { provider: "claude", timestampMs, model, @@ -74,10 +74,40 @@ function parseClaudeRecord(parsed: unknown): UsageRecord | null { speed: usageRecord["speed"] === "fast" ? "fast" : "standard", dedupeKey, }; -} -const orEmpty = (record: UsageRecord | null): readonly UsageRecord[] => - record === null ? [] : [record]; + // Top-level `usage` is the executor's alone. Advisor calls run on another model at its own rates + // and appear only in `iterations`, so each one is its own record under that model. + const records = [executor]; + const iterations: readonly unknown[] = Array.isArray(usageRecord["iterations"]) + ? usageRecord["iterations"] + : []; + for (const [index, iteration] of iterations.entries()) { + if (typeof iteration !== "object" || iteration === null) continue; + const advisor = iteration as Record; + if (advisor["type"] !== "advisor_message") continue; + + const advisorModel = typeof advisor["model"] === "string" ? advisor["model"] : ""; + if (advisorModel.length === 0 || advisorModel === CLAUDE_SYNTHETIC_MODEL) continue; + + records.push({ + provider: "claude", + timestampMs, + model: advisorModel, + sessionId: executor.sessionId, + totals: { + uncachedInputTokens: tokenCount(advisor["input_tokens"]), + cachedInputTokens: tokenCount(advisor["cache_read_input_tokens"]), + cacheCreationTokens: tokenCount(advisor["cache_creation_input_tokens"]), + outputTokens: tokenCount(advisor["output_tokens"]), + reasoningTokens: 0, + }, + reportedCostUsd: null, + speed: executor.speed, + dedupeKey: dedupeKey === null ? null : `${dedupeKey}:advisor:${index}`, + }); + } + return records; +} export const claudeUsageFormat: TranscriptUsageFormat = { selectFields: { @@ -89,8 +119,8 @@ export const claudeUsageFormat: TranscriptUsageFormat = { message: { id: true, model: true, usage: true }, }, usageMarkers: ['"usage"'], - parseLine: (line) => orEmpty(parseClaudeLine(line)), - parseProjected: (projected) => orEmpty(parseClaudeRecord(projected)), + parseLine: (line) => parseClaudeLine(line), + parseProjected: (projected) => parseClaudeRecord(projected), }; export const claudeUsageReader: ProviderUsageReader = { diff --git a/apps/server/src/provider/ProviderInstanceRegistry.test.ts b/apps/server/src/provider/ProviderInstanceRegistry.test.ts index de8fc1deae452..dc9c94e854f68 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.test.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.test.ts @@ -12,16 +12,22 @@ import { type CodexSettings, ProviderDriverKind, type ProviderInstanceConfigMap, + type ResolvedProviderInstanceEnvironment, ProviderInstanceId, + SecretProviderId, + type SecretReference, } from "@t3tools/contracts"; import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; import { HttpClient, HttpClientResponse } from "effect/http"; @@ -37,6 +43,8 @@ import { makeProviderInstanceRegistry } from "./ProviderInstanceRegistry.ts"; import * as ProviderOrchestrationAdapterInfrastructure from "./ProviderOrchestrationAdapterInfrastructure.ts"; import * as PortScanner from "../preview/PortScanner.ts"; import * as ProviderHostLive from "./ProviderHostLive.ts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import type { AnyProviderDriver, ProviderInstance } from "@t3tools/provider-core/server/driver"; vi.mock("@anthropic-ai/claude-agent-sdk", { spy: true }); @@ -164,6 +172,7 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { }), ), Layer.provideMerge(Layer.mock(ServerSecretStore.ServerSecretStore)({})), + Layer.provideMerge(Layer.mock(SecretResolver.SecretResolver)({})), Layer.provideMerge( Layer.succeed(ServerEnvironment.ServerEnvironmentIdentity, { getEnvironmentId: Effect.succeed( @@ -643,3 +652,169 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { }).pipe(Effect.provide(layerTest)), ); }); + +const DisabledByDefaultConfig = Schema.Struct({ + enabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), +}); + +describe("ProviderInstanceRegistry secret environment", () => { + const driverKind = ProviderDriverKind.make("fake"); + const instanceId = ProviderInstanceId.make("fake_main"); + const secret: SecretReference = { + provider: SecretProviderId.make("personal"), + reference: "op://Private/OpenAI/credential", + }; + + function setup( + initial: Record, + configSchema: AnyProviderDriver["configSchema"] = Schema.Unknown, + ) { + const vault = new Map(Object.entries(initial)); + const reads: Array> = []; + const invalidated: Array> = []; + const created: Array = []; + const driver: AnyProviderDriver = { + driverKind, + metadata: { displayName: "Fake" }, + configSchema, + defaultConfig: () => ({}), + create: (input) => + Effect.sync(() => { + created.push(input.environment); + return { instanceId: input.instanceId, enabled: input.enabled } as ProviderInstance; + }), + }; + const gate = Deferred.makeUnsafe(); + const resolver = Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => + Deferred.await(gate).pipe( + Effect.andThen( + Effect.sync(() => { + reads.push(references.map((reference) => reference.reference)); + return references.map((reference) => vault.get(reference.reference)); + }), + ), + ), + invalidate: (references = []) => + Effect.sync(() => { + invalidated.push(references.map((reference) => reference.reference)); + }), + }); + const configMap: ProviderInstanceConfigMap = { + [instanceId]: { + driver: driverKind, + config: {}, + environment: [ + { name: "OPENAI_API_KEY", value: secret, sensitive: true }, + { name: "BASE_URL", value: "https://example.test", sensitive: false }, + ], + }, + }; + const makeGated = makeProviderInstanceRegistry({ drivers: [driver], configMap }).pipe( + Effect.provide(resolver), + ); + // Secrets are read after startup, so wait for the change that applies them. + const make = Effect.gen(function* () { + const built = yield* makeGated; + const changes = yield* built.registry.subscribeChanges; + yield* Deferred.succeed(gate, undefined); + yield* PubSub.take(changes); + return built; + }); + return { vault, reads, invalidated, created, make, makeGated, gate }; + } + + it.effect("starts an instance with the secret its environment points at", () => { + const fixture = setup({ [secret.reference]: "sk-live" }); + return Effect.gen(function* () { + const { registry } = yield* fixture.make; + + expect(yield* registry.getInstance(instanceId)).toBeDefined(); + expect(fixture.created).toEqual([ + [ + { name: "OPENAI_API_KEY", value: "sk-live", sensitive: true }, + { name: "BASE_URL", value: "https://example.test", sensitive: false }, + ], + ]); + }).pipe(Effect.scoped); + }); + + it.effect("marks an instance unavailable and names the variable it could not read", () => { + const fixture = setup({}); + return Effect.gen(function* () { + const { registry } = yield* fixture.make; + + expect(yield* registry.getInstance(instanceId)).toBeUndefined(); + expect(fixture.created).toEqual([]); + const [unavailable] = yield* registry.listUnavailable; + expect(unavailable?.instanceId).toBe(instanceId); + expect(unavailable?.unavailableReason).toContain("OPENAI_API_KEY"); + }).pipe(Effect.scoped); + }); + + it.effect("does not read the secrets of an instance its config decodes as disabled", () => { + const fixture = setup({ [secret.reference]: "sk-live" }, DisabledByDefaultConfig); + return Effect.gen(function* () { + const { mutator } = yield* fixture.makeGated; + yield* mutator.reloadSecrets(); + + expect(fixture.reads).toEqual([]); + expect(fixture.invalidated).toEqual([]); + expect(fixture.created).toHaveLength(1); + }).pipe(Effect.scoped); + }); + + it.effect("re-reads secrets on reload and rebuilds only when they changed", () => { + const fixture = setup({}); + return Effect.gen(function* () { + const { registry, mutator } = yield* fixture.make; + + fixture.vault.set(secret.reference, "sk-live"); + yield* mutator.reloadSecrets(); + expect(fixture.invalidated).toEqual([[secret.reference]]); + expect(yield* registry.getInstance(instanceId)).toBeDefined(); + expect(yield* registry.listUnavailable).toEqual([]); + + yield* mutator.reloadSecrets(instanceId); + expect(fixture.created).toHaveLength(1); + + fixture.vault.set(secret.reference, "sk-rotated"); + yield* mutator.reloadSecrets(instanceId); + expect(fixture.created.map((environment) => environment[0]?.value)).toEqual([ + "sk-live", + "sk-rotated", + ]); + }).pipe(Effect.scoped); + }); + + it.effect("keeps a working instance live when a reload cannot read its secret", () => { + const fixture = setup({ [secret.reference]: "sk-live" }); + return Effect.gen(function* () { + const { registry, mutator } = yield* fixture.make; + + fixture.vault.delete(secret.reference); + yield* mutator.reloadSecrets(); + + expect(yield* registry.getInstance(instanceId)).toBeDefined(); + expect(yield* registry.listUnavailable).toEqual([]); + expect(fixture.created).toHaveLength(1); + }).pipe(Effect.scoped); + }); + + it.effect("starts without waiting for secrets to be read", () => { + const fixture = setup({ [secret.reference]: "sk-live" }); + return Effect.gen(function* () { + const { registry } = yield* fixture.makeGated; + const changes = yield* registry.subscribeChanges; + + expect(yield* registry.getInstance(instanceId)).toBeUndefined(); + const [pending] = yield* registry.listUnavailable; + expect(pending?.unavailableReason).toBe("Reading secrets…"); + + yield* Deferred.succeed(fixture.gate, undefined); + yield* PubSub.take(changes); + expect(yield* registry.getInstance(instanceId)).toBeDefined(); + expect(yield* registry.listUnavailable).toEqual([]); + }).pipe(Effect.scoped); + }); +}); diff --git a/apps/server/src/provider/ProviderInstanceRegistry.ts b/apps/server/src/provider/ProviderInstanceRegistry.ts index 3bbbe2900b9f2..70352c5ef8593 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.ts @@ -3,7 +3,10 @@ import { ProviderInstanceId, type ProviderInstanceConfig, type ProviderInstanceConfigMap, + type ProviderInstanceEnvironment, type ProviderDriverKind, + type ResolvedProviderInstanceEnvironment, + type SecretReference, type ServerProvider, } from "@t3tools/contracts"; import * as Context from "effect/Context"; @@ -11,12 +14,16 @@ import * as Effect from "effect/Effect"; import * as Equal from "effect/Equal"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; + import { buildUnavailableProviderSnapshot } from "./unavailableProviderSnapshot.ts"; import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMutator.ts"; import type { AnyProviderDriver, ProviderInstance } from "@t3tools/provider-core/server/driver"; @@ -39,6 +46,7 @@ interface LiveEntry { readonly instance: ProviderInstance; readonly scope: Scope.Closeable; readonly entry: ProviderInstanceConfig; + readonly environment: ResolvedProviderInstanceEnvironment; } interface RegistryState { @@ -50,6 +58,39 @@ interface RegistryState { const entryEqual = (a: ProviderInstanceConfig, b: ProviderInstanceConfig): boolean => Equal.equals(a, b); +const secretReferencesIn = (environment: ProviderInstanceEnvironment): Array => + environment.flatMap((variable) => (typeof variable.value === "string" ? [] : [variable.value])); + +const secretReferencesOf = (entry: ProviderInstanceConfig): ReadonlyArray => + secretReferencesIn(entry.environment ?? []); + +const withoutSecretReferences = ( + environment: ProviderInstanceEnvironment, +): ResolvedProviderInstanceEnvironment => + environment.flatMap((variable) => + typeof variable.value === "string" ? [{ ...variable, value: variable.value }] : [], + ); + +const readSecretEnvironment = Effect.fn("ProviderInstanceRegistry.readSecretEnvironment")( + function* (environment: ProviderInstanceEnvironment) { + const references = secretReferencesIn(environment); + const values = + references.length === 0 ? [] : yield* (yield* SecretResolver.SecretResolver).read(references); + const unreadable: Array = []; + let index = 0; + const resolved: ResolvedProviderInstanceEnvironment = environment.flatMap((variable) => { + if (typeof variable.value === "string") return [{ ...variable, value: variable.value }]; + const secret = values[index++]; + if (secret === undefined) { + unreadable.push(variable.name); + return []; + } + return [{ ...variable, value: secret, sensitive: true }]; + }); + return { environment: resolved, unreadable }; + }, +); + /** Conflicting saved flags must preserve any explicit disable. */ const resolveEntryEnabled = (entry: ProviderInstanceConfig, typedConfig: unknown): boolean => { const rawConfigEnabled = providerInstanceConfigEnabledFlag(entry.config); @@ -59,19 +100,32 @@ const resolveEntryEnabled = (entry: ProviderInstanceConfig, typedConfig: unknown return entry.enabled ?? providerInstanceConfigEnabledFlag(typedConfig) ?? true; }; +// The enabled state `buildEntry` decodes, so a disabled instance never reads its secrets. +const isEntryEnabled = Effect.fn("ProviderInstanceRegistry.isEntryEnabled")(function* ( + driversById: ReadonlyMap>, + entry: ProviderInstanceConfig, +) { + const driver = driversById.get(entry.driver); + if (!driver) return false; + const decoder = Schema.decodeUnknownEffect(driver.configSchema); + const decoded = yield* decoder(entry.config ?? driver.defaultConfig()).pipe(Effect.option); + return Option.isSome(decoded) && resolveEntryEnabled(entry, decoded.value); +}); + const buildEntry = Effect.fn("ProviderInstanceRegistry.buildEntry")(function* (input: { readonly driversById: ReadonlyMap>; readonly parentScope: Scope.Scope; readonly instanceId: ProviderInstanceId; readonly rawInstanceId: string; readonly entry: ProviderInstanceConfig; + readonly deferSecrets: boolean; }): Effect.fn.Return< | { readonly kind: "live"; readonly live: LiveEntry } | { readonly kind: "unavailable"; readonly snapshot: ServerProvider }, never, - R + R | SecretResolver.SecretResolver > { - const { driversById, parentScope, instanceId, rawInstanceId, entry } = input; + const { driversById, parentScope, instanceId, rawInstanceId, entry, deferSecrets } = input; const driver = driversById.get(entry.driver); if (!driver) { return { @@ -109,6 +163,37 @@ const buildEntry = Effect.fn("ProviderInstanceRegistry.buildEntry")(function* 0) { + return { + kind: "unavailable" as const, + snapshot: yield* buildUnavailableProviderSnapshot({ + driverKind: entry.driver, + instanceId, + displayName: entry.displayName, + accentColor: entry.accentColor, + reason: "Reading secrets…", + }), + }; + } + const { environment, unreadable } = enabled + ? yield* readSecretEnvironment(entry.environment ?? []) + : { environment: withoutSecretReferences(entry.environment ?? []), unreadable: [] }; + // Starting with a blank credential would fail later with a less useful error, or fall back to + // whatever the server inherited. + if (unreadable.length > 0) { + return { + kind: "unavailable" as const, + snapshot: yield* buildUnavailableProviderSnapshot({ + driverKind: entry.driver, + instanceId, + displayName: entry.displayName, + accentColor: entry.accentColor, + reason: `Could not read ${unreadable.join(", ")} from its secret provider. Unlock it, then refresh.`, + }), + }; + } + const childScope = yield* Scope.make(); yield* Scope.addFinalizer(parentScope, Scope.close(childScope, Exit.void).pipe(Effect.ignore)); @@ -117,8 +202,8 @@ const buildEntry = Effect.fn("ProviderInstanceRegistry.buildEntry")(function* (input: { readonly state: RegistryState; readonly driversById: ReadonlyMap>; readonly parentScope: Scope.Scope; -}): ((configMap: ProviderInstanceConfigMap) => Effect.Effect) => { +}): (( + configMap: ProviderInstanceConfigMap, + rebuild?: ReadonlySet, + deferSecrets?: boolean, +) => Effect.Effect) => { const { state, driversById, parentScope } = input; - return (configMap: ProviderInstanceConfigMap) => + return (configMap, rebuild = new Set(), deferSecrets = false) => Effect.gen(function* () { const previousEntries = yield* Ref.get(state.entries); const previousUnavailable = yield* Ref.get(state.unavailable); @@ -175,7 +265,10 @@ const makeReconcile = (input: { continue; } const nextEntry = configMap[instanceId]; - if (nextEntry !== undefined && !entryEqual(live.entry, nextEntry)) { + if ( + rebuild.has(instanceId) || + (nextEntry !== undefined && !entryEqual(live.entry, nextEntry)) + ) { replacedIds.add(instanceId); } } @@ -186,6 +279,20 @@ const makeReconcile = (input: { } } + // Read every secret the builds below need in one go, so a vault unlocks once. + const references = yield* Effect.forEach(nextRaw, ([rawInstanceId, entry]) => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make(rawInstanceId); + const kept = previousEntries.has(instanceId) && !replacedIds.has(instanceId); + return kept || !(yield* isEntryEnabled(driversById, entry)) + ? [] + : secretReferencesOf(entry); + }), + ).pipe(Effect.map((perEntry) => perEntry.flat())); + if (references.length > 0 && !deferSecrets) { + yield* (yield* SecretResolver.SecretResolver).read(references); + } + const builtEntries = new Map(); const builtUnavailable = new Map(); let orderChanged = false; @@ -208,6 +315,7 @@ const makeReconcile = (input: { instanceId, rawInstanceId, entry, + deferSecrets, }); if (result.kind === "live") { builtEntries.set(instanceId, result.live); @@ -260,7 +368,7 @@ export const makeProviderInstanceRegistry = Effect.fn( readonly mutator: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape; }, never, - R | Scope.Scope + R | SecretResolver.SecretResolver | Scope.Scope > { const driversById = new Map>( input.drivers.map((driver) => [driver.driverKind, driver]), @@ -272,7 +380,7 @@ export const makeProviderInstanceRegistry = Effect.fn( // scopes to the *caller's* scope instead of the registry's. const parentScope = yield* Scope.Scope; - const driverContext = yield* Effect.context(); + const driverContext = yield* Effect.context(); const entries = yield* Ref.make>(new Map()); const unavailable = yield* Ref.make>(new Map()); @@ -281,10 +389,76 @@ export const makeProviderInstanceRegistry = Effect.fn( const state: RegistryState = { entries, unavailable, changes }; const reconcileWithR = makeReconcile({ state, driversById, parentScope }); + const configMapRef = yield* Ref.make(input.configMap); + const semaphore = yield* Semaphore.make(1); const reconcile: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape["reconcile"] = - (configMap) => reconcileWithR(configMap).pipe(Effect.provideContext(driverContext)); + (configMap) => + semaphore.withPermits(1)( + Ref.set(configMapRef, configMap).pipe( + Effect.andThen(reconcileWithR(configMap)), + Effect.provideContext(driverContext), + ), + ); - yield* reconcile(input.configMap); + // Rebuilds an instance only when it was unavailable or its secrets changed, so a refresh does + // not restart providers whose credentials are the same. + const reloadSecrets: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape["reloadSecrets"] = + Effect.fn("ProviderInstanceRegistry.reloadSecrets")( + function* (instanceId) { + const configMap = yield* Ref.get(configMapRef); + const targets = yield* Effect.filter(Object.entries(configMap), ([rawInstanceId, entry]) => + (instanceId === undefined || rawInstanceId === instanceId) && + secretReferencesOf(entry).length > 0 + ? isEntryEnabled(driversById, entry) + : Effect.succeed(false), + ); + if (targets.length === 0) return; + const resolver = yield* SecretResolver.SecretResolver; + const references = targets.flatMap(([, entry]) => secretReferencesOf(entry)); + yield* resolver.invalidate(references); + yield* resolver.read(references); + const live = yield* Ref.get(entries); + const rebuild = new Set(); + for (const [rawInstanceId, entry] of targets) { + const current = live.get(ProviderInstanceId.make(rawInstanceId)); + if (current === undefined) continue; + const { environment, unreadable } = yield* readSecretEnvironment(entry.environment ?? []); + // A failed re-read keeps the working instance and the value it already holds. + if (unreadable.length === 0 && !Equal.equals(environment, current.environment)) { + rebuild.add(current.instance.instanceId); + } + } + yield* reconcileWithR(configMap, rebuild); + }, + (effect) => semaphore.withPermits(1)(effect), + Effect.provideContext(driverContext), + ); + + // Reading secrets can wait on a vault unlock, so the server does not wait for it to start. + yield* semaphore.withPermits(1)( + Ref.set(configMapRef, input.configMap).pipe( + Effect.andThen(reconcileWithR(input.configMap, new Set(), true)), + Effect.provideContext(driverContext), + ), + ); + const startupReferences = yield* Effect.filter(Object.entries(input.configMap), ([, entry]) => + secretReferencesOf(entry).length > 0 + ? isEntryEnabled(driversById, entry) + : Effect.succeed(false), + ).pipe(Effect.map((targets) => targets.flatMap(([, entry]) => secretReferencesOf(entry)))); + if (startupReferences.length > 0) { + // The read waits outside the lock so other mutations are not held for a vault prompt; a + // configuration saved meanwhile has already reconciled itself. + yield* Effect.gen(function* () { + yield* (yield* SecretResolver.SecretResolver).read(startupReferences); + yield* semaphore.withPermits(1)( + Effect.gen(function* () { + const current = yield* Ref.get(configMapRef); + if (current === input.configMap) yield* reconcileWithR(current); + }), + ); + }).pipe(Effect.provideContext(driverContext), Effect.forkScoped); + } const registry: ProviderInstanceRegistry["Service"] = { getInstance: (id) => Ref.get(entries).pipe(Effect.map((map) => map.get(id)?.instance)), @@ -307,6 +481,7 @@ export const makeProviderInstanceRegistry = Effect.fn( const mutator: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape = { reconcile, + reloadSecrets, }; return { registry, mutator }; @@ -318,7 +493,7 @@ export const layer = (input: { }): Layer.Layer< ProviderInstanceRegistry | ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator, never, - R + R | SecretResolver.SecretResolver > => Layer.effectContext( makeProviderInstanceRegistry(input).pipe( @@ -331,5 +506,5 @@ export const layer = (input: { ) as Layer.Layer< ProviderInstanceRegistry | ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator, never, - R + R | SecretResolver.SecretResolver >; diff --git a/apps/server/src/provider/ProviderInstanceRegistryHydration.ts b/apps/server/src/provider/ProviderInstanceRegistryHydration.ts index cb778f1638a22..c8f2fb9b33e05 100644 --- a/apps/server/src/provider/ProviderInstanceRegistryHydration.ts +++ b/apps/server/src/provider/ProviderInstanceRegistryHydration.ts @@ -9,6 +9,7 @@ import { ServerSettings, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import type * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -16,6 +17,7 @@ import * as Path from "effect/Path"; import * as Stream from "effect/Stream"; import * as Settings from "../serverSettings.ts"; +import * as SecretResolver from "../secrets/SecretResolver.ts"; import { BUILT_IN_DRIVERS, type BuiltInDriversEnv } from "./builtInDrivers.ts"; import * as ProviderInstanceRegistry from "./ProviderInstanceRegistry.ts"; import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMutator.ts"; @@ -39,6 +41,8 @@ type ProviderInstanceRegistryHydrationEnv = | ServerConfig.ServerConfig | Path.Path | FileSystem.FileSystem + // Secret provider reads. + | Crypto.Crypto // The Claude runner counts its CLIs as running agents for port discovery. | PortScanner.PortDiscovery; @@ -141,7 +145,8 @@ const layerSettingsWatcher = (settingsChanges: Stream.Stream) => ); export const layer: Layer.Layer< - ProviderInstanceRegistry.ProviderInstanceRegistry, + | ProviderInstanceRegistry.ProviderInstanceRegistry + | ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator, never, ProviderInstanceRegistryHydrationEnv > = Layer.unwrap( @@ -167,12 +172,14 @@ export const layer: Layer.Layer< Layer.provide(ProviderOrchestrationAdapterInfrastructure.layer), Layer.provide(AcpRegistrySupport.layerFromHost), Layer.provide(ProviderHostLive.layer), + Layer.provide(SecretResolver.layer), ); return layerSettingsWatcher(settingsChanges).pipe(Layer.provideMerge(layerMutable)); }), ) as Layer.Layer< - ProviderInstanceRegistry.ProviderInstanceRegistry, + | ProviderInstanceRegistry.ProviderInstanceRegistry + | ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator, never, ProviderInstanceRegistryHydrationEnv >; diff --git a/apps/server/src/provider/ProviderInstanceRegistryMutator.ts b/apps/server/src/provider/ProviderInstanceRegistryMutator.ts index 51322826230d8..35eb52d3f2906 100644 --- a/apps/server/src/provider/ProviderInstanceRegistryMutator.ts +++ b/apps/server/src/provider/ProviderInstanceRegistryMutator.ts @@ -1,10 +1,12 @@ /** Reconcile closes removed/replaced scopes before creating replacements: at most one live instance per id. */ -import type { ProviderInstanceConfigMap } from "@t3tools/contracts"; +import type { ProviderInstanceConfigMap, ProviderInstanceId } from "@t3tools/contracts"; import * as Context from "effect/Context"; import type * as Effect from "effect/Effect"; export interface ProviderInstanceRegistryMutatorShape { readonly reconcile: (configMap: ProviderInstanceConfigMap) => Effect.Effect; + /** Re-reads the secrets of every instance, or one, and rebuilds those whose secrets changed. */ + readonly reloadSecrets: (instanceId?: ProviderInstanceId) => Effect.Effect; } export class ProviderInstanceRegistryMutator extends Context.Service< diff --git a/apps/server/src/provider/ProviderRegistry.test.ts b/apps/server/src/provider/ProviderRegistry.test.ts index 14b8dfb2ca150..5e356c84516a8 100644 --- a/apps/server/src/provider/ProviderRegistry.test.ts +++ b/apps/server/src/provider/ProviderRegistry.test.ts @@ -65,6 +65,7 @@ import type { ProviderWorkspaceSnapshot, } from "@t3tools/provider-core/server/driver"; import * as ProviderInstanceRegistry from "./ProviderInstanceRegistry.ts"; +import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMutator.ts"; import * as ProviderRegistry from "./ProviderRegistry.ts"; import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import { makeManualOnlyProviderMaintenanceCapabilities } from "@t3tools/provider-core/server/maintenanceResolver"; @@ -100,6 +101,10 @@ const withBundledCompatibility = (snapshot: ServerProvider) => ModelManifest.BUNDLED_MODEL_MANIFEST.compatibility, ); +const layerUnusedMutator = Layer.mock( + ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator, +)({}); + const layerTestHttpClient = Layer.succeed( HttpClient.HttpClient, HttpClient.make((request) => @@ -944,7 +949,12 @@ it.layer( retainedModels, ); }).pipe( - Effect.provide(ProviderRegistry.layer.pipe(Layer.provide(layerInstanceRegistry))), + Effect.provide( + ProviderRegistry.layer.pipe( + Layer.provide(layerInstanceRegistry), + Layer.provide(layerUnusedMutator), + ), + ), Effect.scoped, ); } @@ -1075,7 +1085,12 @@ it.layer( retainedModels, ); }).pipe( - Effect.provide(ProviderRegistry.layer.pipe(Layer.provide(layerInstanceRegistry))), + Effect.provide( + ProviderRegistry.layer.pipe( + Layer.provide(layerInstanceRegistry), + Layer.provide(layerUnusedMutator), + ), + ), Effect.scoped, ); } @@ -1347,6 +1362,7 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(layerInstanceRegistry), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -1442,6 +1458,7 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(layerInstanceRegistry), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -1562,6 +1579,7 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(instanceRegistryLayer), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -1668,6 +1686,7 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(layerInstanceRegistry), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -1796,6 +1815,7 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(layerInstanceRegistry), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -1899,6 +1919,7 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(layerInstanceRegistry), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -2070,7 +2091,7 @@ it.layer( enabled: false, config: { arbitrary: "payload" }, }, - } as unknown as ContractServerSettings["providerInstances"], + } as unknown as (typeof ServerSettings.Encoded)["providerInstances"], }), ), ); diff --git a/apps/server/src/provider/ProviderRegistry.ts b/apps/server/src/provider/ProviderRegistry.ts index 78b11306c5b6d..efa3385d1927e 100644 --- a/apps/server/src/provider/ProviderRegistry.ts +++ b/apps/server/src/provider/ProviderRegistry.ts @@ -19,6 +19,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; +import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as Semaphore from "effect/Semaphore"; @@ -27,6 +28,7 @@ import * as ModelManifest from "./ModelManifest.ts"; import { applyProviderCompatibility } from "./providerCompatibility.ts"; import * as ServerConfig from "../config.ts"; import * as ProviderInstanceRegistry from "./ProviderInstanceRegistry.ts"; +import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMutator.ts"; import { hydrateCachedProvider, isCachedProviderCorrelated, @@ -61,6 +63,11 @@ export class ProviderRegistry extends Context.Service< instanceId: ProviderInstanceId, ) => Effect.Effect>; + /** Re-reads secret-backed environments first, so a fixed secret applies without a restart. */ + readonly reloadSecretsAndRefresh: ( + instanceId?: ProviderInstanceId, + ) => Effect.Effect>; + /** Only refreshed instances can verify an update; the published list still includes every row. */ readonly refreshAfterUpdate: ( provider: ProviderDriverKind, @@ -330,6 +337,8 @@ export const layer = Layer.effect( ProviderRegistry, Effect.gen(function* () { const instanceRegistry = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; + const instanceRegistryMutator = + yield* ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator; const manifestService = yield* ModelManifest.ModelManifest; const latestVersions = yield* ProviderLatestVersions.ProviderLatestVersions; const serviceScope = yield* Effect.scope; @@ -614,7 +623,9 @@ export const layer = Layer.effect( ); if (claimed.owner) { // Detach shared passes so the first caller's interruption cannot cancel other callers' work. - yield* getLiveSources.pipe( + // Syncing first keeps a refresh right after a rebuild from probing the replaced instance. + yield* syncLiveSourcesAndContinue.pipe( + Effect.andThen(getLiveSources), Effect.flatMap((sources) => Effect.forEach(sources, (source) => refreshOneSource(source), { concurrency: "unbounded", @@ -651,6 +662,7 @@ export const layer = Layer.effect( const refreshInstance = Effect.fn("refreshInstance")(function* ( instanceId: ProviderInstanceId, ) { + yield* syncLiveSourcesAndContinue; const sources = yield* getLiveSources; const providerSource = sources.find((candidate) => candidate.instanceId === instanceId); if (!providerSource) { @@ -659,6 +671,13 @@ export const layer = Layer.effect( return yield* refreshOneSource(providerSource); }); + const reloadSecretsAndRefresh = Effect.fn("reloadSecretsAndRefresh")(function* ( + instanceId?: ProviderInstanceId, + ) { + yield* instanceRegistryMutator.reloadSecrets(instanceId); + return yield* instanceId === undefined ? refresh() : refreshInstance(instanceId); + }); + const refreshAfterUpdate = Effect.fn("refreshAfterUpdate")(function* ( provider: ProviderDriverKind, selectedInstanceId: ProviderInstanceId, @@ -858,6 +877,7 @@ export const layer = Layer.effect( }, ); }), + Scope.provide(serviceScope), ); yield* upsertProviders(fallbackProviders, { publish: false }); @@ -1008,6 +1028,8 @@ export const layer = Layer.effect( refresh(provider).pipe(Effect.catchCause(recoverRefreshFailure)), refreshInstance: (instanceId: ProviderInstanceId) => refreshInstance(instanceId).pipe(Effect.catchCause(recoverRefreshFailure)), + reloadSecretsAndRefresh: (instanceId?: ProviderInstanceId) => + reloadSecretsAndRefresh(instanceId).pipe(Effect.catchCause(recoverRefreshFailure)), refreshWorkspaceSnapshot: (input) => refreshWorkspaceSnapshot(input).pipe(Effect.catchCause(recoverRefreshFailure)), refreshAfterUpdate, diff --git a/apps/server/src/provider/providerCompatibility.test.ts b/apps/server/src/provider/providerCompatibility.test.ts index 12c541fbd776d..8575a73230f85 100644 --- a/apps/server/src/provider/providerCompatibility.test.ts +++ b/apps/server/src/provider/providerCompatibility.test.ts @@ -13,6 +13,7 @@ import * as ModelManifest from "./ModelManifest.ts"; import * as ProviderRegistry from "./ProviderRegistry.ts"; import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions"; import * as ProviderInstanceRegistry from "./ProviderInstanceRegistry.ts"; +import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMutator.ts"; import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; import { makeManualOnlyProviderMaintenanceCapabilities } from "@t3tools/provider-core/server/maintenanceResolver"; import * as Schema from "effect/Schema"; @@ -312,6 +313,15 @@ it.effect("a remote policy refresh preserves a newer health result on the regist assert.strictEqual(updated?.compatibilityAdvisory?.status, "supported"); assert.strictEqual(updated?.status, "error"); assert.strictEqual(updated?.message, "Authentication failed"); - }).pipe(Effect.provide(ProviderRegistry.layer.pipe(Layer.provide(layerDependencies)))); + }).pipe( + Effect.provide( + ProviderRegistry.layer.pipe( + Layer.provide(layerDependencies), + Layer.provide( + Layer.mock(ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator)({}), + ), + ), + ), + ); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 75c592b7811a1..943928c4e18cf 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -192,6 +192,7 @@ function makeRegistry( getProviders: Ref.get(providersRef), refresh: () => Ref.get(providersRef), refreshInstance: () => Ref.get(providersRef), + reloadSecretsAndRefresh: () => Ref.get(providersRef), refreshAfterUpdate: () => Ref.get(providersRef), refreshWorkspaceSnapshot: () => Ref.get(providersRef), getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => diff --git a/apps/server/src/provider/testUtils/providerRegistryMock.ts b/apps/server/src/provider/testUtils/providerRegistryMock.ts index 358d49afe45c2..a4008acc18efb 100644 --- a/apps/server/src/provider/testUtils/providerRegistryMock.ts +++ b/apps/server/src/provider/testUtils/providerRegistryMock.ts @@ -11,6 +11,7 @@ const makeProviderRegistryMock = ( getProviders: Effect.succeed(providers), refresh: () => Effect.succeed(providers), refreshInstance: () => Effect.succeed(providers), + reloadSecretsAndRefresh: () => Effect.succeed(providers), refreshAfterUpdate: () => Effect.succeed(providers), refreshWorkspaceSnapshot: () => Effect.succeed(providers), getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => diff --git a/apps/server/src/pullRequest/PullRequestService.test.ts b/apps/server/src/pullRequest/PullRequestService.test.ts index 6c6c8215aa315..3c511b292afb8 100644 --- a/apps/server/src/pullRequest/PullRequestService.test.ts +++ b/apps/server/src/pullRequest/PullRequestService.test.ts @@ -2189,6 +2189,98 @@ it.effect("does not revive old summaries when project epochs are evicted", () => }), ); +it.effect("a refresh or a finished turn drops the provider's own change request reads", () => + Effect.gen(function* () { + const forgotten: Array<{ readonly repository: string; readonly number?: number } | undefined> = + []; + const service = yield* makeService({ + projects: [ + project({ + id: "p1", + title: "web", + workspaceRoot: "/a", + repository: "acme/web", + provider: "gitlab", + }), + ], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("gitlab"), { + forgetChangeRequestReads: (scope) => Effect.sync(() => void forgotten.push(scope)), + }), + ], + }); + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + + yield* service.invalidate({ reference }); + yield* service.refreshAfterTurn(reference.projectId); + yield* service.invalidate({}); + + assert.deepStrictEqual(forgotten, [ + { repository: "acme/web", number: 1 }, + undefined, + undefined, + ]); + }), +); + +it.effect("a read during a refresh cannot keep the provider's old change request", () => + Effect.gen(function* () { + let hostTitle = "old"; + let held: string | undefined; + let gate: + | { readonly entered: Deferred.Deferred; readonly release: Deferred.Deferred } + | undefined; + const service = yield* makeService({ + projects: [ + project({ + id: "p1", + title: "web", + workspaceRoot: "/a", + repository: "acme/web", + provider: "gitlab", + }), + ], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("gitlab"), { + getChangeRequest: () => + Effect.sync(() => { + held ??= hostTitle; + return { ...hostedChangeRequest("body"), title: held }; + }), + forgetChangeRequestReads: () => + Effect.gen(function* () { + if (gate !== undefined) { + yield* Deferred.succeed(gate.entered, undefined); + yield* Deferred.await(gate.release); + } + held = undefined; + }), + }), + ], + }); + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const refreshWithReadInBetween = (refresh: Effect.Effect) => + Effect.gen(function* () { + gate = { entered: yield* Deferred.make(), release: yield* Deferred.make() }; + const running = yield* refresh.pipe(Effect.forkChild()); + yield* Deferred.await(gate.entered); + yield* service.detail(reference); + yield* Deferred.succeed(gate.release, undefined); + yield* Fiber.join(running); + gate = undefined; + }); + + assert.strictEqual((yield* service.detail(reference)).title, "old"); + hostTitle = "after turn"; + yield* refreshWithReadInBetween(service.refreshAfterTurn(reference.projectId)); + assert.strictEqual((yield* service.detail(reference)).title, "after turn"); + + hostTitle = "after refresh"; + yield* refreshWithReadInBetween(service.invalidate({ reference })); + assert.strictEqual((yield* service.detail(reference)).title, "after refresh"); + }), +); + it.effect("close and reopen notify subscribed readers after invalidating their cached state", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/pullRequest/PullRequestService.ts b/apps/server/src/pullRequest/PullRequestService.ts index 854d182f155ec..9f18426be198e 100644 --- a/apps/server/src/pullRequest/PullRequestService.ts +++ b/apps/server/src/pullRequest/PullRequestService.ts @@ -528,6 +528,9 @@ function withRateLimitBackoff( listChangeRequestStats: wrap("listChangeRequestStats", api.listChangeRequestStats), }), getChangeRequest: wrap("getChangeRequest", api.getChangeRequest), + ...(api.forgetChangeRequestReads === undefined + ? {} + : { forgetChangeRequestReads: api.forgetChangeRequestReads }), ...(api.getChangeRequestPreview === undefined ? {} : { getChangeRequestPreview: wrap("getChangeRequestPreview", api.getChangeRequestPreview) }), @@ -3423,6 +3426,13 @@ export const make = Effect.gen(function* () { return { stats: [...held, ...result.stats] }; }); + const forgetProviderReads = (scope?: { readonly repository: string; readonly number?: number }) => + Effect.forEach( + registry.kinds, + (kind) => registry.get(kind)?.forgetChangeRequestReads?.(scope) ?? Effect.void, + { discard: true }, + ); + const invalidate: PullRequestService["Service"]["invalidate"] = Effect.fn( "PullRequestService.invalidate", )(function* (input, options) { @@ -3438,13 +3448,15 @@ export const make = Effect.gen(function* () { if (reference !== undefined) { yield* canonicalRef(reference).pipe( Effect.flatMap((ref) => - readCache - .invalidate(refScope(ref)) - .pipe(Effect.andThen(Effect.sync(() => bumpRefEpoch(ref)))), + forgetProviderReads({ repository: ref.repository, number: ref.number }).pipe( + Effect.andThen(readCache.invalidate(refScope(ref))), + Effect.andThen(Effect.sync(() => bumpRefEpoch(ref))), + ), ), Effect.ignore, ); } else { + yield* forgetProviderReads(); listingsEpoch = ++epochCounter; everyFileRevisionEpoch = ++epochCounter; viewersByHost.clear(); @@ -3455,22 +3467,27 @@ export const make = Effect.gen(function* () { } }); - const refreshAfterTurn: PullRequestService["Service"]["refreshAfterTurn"] = (projectId) => - Effect.suspend(() => { - listingsEpoch = ++epochCounter; - projectEpochs.delete(projectId); - if (projectEpochs.size >= REF_EPOCH_CAPACITY) { - const oldest = projectEpochs.keys().next().value; - if (oldest !== undefined) { - projectEpochFloor = projectEpochs.get(oldest)!; - projectEpochs.delete(oldest); - } + const refreshAfterTurn: PullRequestService["Service"]["refreshAfterTurn"] = Effect.fn( + "PullRequestService.refreshAfterTurn", + )(function* (projectId) { + // Everything the provider holds, not just the project's repository: a turn can touch a + // change request reached through another repository, and no epoch may move until those + // reads are gone, or a reader in between keeps the old one under the new epoch. + yield* forgetProviderReads(); + listingsEpoch = ++epochCounter; + projectEpochs.delete(projectId); + if (projectEpochs.size >= REF_EPOCH_CAPACITY) { + const oldest = projectEpochs.keys().next().value; + if (oldest !== undefined) { + projectEpochFloor = projectEpochs.get(oldest)!; + projectEpochs.delete(oldest); } - projectEpochs.set(projectId, listingsEpoch); - return readCache - .invalidate(`project:${projectId}`) - .pipe(Effect.andThen(SubscriptionRef.set(pullRequestRefreshes, listingsEpoch))); - }); + } + projectEpochs.set(projectId, listingsEpoch); + const epoch = listingsEpoch; + yield* readCache.invalidate(`project:${projectId}`); + yield* SubscriptionRef.set(pullRequestRefreshes, epoch); + }); const invalidatedByMutation = ( diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 48c4a5c240ea9..40c741fdd68ac 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -219,6 +219,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( settleThread: unused, interruptThread: unused, stopWorkflow: () => Effect.die("unused stopWorkflow"), + forceStopThread: () => Effect.die("unused forceStopThread"), awaitBackgroundTasks: unused, stopDelegatedTasks: unused, getThreadEventSequence: unused, diff --git a/apps/server/src/secrets/SecretResolver.ts b/apps/server/src/secrets/SecretResolver.ts new file mode 100644 index 0000000000000..076e4a0780793 --- /dev/null +++ b/apps/server/src/secrets/SecretResolver.ts @@ -0,0 +1,20 @@ +import * as SecretResolverCore from "@t3tools/secret-provider-core/server/SecretResolver"; +import type { ServerSettings } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { BUILT_IN_SECRET_PROVIDER_DRIVER_LIST } from "./secretProviderDefinitions.ts"; +import * as Settings from "../serverSettings.ts"; + +export const layer = Layer.unwrap( + Effect.gen(function* () { + const serverSettings = yield* Settings.ServerSettingsService; + return SecretResolverCore.layer({ + drivers: BUILT_IN_SECRET_PROVIDER_DRIVER_LIST, + secretProviders: serverSettings.getSettings.pipe( + Effect.map((settings) => settings.secretProviders), + Effect.orElseSucceed((): ServerSettings["secretProviders"] => ({})), + ), + }); + }), +); diff --git a/apps/server/src/secrets/secretProviderDefinitions.test.ts b/apps/server/src/secrets/secretProviderDefinitions.test.ts new file mode 100644 index 0000000000000..b958a480693e1 --- /dev/null +++ b/apps/server/src/secrets/secretProviderDefinitions.test.ts @@ -0,0 +1,12 @@ +import { expect, it } from "@effect/vitest"; + +import { + BUILT_IN_SECRET_PROVIDER_DRIVER_LIST, + secretProviderDefinitions, +} from "./secretProviderDefinitions.ts"; + +it("runs a driver for every secret provider the server and clients read, in their order", () => { + expect(BUILT_IN_SECRET_PROVIDER_DRIVER_LIST.map((driver) => driver.definition)).toEqual( + secretProviderDefinitions.definitions, + ); +}); diff --git a/apps/server/src/secrets/secretProviderDefinitions.ts b/apps/server/src/secrets/secretProviderDefinitions.ts new file mode 100644 index 0000000000000..409df40bf16cf --- /dev/null +++ b/apps/server/src/secrets/secretProviderDefinitions.ts @@ -0,0 +1,7 @@ +import { builtInSecretProviderDefinitions } from "@t3tools/secret-provider-definitions/registry"; +import * as OnePasswordDriver from "@t3tools/secret-provider-onepassword/server/driver"; + +/** In the shared registry's order. */ +export const BUILT_IN_SECRET_PROVIDER_DRIVER_LIST = [OnePasswordDriver.driver] as const; + +export const secretProviderDefinitions = builtInSecretProviderDefinitions; diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index c9d4bcf51ad46..8bf84cd6adc96 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -7,7 +7,9 @@ import { ProviderDriverKind, ProviderInstanceId, resolveProviderInstanceEnabled, + SecretProviderId, ServerSettings, + ServerSettingsPatch, UsageLimitSourceId, } from "@t3tools/contracts"; import { createModelSelection } from "@t3tools/shared/model"; @@ -33,6 +35,7 @@ import { resolveProviderInstanceTerminalEnvironment } from "./terminal/Manager.t const decodeServerSettings = Schema.decodeUnknownEffect(ServerSettings); const decodeServerSettingsJson = Schema.decodeUnknownEffect(Schema.fromJsonString(ServerSettings)); +const decodeServerSettingsPatch = Schema.decodeUnknownEffect(ServerSettingsPatch); const layerServerSettings = () => ServerSettingsModule.layer.pipe( @@ -278,6 +281,25 @@ it.layer(NodeServices.layer)("server settings", (it) => { }), ); + it.effect("stores no secret provider of an unloaded kind or with a config its kind rejects", () => + Effect.gen(function* () { + const service = yield* ServerSettingsModule.ServerSettingsService; + const write = (entry: unknown) => + decodeServerSettingsPatch({ secretProviders: { personal: entry } }).pipe( + Effect.flatMap(service.updateSettings), + Effect.flip, + ); + for (const entry of [ + { kind: "one_password", label: "Personal", config: { account: "--help" } }, + { kind: "one_password", label: "Personal" }, + { kind: "openbao", label: "Vault", config: { address: "https://x" } }, + ]) { + assert.deepInclude(yield* write(entry), { operation: "write-secret-provider" }); + } + assert.deepStrictEqual((yield* service.getSettings).secretProviders, {}); + }).pipe(Effect.provide(layerServerSettings())), + ); + it.effect("saves through a symlinked settings file without replacing the link", () => Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; @@ -1402,6 +1424,27 @@ it.layer(NodeServices.layer)("server settings", (it) => { for (const name of secretNames) { assert.isTrue(Option.isSome(yield* store.get(name)), name); } + const secret = { + provider: SecretProviderId.make("personal"), + reference: "op://Private/Antigravity/token", + }; + const referenced = yield* service.updateProviderInstance({ + operation: "upsert", + instanceId, + instance: { + ...client.providerInstances[instanceId]!, + environment: [ + { name: "API_TOKEN", value: secret, sensitive: true, valueRedacted: true }, + ], + }, + }); + assert.isTrue(Option.isNone(yield* store.get(secretNames[1]!))); + assert.deepEqual( + ServerSettingsModule.redactServerSettingsForClient(referenced).providerInstances[ + instanceId + ]?.environment, + [{ name: "API_TOKEN", value: secret, sensitive: true }], + ); yield* service.updateSettings({ providerInstances: {}, sourceControlHosts: { diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 48839bb8535d9..67a1c3c3414f3 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -8,7 +8,6 @@ import { type ProviderInstanceConfig, type ProviderInstanceMutation, defaultInstanceIdForDriver, - isKnownSecretProviderKind, isProviderDriverKind, ProviderDriverKind, ProviderInstanceId, @@ -53,6 +52,7 @@ import { isModelSelectionProviderEnabled, } from "@t3tools/shared/serverSettings"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; +import { secretProviderDefinitions } from "./secrets/secretProviderDefinitions.ts"; import { readSecretFieldValues, sourceControlHostSecretName, @@ -144,7 +144,7 @@ function withoutUnknownSecretProviderConfigs( ): ServerSettings["secretProviders"] { return Object.fromEntries( Object.entries(settings.secretProviders).map(([id, entry]) => { - if (isKnownSecretProviderKind(entry.kind)) return [id, entry]; + if (secretProviderConfigGuards.has(entry.kind)) return [id, entry]; const { config: _config, ...envelope } = entry; return [id, envelope]; }), @@ -164,6 +164,40 @@ export function applyProviderInstanceMutation( return { ...settings, providerInstances }; } +const secretProviderConfigGuards = new Map( + secretProviderDefinitions.definitions.map((definition) => [ + definition.kind as string, + Schema.is(definition.configSchema), + ]), +); + +// Rejects a kind this server has not loaded rather than storing a config it cannot check. +function ensureSecretProviderWritesAllowed( + patch: ServerSettingsPatch, + settingsPath: string, +): Effect.Effect { + for (const [id, entry] of Object.entries(patch.secretProviders ?? {})) { + if (entry === null) continue; + const isConfig = secretProviderConfigGuards.get(entry.kind); + const problem = + isConfig === undefined + ? `Unsupported secret provider kind: ${entry.kind}.` + : isConfig(entry.config) + ? undefined + : `Invalid ${entry.kind} config for secret provider ${id}.`; + if (problem !== undefined) { + return Effect.fail( + new ServerSettingsError({ + settingsPath, + operation: "write-secret-provider", + cause: new Error(problem), + }), + ); + } + } + return Effect.void; +} + function ensureProviderInstanceMutationAllowed( settings: ServerSettings, mutation: ProviderInstanceMutation, @@ -1185,11 +1219,14 @@ const make = Effect.gen(function* () { getSettings: getMaterializedSettings, updateSettings: (patch) => updateAndPersistSettings((current) => - Effect.succeed(applyServerSettingsPatch(current, patch)), + ensureSecretProviderWritesAllowed(patch, settingsPath).pipe( + Effect.as(applyServerSettingsPatch(current, patch)), + ), ), updateProviderInstance: (mutation, patch = {}) => updateAndPersistSettings((current) => Effect.gen(function* () { + yield* ensureSecretProviderWritesAllowed(patch, settingsPath); yield* ensureProviderInstanceMutationAllowed(current, mutation, settingsPath); const patched = applyServerSettingsPatch(current, patch); return applyProviderInstanceMutation(patched, mutation); diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts index cc2f5266aa59e..77a8fd59fd857 100644 --- a/apps/server/src/settingsSecrets.ts +++ b/apps/server/src/settingsSecrets.ts @@ -101,16 +101,19 @@ export function mapSettingsSecrets( ...(instance.environment ? { environment: instance.environment.map((variable) => { + const { valueRedacted: _omit, ...rest } = variable; + // A secret reference is not itself secret; storage sees the value cleared. + const reference = typeof variable.value === "string" ? undefined : variable.value; const result = map({ secretName: `provider-env-${encode(instanceId)}-${encode(variable.name)}`, kind: "environment", - value: variable.value, - redacted: variable.valueRedacted === true, + value: typeof variable.value === "string" ? variable.value : "", + redacted: reference === undefined && variable.valueRedacted === true, sensitive: variable.sensitive, providerInstanceId: instanceId, environmentVariable: variable.name, }); - const { valueRedacted: _omit, ...rest } = variable; + if (reference !== undefined) return { ...rest, value: reference }; return { ...rest, value: result.value, diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index 5b166d8b1452b..7e92567772950 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -10,20 +10,25 @@ import { TerminalProviderInstanceNotFoundError, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Base64Url from "effect/encoding/Base64Url"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; import * as Path from "effect/Path"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SynchronizedRef from "effect/SynchronizedRef"; +import * as TestClock from "effect/testing/TestClock"; import { expect, vi } from "vite-plus/test"; import * as ProcessRunner from "../processRunner.ts"; @@ -756,6 +761,154 @@ it.layer( }), ); + // Answers process-table queries after `queryDuration`, honoring the caller's + // timeout the way ProcessRunner's `timedOutResult` mode does. + const slowProcessTable = ( + platform: "linux" | "win32", + queryDuration: Duration.Input, + queries: Array<{ start: number; end?: number }>, + started: ReadonlyArray>, + ): ProcessRunner.ProcessRunner["Service"] => ({ + run: (input) => + Effect.gen(function* () { + const query: { start: number; end?: number } = { start: yield* Clock.currentTimeMillis }; + queries.push(query); + const deferred = started[queries.length - 1]; + if (deferred) yield* Deferred.succeed(deferred, undefined); + const completed = yield* Effect.sleep(queryDuration).pipe( + Effect.timeoutOption(input.timeout ?? "1 hour"), + ); + query.end = yield* Clock.currentTimeMillis; + return { + stdout: Option.isNone(completed) + ? "" + : platform === "win32" + ? "100|9000|node.exe\n" + : " 100 9000 node\n", + stderr: "", + code: Option.isSome(completed) ? ChildProcessSpawner.ExitCode(0) : null, + timedOut: Option.isNone(completed), + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }; + }), + }); + + const createFallbackManager = ( + platform: "linux" | "win32", + runner: ProcessRunner.ProcessRunner["Service"], + ) => + createManager(5, { subprocessPollIntervalMs: 1_000, processKillGraceMs: 0 }).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.provide(layerWithHostPlatform(platform)), + ); + + it.effect.each(["linux", "win32"] as const)( + "reports activity from a slow but successful %s process-table query", + (platform) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const queries: Array<{ start: number; end?: number }> = []; + const { manager, getEvents } = yield* createFallbackManager( + platform, + slowProcessTable(platform, "5 seconds", queries, [started]), + ); + + yield* manager.open(openInput()); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(started); + yield* TestClock.adjust("5 seconds"); + + expect( + (yield* getEvents).some( + (event) => + event.type === "activity" && event.hasRunningSubprocess && event.label === "node", + ), + ).toBe(true); + yield* manager.close({ threadId: "thread-1" }); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("idles at least as long as a slow process-table query took before the next", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const queries: Array<{ start: number; end?: number }> = []; + const { manager } = yield* createFallbackManager( + "linux", + slowProcessTable("linux", "5 seconds", queries, [started]), + ); + + yield* manager.open(openInput()); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(started); + yield* TestClock.adjust("5 seconds"); + yield* TestClock.adjust(Duration.millis(19_999)); + + expect(queries).toHaveLength(1); + yield* TestClock.adjust("1 millis"); + const [slow, next] = queries; + expect(next?.start).toBe(slow!.end! + 20_000); + yield* manager.close({ threadId: "thread-1" }); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("ends a hung process-table query and paces the retry", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const queries: Array<{ start: number; end?: number }> = []; + const { manager } = yield* createFallbackManager( + "win32", + slowProcessTable("win32", "1 day", queries, [started]), + ); + + yield* manager.open(openInput()); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(started); + yield* TestClock.adjust("2 minutes"); + yield* TestClock.adjust("2 minutes"); + + const [hung, next] = queries; + expect(hung!.end! - hung!.start).toBe(15_000); + expect(next!.start - hung!.end!).toBe(60_000); + yield* manager.close({ threadId: "thread-1" }); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect.each([ + ["linux", 1_000], + ["win32", 1_500], + ] as const)( + "bounds closeIdle's %s query so other operations on the thread still run", + ([platform, budgetMs]) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const queries: Array<{ start: number; end?: number }> = []; + const { manager, ptyAdapter } = yield* createFallbackManager( + platform, + slowProcessTable(platform, "1 day", queries, [started]), + ); + yield* manager.open(openInput()); + + const closing = yield* manager + .closeIdle({ threadId: "thread-1" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(started); + const resizing = yield* manager + .resize({ threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID, cols: 80, rows: 30 }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(Duration.millis(budgetMs)); + + expect(closing.pollUnsafe()).toBeDefined(); + expect(resizing.pollUnsafe()).toBeDefined(); + expect(queries[0]!.end! - queries[0]!.start).toBe(budgetMs); + expect(ptyAdapter.processes[0]!.resizeCalls).toEqual([{ cols: 80, rows: 30 }]); + expect(ptyAdapter.processes[0]!.killed).toBe(false); + yield* manager.close({ threadId: "thread-1" }); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("keeps terminals that get input or output while closeIdle checks them", () => Effect.gen(function* () { const ptyAdapter = new FakePtyAdapter(); diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index a08b3e4afe784..bd7ca2c92e3c2 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -46,6 +46,7 @@ import { getTerminalLabel } from "@t3tools/shared/terminalLabels"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Base64Url from "effect/encoding/Base64Url"; import * as Equal from "effect/Equal"; @@ -55,6 +56,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -95,6 +97,12 @@ const MAX_HISTORY_CHUNK_LENGTH = 16 * 1024; const DEFAULT_PERSIST_DEBOUNCE_MS = 250; const DEFAULT_SUBPROCESS_POLL_INTERVAL_MS = 1_000; const MAX_SUBPROCESS_POLL_INTERVAL_MS = 60_000; +// `ps` and WMI can take seconds on a loaded host and their tables are still +// valid then; the timeout only ends a query that is truly stuck. +const PROCESS_TABLE_QUERY_TIMEOUT = "15 seconds"; +// closeIdle queries while holding the thread lock, so it keeps a short budget +// rather than stalling other operations on that thread behind a slow table. +const CLOSE_IDLE_QUERY_TIMEOUT = { posix: "1 second", win32: "1500 millis" } as const; // Discover children started without terminal input or output. const QUIET_SUBPROCESS_SNAPSHOT_EVERY_TICKS = 30; const DEFAULT_PROCESS_KILL_GRACE_MS = 1_000; @@ -675,8 +683,17 @@ interface TerminalProcessTableSnapshot { readonly commandById: ReadonlyMap; } -function subprocessSnapshotPollDelayMs(pollIntervalMs: number, failureCount: number): number { - return Math.min(pollIntervalMs * 2 ** failureCount, MAX_SUBPROCESS_POLL_INTERVAL_MS); +// Idles at least four times as long as the last fallback query took, so a slow +// process table never keeps the poll loop busy most of the time. +function subprocessSnapshotPollDelayMs( + pollIntervalMs: number, + failureCount: number, + queryElapsedMs: number, +): number { + return Math.min( + Math.max(pollIntervalMs * 2 ** failureCount, queryElapsedMs * 4), + MAX_SUBPROCESS_POLL_INTERVAL_MS, + ); } function parsePosixProcessTable(stdout: string): TerminalProcessTableSnapshot { @@ -764,6 +781,7 @@ const resolvePosixPsCommand = Effect.fn("terminal.resolvePosixPsCommand")(functi const posixProcessTableSnapshot = Effect.fn("terminal.posixProcessTableSnapshot")(function* ( psCommand: string, + timeout: Duration.Input, ): Effect.fn.Return< TerminalProcessTableSnapshot, TerminalSubprocessCheckError, @@ -774,7 +792,7 @@ const posixProcessTableSnapshot = Effect.fn("terminal.posixProcessTableSnapshot" .run({ command: psCommand, args: ["-eo", "pid=,ppid=,comm="], - timeout: "1 second", + timeout, maxOutputBytes: 524_288, outputMode: "truncate", timeoutBehavior: "timedOutResult", @@ -800,48 +818,46 @@ const posixProcessTableSnapshot = Effect.fn("terminal.posixProcessTableSnapshot" return parsePosixProcessTable(result.stdout); }); -const windowsProcessTableSnapshot = Effect.fn("terminal.windowsProcessTableSnapshot")( - function* (): Effect.fn.Return< - TerminalProcessTableSnapshot, - TerminalSubprocessCheckError, - ProcessRunner.ProcessRunner - > { - const processRunner = yield* ProcessRunner.ProcessRunner; - const command = - 'Get-CimInstance Win32_Process -ErrorAction Stop | ForEach-Object { Write-Output "$($_.ProcessId)|$($_.ParentProcessId)|$($_.Name)" }'; - const result = yield* processRunner - .run({ - command: "powershell.exe", - args: ["-NoProfile", "-NonInteractive", "-Command", command], - timeout: "1500 millis", - maxOutputBytes: 262_144, - outputMode: "truncate", - timeoutBehavior: "timedOutResult", - }) - .pipe( - Effect.mapError( - (cause) => new TerminalSubprocessCheckError({ cause, command: "powershell" }), - ), - ); - if (result.code !== 0 || result.timedOut || result.stdoutTruncated) { - return yield* new TerminalSubprocessCheckError({ - command: "powershell", - exitCode: result.code, - timedOut: result.timedOut, - stdoutTruncated: result.stdoutTruncated, - }); - } - const processes = result.stdout.split(/\r?\n/g).flatMap((line) => { - const [pidRaw, ppidRaw, name = ""] = line.trim().split("|", 3); - const pid = Number(pidRaw); - const ppid = Number(ppidRaw); - return Number.isInteger(pid) && pid > 0 && Number.isInteger(ppid) - ? [{ pid, ppid, name }] - : []; +const windowsProcessTableSnapshot = Effect.fn("terminal.windowsProcessTableSnapshot")(function* ( + timeout: Duration.Input, +): Effect.fn.Return< + TerminalProcessTableSnapshot, + TerminalSubprocessCheckError, + ProcessRunner.ProcessRunner +> { + const processRunner = yield* ProcessRunner.ProcessRunner; + const command = + 'Get-CimInstance Win32_Process -ErrorAction Stop | ForEach-Object { Write-Output "$($_.ProcessId)|$($_.ParentProcessId)|$($_.Name)" }'; + const result = yield* processRunner + .run({ + command: "powershell.exe", + args: ["-NoProfile", "-NonInteractive", "-Command", command], + timeout, + maxOutputBytes: 262_144, + outputMode: "truncate", + timeoutBehavior: "timedOutResult", + }) + .pipe( + Effect.mapError( + (cause) => new TerminalSubprocessCheckError({ cause, command: "powershell" }), + ), + ); + if (result.code !== 0 || result.timedOut || result.stdoutTruncated) { + return yield* new TerminalSubprocessCheckError({ + command: "powershell", + exitCode: result.code, + timedOut: result.timedOut, + stdoutTruncated: result.stdoutTruncated, }); - return processTableSnapshotFromProcesses(processes); - }, -); + } + const processes = result.stdout.split(/\r?\n/g).flatMap((line) => { + const [pidRaw, ppidRaw, name = ""] = line.trim().split("|", 3); + const pid = Number(pidRaw); + const ppid = Number(ppidRaw); + return Number.isInteger(pid) && pid > 0 && Number.isInteger(ppid) ? [{ pid, ppid, name }] : []; + }); + return processTableSnapshotFromProcesses(processes); +}); interface TerminalHistoryChunk { data: string; @@ -1557,54 +1573,74 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func }, ); // Share one process table per poll; per-terminal scans can exhaust PIDs on busy hosts. - const fallbackProcessTableSnapshot = ( - platform === "win32" - ? windowsProcessTableSnapshot() - : posixProcessTableSnapshot(yield* resolvePosixPsCommand()) - ).pipe(Effect.provideService(ProcessRunner.ProcessRunner, processRunner)); - const fetchProcessTableSnapshot: Effect.Effect< - { - readonly snapshot: TerminalProcessTableSnapshot; - readonly snapshotSucceeded: boolean; - }, - TerminalSubprocessCheckError - > = options.processTable - ? options.processTable.pipe( - Effect.map((entries) => ({ - snapshot: processTableSnapshotFromProcesses(entries), - snapshotSucceeded: true, - })), - Effect.catch(() => - fallbackProcessTableSnapshot.pipe( - Effect.map((snapshot) => ({ snapshot, snapshotSucceeded: false })), + // Timed on success and failure so the poll loop can pace the next query. + const posixPsCommand = platform === "win32" ? undefined : yield* resolvePosixPsCommand(); + const fallbackProcessTableSnapshot = (timeout: Duration.Input) => + (posixPsCommand === undefined + ? windowsProcessTableSnapshot(timeout) + : posixProcessTableSnapshot(posixPsCommand, timeout) + ).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, processRunner), + Effect.result, + Effect.timed, + Effect.map(([elapsed, snapshot]) => ({ + snapshot, + queryElapsedMs: Duration.toMillis(elapsed), + })), + ); + // `fallbackTimeout` only bounds the ps/WMI fallback; a native process table + // or custom inspector is unaffected. + const fetchProcessTableSnapshot = ( + fallbackTimeout: Duration.Input, + ): Effect.Effect<{ + readonly snapshot: Result.Result; + readonly snapshotSucceeded: boolean; + readonly queryElapsedMs: number; + }> => + options.processTable + ? options.processTable.pipe( + Effect.map((entries) => ({ + snapshot: Result.succeed(processTableSnapshotFromProcesses(entries)), + snapshotSucceeded: true, + queryElapsedMs: 0, + })), + Effect.catch(() => + fallbackProcessTableSnapshot(fallbackTimeout).pipe( + Effect.map((fallback) => ({ ...fallback, snapshotSucceeded: false })), + ), ), - ), - ) - : fallbackProcessTableSnapshot.pipe( - Effect.map((snapshot) => ({ snapshot, snapshotSucceeded: true })), - ); + ) + : fallbackProcessTableSnapshot(fallbackTimeout).pipe( + Effect.map((fallback) => ({ + ...fallback, + snapshotSucceeded: Result.isSuccess(fallback.snapshot), + })), + ); const customSubprocessInspector = options.subprocessInspector; - const acquireSubprocessInspector: Effect.Effect< - { - readonly inspector: TerminalSubprocessInspector; - readonly snapshotSucceeded: boolean; - }, - TerminalSubprocessCheckError - > = + const acquireSubprocessInspector = ( + fallbackTimeout: Duration.Input, + ): Effect.Effect<{ + readonly inspector: Result.Result; + readonly snapshotSucceeded: boolean; + readonly queryElapsedMs: number; + }> => customSubprocessInspector !== undefined - ? Effect.succeed({ inspector: customSubprocessInspector, snapshotSucceeded: true }) + ? Effect.succeed({ + inspector: Result.succeed(customSubprocessInspector), + snapshotSucceeded: true, + queryElapsedMs: 0, + }) : Effect.map( - fetchProcessTableSnapshot, - ({ - snapshot, - snapshotSucceeded, - }): { - readonly inspector: TerminalSubprocessInspector; - readonly snapshotSucceeded: boolean; - } => ({ - inspector: (terminalPid) => - Effect.succeed(deriveSubprocessInspectResult(snapshot, terminalPid, platform)), + fetchProcessTableSnapshot(fallbackTimeout), + ({ snapshot, snapshotSucceeded, queryElapsedMs }) => ({ + inspector: Result.map( + snapshot, + (table): TerminalSubprocessInspector => + (terminalPid) => + Effect.succeed(deriveSubprocessInspectResult(table, terminalPid, platform)), + ), snapshotSucceeded, + queryElapsedMs, }), ); const subprocessPollIntervalMs = @@ -2632,39 +2668,29 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func })); if (runningSessions.length === 0) { - return true; + return { snapshotSucceeded: true, queryElapsedMs: 0 }; } if ( runningSessions.every(({ session }) => isQuiet(session)) && skippedQuietTicks < QUIET_SUBPROCESS_SNAPSHOT_EVERY_TICKS - 1 ) { skippedQuietTicks += 1; - return true; + return { snapshotSucceeded: true, queryElapsedMs: 0 }; } skippedQuietTicks = 0; - const inspectorOption = yield* acquireSubprocessInspector.pipe( - Effect.asSome, - Effect.catch((reason) => - Effect.logWarning("failed to snapshot processes for terminal subprocess polling", { - reason, - }).pipe( - Effect.as( - Option.none<{ - readonly inspector: TerminalSubprocessInspector; - readonly snapshotSucceeded: boolean; - }>(), - ), - ), - ), + const { inspector, snapshotSucceeded, queryElapsedMs } = yield* acquireSubprocessInspector( + PROCESS_TABLE_QUERY_TIMEOUT, ); - - if (Option.isNone(inspectorOption)) { + if (Result.isFailure(inspector)) { + yield* Effect.logWarning("failed to snapshot processes for terminal subprocess polling", { + reason: inspector.failure, + }); quietSessions = new WeakMap(); - return false; + return { snapshotSucceeded: false, queryElapsedMs }; } - const { inspector: subprocessInspector, snapshotSucceeded } = inspectorOption.value; + const subprocessInspector = inspector.success; const checkSubprocessActivity = Effect.fn("terminal.checkSubprocessActivity")(function* ( candidate: (typeof runningSessions)[number], ) { @@ -2760,7 +2786,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func discard: true, }); - return snapshotSucceeded; + return { snapshotSucceeded, queryElapsedMs }; }); const hasRunningSessions = readManagerState.pipe( @@ -2775,13 +2801,14 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func Effect.flatMap((active) => active ? pollSubprocessActivity().pipe( - Effect.flatMap((snapshotSucceeded) => { + Effect.flatMap(({ snapshotSucceeded, queryElapsedMs }) => { subprocessSnapshotFailureCount = snapshotSucceeded ? 0 : Math.min(subprocessSnapshotFailureCount + 1, 30); const delayMs = subprocessSnapshotPollDelayMs( subprocessPollIntervalMs, subprocessSnapshotFailureCount, + queryElapsedMs, ); return Effect.sleep(delayMs); }), @@ -3417,7 +3444,10 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func running.map((session) => [session.terminalId, terminalActivityMark(session)]), ); // Inspect now so commands started after the last poll still protect their terminals. - const { inspector } = yield* acquireSubprocessInspector; + const { inspector: inspectorResult } = yield* acquireSubprocessInspector( + platform === "win32" ? CLOSE_IDLE_QUERY_TIMEOUT.win32 : CLOSE_IDLE_QUERY_TIMEOUT.posix, + ); + const inspector = yield* Effect.fromResult(inspectorResult); yield* Effect.forEach( running, (session) => diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index bee7a8a480bc1..33f2332dbd4a5 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -2299,7 +2299,7 @@ describe("UsageService", () => { 60 + 20 * (version === 4 ? 1 : 2), ); yield* upgraded.awaitPersisted; - assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v9.json"); + assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v10.json"); assert.strictEqual( yield* Effect.promise(() => NodeFSP.readFile(legacyPath, "utf8")), legacy, diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index 552c97585b87e..477b8bb6fc4f6 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -218,7 +218,11 @@ describe("scan cache round trip", () => { }; const restored = decodeScanCache(JSON.parse(JSON.stringify(legacy)), TEST_FORMAT_MAP); - expect(restored.get("/gone.jsonl")).toEqual(original.get("/gone.jsonl")); + expect(restored.get("/gone.jsonl")).toEqual({ + ...original.get("/gone.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); expect(restored.get("/gone-codex.jsonl")).toEqual({ ...original.get("/gone-codex.jsonl"), size: -1, @@ -245,6 +249,18 @@ describe("scan cache round trip", () => { }, ); + it("re-parses a Claude transcript cached before advisor usage was counted", () => { + const original = cacheWith([["/claude.jsonl", 100, [record()]]]); + const v9 = { ...JSON.parse(makeScanCacheWriter()(original, {})), version: 9 }; + + const restored = decodeScanCache(JSON.parse(JSON.stringify(v9)), TEST_FORMAT_MAP); + expect(restored.get("/claude.jsonl")).toEqual({ + ...original.get("/claude.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); + }); + it("rewrites only changed entries and still restores the whole cache", () => { const write = makeScanCacheWriter(); const cache = cacheWith([ diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index a74bc1840c025..70c0c3df17e3e 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -9,7 +9,7 @@ import * as Schema from "effect/Schema"; import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; -const USAGE_SCAN_CACHE_VERSION = 9 as const; +const USAGE_SCAN_CACHE_VERSION = 10 as const; const SPEED_COMPATIBLE_SINCE_VERSION = 4; /** @@ -17,8 +17,9 @@ const SPEED_COMPATIBLE_SINCE_VERSION = 4; * replace it, dropping saved usage for deleted transcripts. Separate files keep * both. */ -export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v9.json"; +export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v10.json"; export const LEGACY_SCAN_CACHE_FILE_NAMES = [ + "usage-scan-cache-v9.json", "usage-scan-cache-v8.json", "usage-scan-cache-v7.json", "usage-scan-cache-v6.json", @@ -291,10 +292,12 @@ export function decodeScanCache( } // Keep history for deleted rollouts, but re-parse incompatible reducer state. // v9 repairs skipped initial Codex requests; clean v8 entries can still resume. + // v10 adds Claude advisor calls, which no older Claude entry holds. const legacy = - format.state !== undefined && - (version < 6 || - (entry.p === "codex" && (version < 8 || (version < 9 && malformedRecords > 0)))); + (entry.p === "claude" && version < 10) || + (format.state !== undefined && + (version < 6 || + (entry.p === "codex" && (version < 8 || (version < 9 && malformedRecords > 0))))); // A corrupt state disqualifies the entry: resuming with it would attach // appended usage to the wrong model or replay fork-copied history. if (!legacy && !isValidState.get(entry.p)?.(entry.cs)) continue; diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index cbc3efcc3b4e7..d05a41a6f3824 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -920,6 +920,54 @@ it.effect("retries a fetch that lost a ref update to another Git process", () => }).pipe(Effect.provide(layerServerConfig.pipe(Layer.provideMerge(NodeServices.layer)))), ); +it.effect("lets a worktree checkout that is still working run past five minutes", () => + Effect.gen(function* () { + const checkoutStarted = yield* Deferred.make(); + const checkoutFinished = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) + return yield* Effect.die("expected Git command"); + if (!command.args.includes("worktree")) return makeSuccessfulHandle(""); + yield* Deferred.succeed(checkoutStarted, undefined); + return ChildProcessSpawner.makeHandle({ + ...makeSuccessfulHandle(""), + exitCode: Deferred.await(checkoutFinished).pipe( + Effect.as(ChildProcessSpawner.ExitCode(0)), + ), + stderr: Stream.encodeText( + Stream.concat( + Stream.make("Updating files: 40% (49052/122629)\n"), + Stream.fromEffect( + Deferred.await(checkoutFinished).pipe( + Effect.as("Updating files: 100% (122629/122629), done.\n"), + ), + ), + ), + ), + }); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const path = yield* Path.Path; + const cwd = yield* makeTmpDir(); + const worktreePath = path.join(cwd, "worktree"); + + const creation = yield* driver + .createWorktree({ cwd, refName: "main", newRefName: "t3/slow-checkout", path: worktreePath }) + .pipe(Effect.forkChild); + yield* Deferred.await(checkoutStarted); + yield* TestClock.adjust("6 minutes"); + yield* Deferred.succeed(checkoutFinished, undefined); + + const result = yield* Fiber.join(creation); + assert.equal(result.worktree.path, worktreePath); + assert.equal(result.worktree.refName, "t3/slow-checkout"); + }).pipe(Effect.provide(layerServerConfig.pipe(Layer.provideMerge(NodeServices.layer)))), +); + describe("rejected git refs", () => { const remoteUrl = "https://user:token-123@git.example.invalid/org/secret-repo.git"; const failedPush = `error: failed to push some refs to '${remoteUrl}'`; diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 03f0286ce50d7..3a7c99fbc3d31 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -62,7 +62,6 @@ const branchBaseConfigKey = (branch: string) => `branch.${branch}.t3-merge-base` */ const legacyBranchBaseConfigKey = (branch: string) => `branch.${branch}.gh-merge-base`; const gitProcesses = Semaphore.makeUnsafe(8); -const WORKTREE_ADD_TIMEOUT_MS = 300_000; const WORKTREE_REMOVE_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000; const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]"; @@ -3877,7 +3876,9 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ["-c", `checkout.workers=${checkoutWorkers}`, ...args], { fallbackErrorDetail: "git worktree add failed", - timeoutMs: WORKTREE_ADD_TIMEOUT_MS, + // Large checkouts and post-checkout hooks can legitimately run for many + // minutes; a fixed deadline kills them and leaves an orphaned worktree. + timeoutMs: null, ...(onCheckoutProgress ? { // Git only prints checkout progress when stderr is a tty or the diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 354315ab6fc8e..b77c057e07b2e 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -1795,6 +1795,7 @@ export const layerWsRpc = ( threadFind: true, threadFindProgressive: true, secretProviderSettings: true, + secretReferences: true, ...Option.match(scratchWorkspaceRoot, { onNone: () => ({}), onSome: (root) => ({ scratchWorkspaceRoot: root }), @@ -1918,6 +1919,8 @@ export const layerWsRpc = ( ), ), [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: (input) => threadManagement.stopWorkflow(input), + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: (input) => + threadManagement.forceStopThread(input), [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: (input) => readWorkflowScript({ scriptPath: input.scriptPath }), [ORCHESTRATION_V2_WS_METHODS.getTurnItem]: (input) => @@ -2316,9 +2319,11 @@ export const layerWsRpc = ( cwd: input.cwd, fresh: input.fresh === true, }) - : input.instanceId !== undefined - ? providerRegistry.refreshInstance(input.instanceId) - : providerRegistry.refresh(); + : input.reloadSecrets === true + ? providerRegistry.reloadSecretsAndRefresh(input.instanceId) + : input.instanceId !== undefined + ? providerRegistry.refreshInstance(input.instanceId) + : providerRegistry.refresh(); if (input.refreshModels) { const instances = yield* providerInstances.listInstances; for (const instance of instances) { diff --git a/apps/web/package.json b/apps/web/package.json index 18d76052bbe9d..d789fc4c298d5 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -36,6 +36,8 @@ "@t3tools/client-runtime": "workspace:*", "@t3tools/contracts": "workspace:*", "@t3tools/provider-core": "workspace:*", + "@t3tools/secret-provider-definitions": "workspace:*", + "@t3tools/secret-provider-onepassword": "workspace:*", "@t3tools/shared": "workspace:*", "@tanstack/react-pacer": "^0.19.4", "@tanstack/react-router": "^1.160.2", diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 6512d85793c29..2090f5afd200b 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -2350,7 +2350,8 @@ export default function ChatView(props: ChatViewProps) { const { acknowledgeActiveThreadWoke, activeBranchMismatchKey, - backgroundWorkNow, + attentionNow, + workNow, childInputNow, goalNow, handleSwitchCheckoutToThread, @@ -2994,10 +2995,7 @@ export default function ChatView(props: ChatViewProps) { const onExpandTimelineImageStable = useStableCallback(onExpandTimelineImage); const shownThreadSyncPhase = activeEnvironmentUnavailable ? null : delayedThreadSyncPhase; const hasThreadNow = - childInputNow !== null || - backgroundWorkNow !== null || - goalNow !== null || - shownThreadSyncPhase !== null; + attentionNow !== null || workNow !== null || goalNow !== null || shownThreadSyncPhase !== null; const queuedRunCount = serverProjection ? getUserQueuedThreadRuns(serverProjection).length : 0; // The queue control is mounted for every server thread but joins the footer only while it shows // something, by its own rule: an optimistic message stops counting once the server has it. @@ -3036,8 +3034,8 @@ export default function ChatView(props: ChatViewProps) { )} diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts index 8f4b15cc11f91..bd74694dc46af 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts @@ -1841,10 +1841,10 @@ describe("deriveMessagesTimelineRows", () => { const settled = rows({ resume: "completed", working: false }); expect(shape(settled)).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:launch-answer", "user:resume", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:resume-answer", ]); @@ -1857,17 +1857,17 @@ describe("deriveMessagesTimelineRows", () => { }); expect(shape(expanded)).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "work-toggle", "assistant:launch-answer", "user:resume", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:resume-answer", ]); expect(shape(rows({ resume: "running", working: true }))).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:launch-answer", "user:resume", "working", @@ -1876,7 +1876,7 @@ describe("deriveMessagesTimelineRows", () => { expect(shape(rows({ resume: "failed", working: false }))).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:launch-answer", "user:resume", "work", diff --git a/apps/web/src/components/chat/ThreadActivityBanners.tsx b/apps/web/src/components/chat/ThreadActivityBanners.tsx index 168b674d26c92..8bd26e21a04dc 100644 --- a/apps/web/src/components/chat/ThreadActivityBanners.tsx +++ b/apps/web/src/components/chat/ThreadActivityBanners.tsx @@ -7,17 +7,20 @@ import type { EnvironmentThreadShell, ThreadStructure } from "@t3tools/client-ru import { presentPendingBackgroundWork, presentProviderGoal, + type RunStopPresentation, } from "@t3tools/client-runtime/state/thread-execution"; import { childThreadInputsTitle } from "@t3tools/client-runtime/state/threads"; import { formatDuration } from "@t3tools/shared/orchestrationTiming"; import * as DateTime from "effect/DateTime"; import { CircleDashedIcon, + CircleStopIcon, GitBranchIcon, MessageCircleIcon, ShieldQuestionIcon, TargetIcon, TerminalIcon, + TriangleAlertIcon, } from "lucide-react"; import { Fragment } from "react"; import { tickTextContent } from "~/lib/tickTextContent"; @@ -105,6 +108,46 @@ export function goalNowItem(input: { }; } +/** A Stop the provider has not finished; Force stop once it stops answering. */ +export function runStopNowItem(input: { + readonly threadId: ThreadId; + readonly stop: RunStopPresentation; + readonly canForceStop: boolean; + readonly forceStopping: boolean; + readonly onForceStop: () => void; +}): ThreadNowItem { + const unresponsive = input.stop.state === "unresponsive"; + const requestedAtMs = Date.parse(input.stop.requestedAt); + return { + id: `run-stop:${input.threadId}`, + icon: unresponsive ? ( + + ) : ( + + ), + title: unresponsive ? "Agent not responding to Stop" : "Stopping", + description: unresponsive + ? "It may be short on memory. Force stop kills its processes." + : undefined, + actions: ( + <> + formatDuration(Math.max(1_000, nowMs - requestedAtMs)), + true, + )} + className="tabular-nums" + /> + {unresponsive && input.canForceStop ? ( + + {input.forceStopping ? "Force stopping..." : "Force stop"} + + ) : null} + + ), + }; +} + /** Background commands and subagents, for the end of the timeline. */ export function backgroundWorkNowItem(input: { readonly threadId: ThreadId; diff --git a/apps/web/src/components/chat/WorktreeSetupCard.browser.test.tsx b/apps/web/src/components/chat/WorktreeSetupCard.browser.test.tsx index 1a34f537efd15..eddd6691884ef 100644 --- a/apps/web/src/components/chat/WorktreeSetupCard.browser.test.tsx +++ b/apps/web/src/components/chat/WorktreeSetupCard.browser.test.tsx @@ -53,13 +53,13 @@ it("ticks the running stage while finished stages keep their duration", async () // Assert synchronously: expect.element polls on timers these tests fake. const section = screen.getByRole("region", { name: "Worktree setup" }); - expect(section.element().textContent).toContain("Fetch base branch3.0s"); - expect(section.element().textContent).toContain("Check out files7.0s"); + expect(section.element().textContent).toContain("Fetch base branch3s"); + expect(section.element().textContent).toContain("Check out files7s"); await act(async () => { await vi.advanceTimersByTimeAsync(5_000); }); - expect(section.element().textContent).toContain("Fetch base branch3.0s"); + expect(section.element().textContent).toContain("Fetch base branch3s"); expect(section.element().textContent).toContain("Check out files12s"); }); @@ -70,9 +70,9 @@ it("stops a stage interrupted by the setup ending at the setup's end", async () await using screen = await render(card(failed)); const section = screen.getByRole("region", { name: "Worktree setup" }); - expect(section.element().textContent).toContain("Check out files2.0s"); + expect(section.element().textContent).toContain("Check out files2s"); await act(async () => { await vi.advanceTimersByTimeAsync(5_000); }); - expect(section.element().textContent).toContain("Check out files2.0s"); + expect(section.element().textContent).toContain("Check out files2s"); }); diff --git a/apps/web/src/components/chat/useForceStopThread.ts b/apps/web/src/components/chat/useForceStopThread.ts new file mode 100644 index 0000000000000..00876c758a8b6 --- /dev/null +++ b/apps/web/src/components/chat/useForceStopThread.ts @@ -0,0 +1,59 @@ +import { useAtomValue } from "@effect/atom-react"; +import type { EnvironmentId, RunId, ThreadId } from "@t3tools/contracts"; +import { useState } from "react"; + +import { useServerConfigs } from "../../state/entities"; +import { orchestrationEnvironment } from "../../state/orchestration"; +import { + isAtomCommandInterrupted, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { toastManager } from "../ui/toast"; + +/** Force stop for an agent that no longer answers Stop; shared by every place that offers it. */ +export function useForceStopThread(environmentId: EnvironmentId) { + const forceStop = useAtomCommand(orchestrationEnvironment.forceStopThread); + const permitted = useAtomValue( + orchestrationEnvironment.forceStopThread.permissionAtom(environmentId), + ); + const supported = + useServerConfigs().get(environmentId)?.environment.capabilities.threadForceStop === true; + const canForceStop = permitted && supported; + const [stoppingThreadId, setStoppingThreadId] = useState(null); + /** Pass the run that ignored Stop, so a Force stop left open cannot hit a newer run. */ + const forceStopThread = async (threadId: ThreadId, runId?: RunId) => { + if (!canForceStop || stoppingThreadId !== null) return; + setStoppingThreadId(threadId); + const result = await forceStop({ + environmentId, + input: runId === undefined ? { threadId } : { threadId, runId }, + }); + setStoppingThreadId(null); + if (result._tag === "Failure") { + if (isAtomCommandInterrupted(result)) return; + const error = squashAtomCommandFailure(result); + toastManager.add({ + type: "error", + title: "Could not force stop the agent", + description: error instanceof Error ? error.message : undefined, + }); + return; + } + const others = result.value.sharedWithThreadIds.length; + if (result.value.outcome === "releasing") { + toastManager.add({ + type: "info", + title: "Turn stopped; the agent is still shutting down", + }); + } + if (result.value.outcome === "shared") { + toastManager.add({ + type: "info", + title: "Turn stopped; the agent process kept running", + description: `It is shared with ${others === 0 ? "other" : others} working ${others === 1 ? "thread" : "threads"}. Stop those to free it.`, + }); + } + }; + return { canForceStop, stoppingThreadId, forceStopThread }; +} diff --git a/apps/web/src/components/chat/useThreadLifecycle.tsx b/apps/web/src/components/chat/useThreadLifecycle.tsx index b1cf8a4a6af4c..0ad8acacf71ce 100644 --- a/apps/web/src/components/chat/useThreadLifecycle.tsx +++ b/apps/web/src/components/chat/useThreadLifecycle.tsx @@ -58,9 +58,22 @@ import { appAtomRegistry } from "../../rpc/atomRegistry"; import { readEnvironmentScope } from "../../state/session"; import { useAtomCommand } from "../../state/use-atom-command"; import { useOrchestrationCommand } from "../../state/use-orchestration-command"; -import { backgroundWorkNowItem, childInputNowItem, goalNowItem } from "./ThreadActivityBanners"; +import { + backgroundWorkNowItem, + childInputNowItem, + goalNowItem, + runStopNowItem, +} from "./ThreadActivityBanners"; +import { useForceStopThread } from "./useForceStopThread"; +import { useAtomValue } from "@effect/atom-react"; +import type { RunStopPresentation } from "@t3tools/client-runtime/state/thread-execution"; +import { Atom } from "effect/reactivity"; +import { environmentThreadDetails } from "../../state/threads"; import type { useLocalDispatchState } from "./useLocalDispatchState"; const RELATED_THREAD_SHELL_WAIT_MS = 1_000; +const NO_RUN_STOP_ATOM = Atom.make(null).pipe( + Atom.withLabel("web-thread-run-stop:empty"), +); type PendingThreadRequests = ReturnType; type LocalDispatchState = ReturnType; @@ -509,6 +522,22 @@ export function useThreadLifecycle(input: ThreadLifecycleInput) { }) : null; + const forceStop = useForceStopThread(environmentId); + const anyRunStop = useAtomValue( + activeThreadRef ? environmentThreadDetails.runStopAtom(activeThreadRef) : NO_RUN_STOP_ATOM, + ); + // A background Stop matters only while that work is still pending. + const runStop = anyRunStop?.background && activeBackgroundTasks.length === 0 ? null : anyRunStop; + const runStopNow = + activeThread && runStop !== null + ? runStopNowItem({ + threadId: activeThread.id, + stop: runStop, + canForceStop: forceStop.canForceStop, + forceStopping: forceStop.stoppingThreadId === activeThread.id, + onForceStop: () => void forceStop.forceStopThread(activeThread.id, runStop.runId), + }) + : null; const backgroundWorkNow = activeThread ? backgroundWorkNowItem({ threadId: activeThread.id, @@ -592,7 +621,9 @@ export function useThreadLifecycle(input: ThreadLifecycleInput) { shortcuts, acknowledgeActiveThreadWoke, activeBranchMismatchKey, - backgroundWorkNow, + // A Stop that went unanswered needs the user; one in progress is the thread's work. + attentionNow: runStop?.state === "unresponsive" ? runStopNow : childInputNow, + workNow: runStop?.state === "stopping" ? runStopNow : backgroundWorkNow, childInputNow, goalNow, handleSwitchCheckoutToThread, diff --git a/apps/web/src/components/guest/GuestPage.tsx b/apps/web/src/components/guest/GuestPage.tsx index 21c35447ddf8c..7e41e9a8ca297 100644 --- a/apps/web/src/components/guest/GuestPage.tsx +++ b/apps/web/src/components/guest/GuestPage.tsx @@ -1,6 +1,7 @@ import { EnvironmentId, GUEST_MESSAGE_MAX_LENGTH, + GUEST_HISTORY_ROUTE, GUEST_NAME_MAX_LENGTH, GUEST_ROUTE_PREFIX, GUEST_SHARE_PARAM, @@ -8,6 +9,7 @@ import { type GuestRedeemRequest, type GuestShareView, type GuestThreadModel, + OrchestrationV2ThreadHistoryPage, type OrchestrationV2ThreadProjection, type ThreadGuestMessage, type ThreadGuestMessageStatus, @@ -46,7 +48,7 @@ import { GuestMessageAttachments } from "../chat/GuestMessageAttachments"; import { HostImageUrlContext } from "../chat/hostImageUrl"; import { PierreEntryIcon } from "../chat/PierreEntryIcon"; import { GuestMessageName, guestBubbleClassName } from "../chat/guestMessageStyle"; -import { MessagesTimeline } from "../chat/MessagesTimeline"; +import { MessagesTimeline, type MessagesTimelineHistoryControls } from "../chat/MessagesTimeline"; import { ProviderInstanceIcon } from "../chat/ProviderInstanceIcon"; import { getDisplayModelName, getTriggerDisplayModelName } from "../chat/providerIconUtils"; import { ScrollToEndButton } from "../chat/TimelineOverlays"; @@ -131,6 +133,32 @@ function takeGuestEntry(): GuestEntry { } const decodeErrorBody = Schema.decodeUnknownOption(Schema.Struct({ message: Schema.String })); +const decodeHistoryPage = Schema.decodeUnknownOption( + Schema.toCodecJson(OrchestrationV2ThreadHistoryPage), +); + +/** Resolves to the page, or to a message the guest can read. */ +async function fetchGuestHistoryPage( + shareId: string, + cursor: string, +): Promise { + const failed = "Could not load earlier turns. Try again."; + try { + const response = await fetch( + `${GUEST_HISTORY_ROUTE}?${GUEST_SHARE_PARAM}=${encodeURIComponent(shareId)}&cursor=${encodeURIComponent(cursor)}`, + ); + const json: unknown = await response.json().catch(() => null); + if (!response.ok) { + return Option.match(decodeErrorBody(json), { + onNone: () => failed, + onSome: ({ message }) => message, + }); + } + return Option.getOrElse(decodeHistoryPage(json), () => failed); + } catch { + return "Could not reach the server."; + } +} /** Resolves to null on success, or to a message the guest can read. */ async function postGuest( @@ -241,6 +269,32 @@ function GuestSession({ shareId }: { readonly shareId: string }) { // missing, invalid, or ended session. Network drops reconnect on their own. const [rejected, setRejected] = useState(false); + const [historyLoad, setHistoryLoad] = useState<{ + readonly cursor: string; + readonly error: string | null; + } | null>(null); + const historyCursor = stream.historyCursor; + // A load for an older cursor is stale: a page or a reconnect snapshot moved the cursor on. + const currentLoad = historyLoad?.cursor === historyCursor ? historyLoad : null; + const loadingHistory = currentLoad !== null && currentLoad.error === null; + const historyError = currentLoad?.error ?? null; + const loadEarlier = async () => { + if (historyCursor === null || loadingHistory) return; + setHistoryLoad({ cursor: historyCursor, error: null }); + const result = await fetchGuestHistoryPage(shareId, historyCursor); + if (typeof result === "string") setHistoryLoad({ cursor: historyCursor, error: result }); + else dispatch({ type: "history-page", cursor: historyCursor, page: result }); + }; + const historyControls: MessagesTimelineHistoryControls | undefined = + historyCursor === null + ? undefined + : { + hasMoreHistory: true, + loading: loadingHistory, + error: historyError, + onLoadEarlier: () => void loadEarlier(), + }; + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- sets document.title to the shared thread's title useEffect(() => { if (stream.view) document.title = stream.view.title; @@ -273,6 +327,7 @@ function GuestSession({ shareId }: { readonly shareId: string }) { view={stream.view} projection={stream.projection} awaitingRowIds={stream.awaitingRowIds} + historyControls={historyControls} /> ); } @@ -358,10 +413,12 @@ function GuestThread({ view: { shareId, title, expiresAt, messages, model }, projection, awaitingRowIds, + historyControls, }: { readonly view: GuestShareView; readonly projection: OrchestrationV2ThreadProjection; readonly awaitingRowIds: GuestStreamState["awaitingRowIds"]; + readonly historyControls: MessagesTimelineHistoryControls | undefined; }) { // No ":" so the timeline finds no environment thread behind the key. const threadKey = `guest-${shareId}`; @@ -467,6 +524,7 @@ function GuestThread({ onFileDownload={openFile} renderHtmlRender={renderHtmlRender} turnItemSource={turnItemSource} + historyControls={historyControls} activeThreadEnvironmentId={GUEST_ENVIRONMENT_ID} markdownCwd={undefined} resolvedTheme={resolvedTheme} diff --git a/apps/web/src/components/guest/guestStream.logic.test.ts b/apps/web/src/components/guest/guestStream.logic.test.ts index fbdf9ce1927c2..674227dc171ff 100644 --- a/apps/web/src/components/guest/guestStream.logic.test.ts +++ b/apps/web/src/components/guest/guestStream.logic.test.ts @@ -112,6 +112,45 @@ describe("applyGuestStreamEvent", () => { expect(apply([itemEvent(2, reply("a1", "Changed"))], once)).toBe(once); }); + it("merges older pages in front, only for the cursor the thread is at", () => { + const windowed: GuestStreamEvent = { + type: "thread", + item: { + kind: "snapshot", + snapshotSequence: 1, + projection, + historyCursor: "cursor-1", + hasMoreHistory: true, + }, + }; + const page = (id: string, nextCursor: string | null) => ({ + snapshotSequence: 1, + items: [ + { + position: 0, + visibility: "local" as const, + sourceThreadId: projection.thread.id, + sourceItemId: TurnItemId.make(id), + item: reply(id, "Earlier"), + }, + ], + nextCursor, + hasMoreHistory: nextCursor !== null, + }); + const live = apply([windowed, itemEvent(2, reply("a1", "Hi"))]); + expect(live.historyCursor).toBe("cursor-1"); + // A page fetched before a reconnect moved the cursor changes nothing. + const stale = { type: "history-page" as const, cursor: "cursor-0", page: page("old", null) }; + expect(applyGuestStreamEvent(live, stale)).toBe(live); + const merged = applyGuestStreamEvent(live, { + type: "history-page", + cursor: "cursor-1", + page: page("o1", null), + }); + expect(itemIds(merged)).toEqual(["o1", "a1"]); + expect(merged.historyCursor).toBeNull(); + }); + it("replaces the thread on a new snapshot, as after a reconnect", () => { const before = apply([snapshot(1), itemEvent(2, reply("a1", "Hi")), { type: "ended" }]); const after = apply([snapshot(1)], { ...before, ended: false }); diff --git a/apps/web/src/components/guest/guestStream.logic.ts b/apps/web/src/components/guest/guestStream.logic.ts index 0b37d66eeec84..03eafb6d348ee 100644 --- a/apps/web/src/components/guest/guestStream.logic.ts +++ b/apps/web/src/components/guest/guestStream.logic.ts @@ -9,11 +9,13 @@ import { type GuestMessageId, type GuestShareView, type OrchestrationV2ProjectedTurnItem, + type OrchestrationV2ThreadHistoryPage, type OrchestrationV2ThreadProjection, type OrchestrationV2TurnItem, type ThreadGuestMessage, } from "@t3tools/contracts"; import { applyOrchestrationV2ProjectionEvent } from "@t3tools/client-runtime/state/orchestration-v2-projection"; +import { mergeOlderHistoryIntoProjection } from "@t3tools/client-runtime/state/thread-history-merge"; import { boundedSnapshotProjection } from "@t3tools/shared/orchestrationV2BoundedSnapshot"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; @@ -25,6 +27,8 @@ export interface GuestStreamState { readonly sequence: number; /** Set while the snapshot is a window of a longer history, as the owner's client tracks it. */ readonly partial: { readonly latestLocalTurnOrdinal: number | null } | null; + /** Where "Load earlier turns" continues, or null once the whole thread is loaded. */ + readonly historyCursor: string | null; readonly ended: boolean; /** * Messages this page saw waiting that have no transcript row yet. Survives reconnects, and a @@ -38,6 +42,7 @@ export const initialGuestStreamState: GuestStreamState = { projection: null, sequence: 0, partial: null, + historyCursor: null, ended: false, awaitingRowIds: new Set(), }; @@ -76,16 +81,32 @@ const decodeGuestStreamEvent = Schema.decodeUnknownOption( export const parseGuestStreamEvent = (data: string): GuestStreamEvent | null => Option.getOrNull(decodeGuestStreamEvent(data)); +/** A page of older turns, fetched from `cursor`. */ +export type GuestHistoryPageLoaded = { + readonly type: "history-page"; + readonly cursor: string; + readonly page: OrchestrationV2ThreadHistoryPage; +}; + /** * Applies the owner's thread stream the way the owner's client does (see * EnvironmentThreadState in client-runtime): a snapshot replaces the thread, and later events - * pass through the same projection reducer, once each. + * pass through the same projection reducer, once each. Older pages merge in front, as they do in + * the owner's view. */ export function applyGuestStreamEvent( state: GuestStreamState, - event: GuestStreamEvent, + event: GuestStreamEvent | GuestHistoryPageLoaded, ): GuestStreamState { switch (event.type) { + case "history-page": + // A snapshot since the request replaced the thread and its cursor. + if (state.projection === null || state.historyCursor !== event.cursor) return state; + return { + ...state, + projection: mergeOlderHistoryIntoProjection(state.projection, event.page.items), + historyCursor: event.page.hasMoreHistory ? event.page.nextCursor : null, + }; case "view": // The page follows the share it names; a view of another one means the link changed. if (state.view !== null && state.view.shareId !== event.view.shareId) { @@ -108,6 +129,7 @@ export function applyGuestStreamEvent( item.hasMoreHistory === true || (item.historyCursor ?? null) !== null ? { latestLocalTurnOrdinal: item.latestLocalTurnOrdinal ?? null } : null, + historyCursor: item.hasMoreHistory === true ? (item.historyCursor ?? null) : null, awaitingRowIds: withoutPlaced( state.awaitingRowIds, projection.visibleTurnItems.map((row) => row.item), diff --git a/apps/web/src/components/settings/AddProviderInstanceDialog.tsx b/apps/web/src/components/settings/AddProviderInstanceDialog.tsx index 6e245af143dd8..3097cb70a9fbf 100644 --- a/apps/web/src/components/settings/AddProviderInstanceDialog.tsx +++ b/apps/web/src/components/settings/AddProviderInstanceDialog.tsx @@ -55,6 +55,7 @@ import { resolveOfficialAcpRegistryIconUrl } from "./AcpRegistryIcon"; import { AddManagedCodexAccountDialog } from "./CodexSetupSection"; import { MultipleAccountsNote } from "./MultipleAccountsNote"; import { ProviderEnvironmentSection } from "./ProviderInstanceCard"; +import { useSecretSources } from "./SecretSourceInput"; /** Limit the slug to 48 chars so the composed ProviderInstanceId fits its 64-char cap. */ function slugifyLabel(value: string): string { @@ -160,6 +161,7 @@ export function AddProviderInstanceDialog({ onCreated, }: AddProviderInstanceDialogProps) { const settings = useEnvironmentSettings(environmentId); + const secretSources = useSecretSources(environmentId); const persistProviderInstance = usePersistEnvironmentProviderInstanceMutation(environmentId); const canManageProviders = useEnvironmentScope(environmentId, AuthProvidersManageScope); @@ -536,6 +538,7 @@ export function AddProviderInstanceDialog({ ) : null} diff --git a/apps/web/src/components/settings/DiagnosticsSettings.tsx b/apps/web/src/components/settings/DiagnosticsSettings.tsx index 613074b2d914b..0fcea4fa5de00 100644 --- a/apps/web/src/components/settings/DiagnosticsSettings.tsx +++ b/apps/web/src/components/settings/DiagnosticsSettings.tsx @@ -48,7 +48,7 @@ function formatCount(value: number): string { function formatDuration(value: number): string { if (value < 1_000) return `${Math.round(value)} ms`; - return `${(value / 1_000).toFixed(value >= 10_000 ? 1 : 2)} s`; + return `${(value / 1_000).toFixed(value >= 10_000 ? 1 : 2).replace(/\.0+$/, "")} s`; } function formatBytes(value: number): string { @@ -438,10 +438,10 @@ const RESOURCE_HISTORY_WINDOWS = [ ] as const; function formatCpuTime(seconds: number): string { - if (seconds < 60) return `${seconds.toFixed(seconds >= 10 ? 1 : 2)}s`; + if (seconds < 60) return `${seconds.toFixed(seconds >= 10 ? 1 : 2).replace(/\.0+$/, "")}s`; const minutes = seconds / 60; - if (minutes < 60) return `${minutes.toFixed(minutes >= 10 ? 1 : 2)}m`; - return `${(minutes / 60).toFixed(2)}h`; + if (minutes < 60) return `${minutes.toFixed(minutes >= 10 ? 1 : 2).replace(/\.0+$/, "")}m`; + return `${(minutes / 60).toFixed(2).replace(/\.0+$/, "")}h`; } function formatShortProcessName(command: string): string { diff --git a/apps/web/src/components/settings/ProviderInstanceCard.test.ts b/apps/web/src/components/settings/ProviderInstanceCard.test.ts index c681f78765178..143d64d5c1bdf 100644 --- a/apps/web/src/components/settings/ProviderInstanceCard.test.ts +++ b/apps/web/src/components/settings/ProviderInstanceCard.test.ts @@ -1,8 +1,9 @@ import { describe, expect, it } from "vite-plus/test"; -import type { ServerProviderModel } from "@t3tools/contracts"; +import { SecretProviderId, type ServerProviderModel } from "@t3tools/contracts"; import { deriveProviderModelsForDisplay, + nextProviderEnvironmentWithFieldSecret, nextProviderEnvironmentWithFieldValue, } from "./ProviderInstanceCard"; @@ -75,6 +76,20 @@ describe("provider environment helpers", () => { ).toEqual([{ name: "CURSOR_API_KEY", value: "new-key", sensitive: true }]); }); + it("points a stored provider secret at a secret provider and drops the stored value", () => { + const secret = { + provider: SecretProviderId.make("personal"), + reference: "op://Private/Cursor/credential", + }; + const stored = [{ name: "CURSOR_API_KEY", value: "", sensitive: true, valueRedacted: true }]; + const referenced = nextProviderEnvironmentWithFieldSecret(stored, cursorApiKeyField, secret); + + expect(referenced).toEqual([{ name: "CURSOR_API_KEY", value: secret, sensitive: true }]); + expect( + nextProviderEnvironmentWithFieldSecret(referenced, cursorApiKeyField, undefined), + ).toEqual([]); + }); + it("applies the secure field default when replacing an existing non-sensitive value", () => { expect( nextProviderEnvironmentWithFieldValue( diff --git a/apps/web/src/components/settings/ProviderInstanceCard.tsx b/apps/web/src/components/settings/ProviderInstanceCard.tsx index 88361eac68bc8..d8b5c71fc0f98 100644 --- a/apps/web/src/components/settings/ProviderInstanceCard.tsx +++ b/apps/web/src/components/settings/ProviderInstanceCard.tsx @@ -15,7 +15,7 @@ import { import { Lock as LockGlyph, LockOpen } from "lucide"; import * as Arr from "effect/Array"; import * as Result from "effect/Result"; -import { useState, type ReactElement, type ReactNode } from "react"; +import { Fragment, useState, type ReactElement, type ReactNode } from "react"; import { isProviderDriverKind, resolveProviderInstanceEnabled, @@ -26,6 +26,7 @@ import { type EnvironmentId, type ProjectId, type ProviderDriverKind, + type SecretReference, type ServerProvider, type ServerProviderModel, } from "@t3tools/contracts"; @@ -43,7 +44,7 @@ import { Badge } from "../ui/badge"; import { Button } from "../ui/button"; import { MorphIcon } from "~/components/MorphIcon"; import { DraftInput } from "../ui/draft-input"; -import { SecretInput } from "./SecretInput"; +import { SecretSourceInput, type SecretSource } from "./SecretSourceInput"; import { Popover, PopoverPopup, PopoverTrigger } from "../ui/popover"; import { Switch } from "../ui/switch"; import { stackedThreadToast, toastManager } from "../ui/toast"; @@ -98,7 +99,7 @@ const nextEnvironmentVariableDraftId = () => `provider-env-${environmentVariable type EnvironmentDraftRow = { readonly id: string; readonly name: string; - readonly value: string; + readonly value: ProviderInstanceEnvironmentVariable["value"]; readonly sensitive: boolean; readonly valueRedacted?: boolean; }; @@ -119,6 +120,14 @@ function makeEnvironmentDraftRow( const toEnvironmentDraftRows = (environment: ReadonlyArray) => environment.map(makeEnvironmentDraftRow); +function environmentValuesEqual( + left: ProviderInstanceEnvironmentVariable["value"], + right: ProviderInstanceEnvironmentVariable["value"], +): boolean { + if (typeof left === "string" || typeof right === "string") return left === right; + return left.provider === right.provider && left.reference === right.reference; +} + function providerEnvironmentsEqual( left: ReadonlyArray, right: ReadonlyArray, @@ -130,7 +139,7 @@ function providerEnvironmentsEqual( return ( other !== undefined && variable.name === other.name && - variable.value === other.value && + environmentValuesEqual(variable.value, other.value) && variable.sensitive === other.sensitive && variable.valueRedacted === other.valueRedacted ); @@ -247,12 +256,28 @@ export function nextProviderEnvironmentWithFieldValue( return next; } +export function nextProviderEnvironmentWithFieldSecret( + environment: ReadonlyArray | undefined, + field: ProviderEnvironmentField, + secret: SecretReference | undefined, +): ReadonlyArray { + const others = providerEnvironmentWithoutNames(environment, new Set([field.name])); + return secret === undefined + ? others + : [...others, { name: field.name, value: secret, sensitive: true }]; +} + function ProviderEnvironmentFieldRow(props: { readonly field: ProviderEnvironmentField; readonly variable: ProviderInstanceEnvironmentVariable | undefined; readonly idPrefix: string; readonly onCommit: (field: ProviderEnvironmentField, value: string) => void; + readonly onSecretChange: ( + field: ProviderEnvironmentField, + secret: SecretReference | undefined, + ) => void; readonly onRemove: (field: ProviderEnvironmentField) => void; + readonly secretSources: ReadonlyArray; }) { const inputId = `${props.idPrefix}-environment-${props.field.name}`; const value = props.variable?.valueRedacted ? "" : (props.variable?.value ?? ""); @@ -269,13 +294,13 @@ function ProviderEnvironmentFieldRow(props: { size="sm" className="min-w-0 flex-1" autoComplete="off" - value={value} + value={typeof value === "string" ? value : ""} onCommit={(next) => props.onCommit(props.field, next)} placeholder={props.field.placeholder} spellCheck={false} /> ) : ( - props.onSecretChange(props.field, secret)} /> )} {props.variable ? ( @@ -306,6 +334,7 @@ function ProviderEnvironmentFieldRow(props: { export function ProviderEnvironmentSection(props: { readonly environment: ReadonlyArray; readonly onChange: (environment: ReadonlyArray) => void; + readonly secretSources?: ReadonlyArray; }) { const [storedDraft, setDraft] = useState(() => seedRowsDraft(props.environment, toEnvironmentDraftRows), @@ -328,7 +357,7 @@ export function ProviderEnvironmentSection(props: { if (!ENVIRONMENT_VARIABLE_NAME_PATTERN.test(name)) { if ( name.length > 0 || - row.value.length > 0 || + row.value !== "" || row.sensitive !== true || row.valueRedacted !== undefined ) { @@ -357,6 +386,16 @@ export function ProviderEnvironmentSection(props: { publishRows(nextRows); }; + const setVariableSecret = (id: string, secret: SecretReference | undefined) => { + const nextRows = rows.map((row) => { + if (row.id !== id) return row; + const { valueRedacted: _valueRedacted, ...rest } = row; + return { ...rest, value: secret ?? "" }; + }); + setRows(nextRows); + publishRows(nextRows); + }; + const removeVariable = (id: string) => { const nextRows = rows.filter((row) => row.id !== id); setRows(nextRows); @@ -387,76 +426,20 @@ export function ProviderEnvironmentSection(props: { > {rows.length > 0 ? (
- {rows.map((variable, index) => ( -
+ {rows.map((variable, index) => { + const nameInput = ( updateVariable(variable.id, { name: name.trim() })} placeholder="VARIABLE_NAME" spellCheck={false} aria-label={`Environment variable name ${index + 1}`} /> - - = - - {variable.sensitive ? ( - updateVariable(variable.id, { value })} - placeholder="value" - configured={variable.valueRedacted === true} - spellCheck={false} - aria-label={`Environment variable value ${index + 1}`} - /> - ) : ( - updateVariable(variable.id, { value })} - autoComplete="off" - placeholder="value" - spellCheck={false} - aria-label={`Environment variable value ${index + 1}`} - /> - )} - - { - const sensitive = !variable.sensitive; - updateVariable(variable.id, { - sensitive, - ...(sensitive && variable.valueRedacted === undefined - ? {} - : { valueRedacted: sensitive ? variable.valueRedacted : false }), - }); - }} - aria-pressed={variable.sensitive} - aria-label={`Mark environment variable ${variable.name || index + 1} as sensitive`} - > - - - } - /> - - {variable.sensitive ? "Sensitive, stored separately" : "Plain text"} - - + ); + const removeButton = ( -
- ))} + ); + const sensitiveToggle = + typeof variable.value === "string" ? ( + + { + const sensitive = !variable.sensitive; + updateVariable(variable.id, { + sensitive, + ...(sensitive && variable.valueRedacted === undefined + ? {} + : { valueRedacted: sensitive ? variable.valueRedacted : false }), + }); + }} + aria-pressed={variable.sensitive} + aria-label={`Mark environment variable ${variable.name || index + 1} as sensitive`} + > + + + } + /> + + {variable.sensitive ? "Sensitive, stored separately" : "Plain text"} + + + ) : null; + const layout = (source: ReactNode, value: ReactNode) => ( +
+
+ {nameInput} + {source === null ? null :
{source}
} + {removeButton} +
+
+ {value} + {sensitiveToggle} +
+
+ ); + return ( + + {variable.sensitive ? ( + updateVariable(variable.id, { value })} + placeholder="value" + configured={variable.valueRedacted === true} + spellCheck={false} + aria-label={`Environment variable value ${index + 1}`} + sources={props.secretSources ?? []} + onSecretChange={(secret) => setVariableSecret(variable.id, secret)} + > + {(parts) => layout(parts.source, parts.value)} + + ) : ( + layout( + null, + updateVariable(variable.id, { value })} + autoComplete="off" + placeholder="value" + spellCheck={false} + aria-label={`Environment variable value ${index + 1}`} + />, + ) + )} + + ); + })}

Sensitive values are stored separately and never returned to the app.

@@ -584,6 +653,7 @@ interface ProviderInstanceCardProps { readonly isUpdating?: boolean | undefined; readonly onAcceptUrlAuth?: ((action: AcpRegistryUrlAuthAction) => void) | undefined; readonly environmentId?: EnvironmentId | undefined; + readonly secretSources?: ReadonlyArray; readonly projectOverride?: ProjectOverrideControl | undefined; readonly acpProjects?: | ReadonlyArray<{ @@ -595,6 +665,7 @@ interface ProviderInstanceCardProps { } const EMPTY_ACP_PROJECTS: NonNullable = []; +const NO_SECRET_SOURCES: ReadonlyArray = []; const NO_CUSTOM_MODELS: ReadonlyArray = []; /** Unknown drivers stay read-only to preserve fork config. projectOverride switches affect only the @@ -627,6 +698,7 @@ export function ProviderInstanceCard({ onAcceptUrlAuth, environmentId, acpProjects = EMPTY_ACP_PROJECTS, + secretSources = NO_SECRET_SOURCES, projectOverride, }: ProviderInstanceCardProps) { const enabled = projectOverride?.effectiveEnabled ?? resolveProviderInstanceEnabled(instance); @@ -781,6 +853,12 @@ export function ProviderInstanceCard({ const updateEnvironmentField = (field: ProviderEnvironmentField, value: string) => { updateEnvironment(nextProviderEnvironmentWithFieldValue(instance.environment, field, value)); }; + const updateEnvironmentFieldSecret = ( + field: ProviderEnvironmentField, + secret: SecretReference | undefined, + ) => { + updateEnvironment(nextProviderEnvironmentWithFieldSecret(instance.environment, field, secret)); + }; const removeEnvironmentField = (field: ProviderEnvironmentField) => { updateEnvironment(providerEnvironmentWithoutNames(instance.environment, new Set([field.name]))); }; @@ -1182,7 +1260,9 @@ export function ProviderInstanceCard({ variable={readProviderEnvironmentVariable(instance.environment, field.name)} idPrefix={`provider-instance-${instanceId}`} onCommit={updateEnvironmentField} + onSecretChange={updateEnvironmentFieldSecret} onRemove={removeEnvironmentField} + secretSources={secretSources} /> ))}
@@ -1225,6 +1305,7 @@ export function ProviderInstanceCard({ {environmentId !== undefined && liveProvider?.driver === "acpRegistry" ? ( diff --git a/apps/web/src/components/settings/ProviderSettingsPanel.tsx b/apps/web/src/components/settings/ProviderSettingsPanel.tsx index 9dae72e98ff50..b5fb648ff97c2 100644 --- a/apps/web/src/components/settings/ProviderSettingsPanel.tsx +++ b/apps/web/src/components/settings/ProviderSettingsPanel.tsx @@ -98,6 +98,7 @@ import { stackedThreadToast, toastManager } from "../ui/toast"; import { AddProviderInstanceDialog } from "./AddProviderInstanceDialog"; import { ExpandableText } from "./ExpandableText"; import { ProviderInstanceCard } from "./ProviderInstanceCard"; +import { useSecretSources } from "./SecretSourceInput"; import { UsageProviderSettings } from "./UsageProviderSettings"; import { ProviderSetupSection, readAntigravityAuthMethod } from "./ProviderSetupSection"; import { ProviderAuthenticationSection } from "./ProviderAuthenticationSection"; @@ -635,7 +636,7 @@ export function EnvironmentProviderSettings({ void (async () => { const result = await refreshServerProviders({ environmentId, - input: { refreshModels: true }, + input: { refreshModels: true, reloadSecrets: true }, }); refreshingRef.current = false; setIsRefreshingProviders(false); @@ -1247,6 +1248,7 @@ function ProviderInstanceRow({ readonly actions: ProviderInstanceRowActions; }) { const allProjects = useProjects(); + const secretSources = useSecretSources(environmentId); const acpProjects = mode === "editor" && liveProvider?.driver === "acpRegistry" ? allProjects.filter((project) => project.environmentId === environmentId) @@ -1285,6 +1287,7 @@ function ProviderInstanceRow({ return ( = 10 ? 1 : 2)}s`; + if (seconds < 60) return `${seconds.toFixed(seconds >= 10 ? 1 : 2).replace(/\.0+$/, "")}s`; const minutes = seconds / 60; - if (minutes < 60) return `${minutes.toFixed(minutes >= 10 ? 1 : 2)}m`; - return `${(minutes / 60).toFixed(2)}h`; + if (minutes < 60) return `${minutes.toFixed(minutes >= 10 ? 1 : 2).replace(/\.0+$/, "")}m`; + return `${(minutes / 60).toFixed(2).replace(/\.0+$/, "")}h`; } function formatDurationMicros(value: number): string { if (value < 1_000) return `${Math.round(value)} µs`; - if (value < 1_000_000) return `${(value / 1_000).toFixed(2)} ms`; - return `${(value / 1_000_000).toFixed(2)} s`; + if (value < 1_000_000) return `${(value / 1_000).toFixed(2).replace(/\.0+$/, "")} ms`; + return `${(value / 1_000_000).toFixed(2).replace(/\.0+$/, "")} s`; } function formatSampleInterval(valueMs: number): string { @@ -788,7 +788,7 @@ function AttributionTable({ entries }: { entries: ReadonlyArray {entry.count} - {(entry.durationMs / 1_000).toFixed(2)}s + {(entry.durationMs / 1_000).toFixed(2).replace(/\.0+$/, "")}s ))} diff --git a/apps/web/src/components/settings/SecretProviderDialog.logic.ts b/apps/web/src/components/settings/SecretProviderDialog.logic.ts index 11139d9c7aa67..ab5872955def8 100644 --- a/apps/web/src/components/settings/SecretProviderDialog.logic.ts +++ b/apps/web/src/components/settings/SecretProviderDialog.logic.ts @@ -1,11 +1,13 @@ import { - OnePasswordSecretProviderConfig, SecretProviderId, SecretProviderKind, SecretProviderLabel, - type OnePasswordAccount, type SecretProviderConfig, } from "@t3tools/contracts"; +import { + OnePasswordSecretProviderConfig, + type OnePasswordAccount, +} from "@t3tools/secret-provider-onepassword/shared/definition"; import * as Option from "effect/Option"; import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; diff --git a/apps/web/src/components/settings/SecretSourceInput.browser.test.tsx b/apps/web/src/components/settings/SecretSourceInput.browser.test.tsx new file mode 100644 index 0000000000000..e1b67465c2168 --- /dev/null +++ b/apps/web/src/components/settings/SecretSourceInput.browser.test.tsx @@ -0,0 +1,82 @@ +import { SecretProviderId, SecretProviderKind } from "@t3tools/contracts"; +import { describe, expect, it, vi } from "vite-plus/test"; +import { userEvent } from "vite-plus/test/browser"; + +import { render, type Screen } from "../../../test/render"; +import { SecretSourceInput } from "./SecretSourceInput"; + +const personal = SecretProviderId.make("personal"); +const work = SecretProviderId.make("work"); +const sources = [ + { id: personal, kind: SecretProviderKind.make("one_password"), label: "Personal" }, + { id: work, kind: SecretProviderKind.make("one_password"), label: "Work" }, +]; + +const reference = (screen: Screen) => screen.getByRole("textbox", { name: "API key reference" }); + +async function commitReference(screen: Screen, value: string) { + await userEvent.clear(reference(screen)); + await userEvent.type(reference(screen), value); + await userEvent.keyboard("{Enter}"); +} + +describe("SecretSourceInput", () => { + it("moves a reference to another provider only once one is committed for it", async () => { + const onSecretChange = vi.fn(); + await using screen = await render( + {}} + onSecretChange={onSecretChange} + />, + ); + + await userEvent.click(screen.getByRole("combobox", { name: "API key source" })); + await userEvent.click(screen.getByRole("option", { name: "Work" })); + expect(onSecretChange).not.toHaveBeenCalled(); + await expect.element(reference(screen)).toHaveValue(""); + + await commitReference(screen, "op://Shared/OpenAI/credential"); + expect(onSecretChange).toHaveBeenCalledExactlyOnceWith({ + provider: work, + reference: "op://Shared/OpenAI/credential", + }); + }); + + it("removes the reference when its field is cleared", async () => { + const onSecretChange = vi.fn(); + await using screen = await render( + {}} + onSecretChange={onSecretChange} + />, + ); + + await userEvent.clear(reference(screen)); + await userEvent.keyboard("{Enter}"); + expect(onSecretChange).toHaveBeenCalledExactlyOnceWith(undefined); + }); + + it("rejects a reference 1Password cannot read", async () => { + const onSecretChange = vi.fn(); + await using screen = await render( + {}} + onSecretChange={onSecretChange} + />, + ); + + await commitReference(screen, "Private/OpenAI/credential"); + expect(onSecretChange).not.toHaveBeenCalled(); + await expect.element(reference(screen)).toHaveAttribute("aria-invalid", "true"); + await expect.element(reference(screen)).toHaveValue("Private/OpenAI/credential"); + }); +}); diff --git a/apps/web/src/components/settings/SecretSourceInput.tsx b/apps/web/src/components/settings/SecretSourceInput.tsx new file mode 100644 index 0000000000000..092e59b9dcd0f --- /dev/null +++ b/apps/web/src/components/settings/SecretSourceInput.tsx @@ -0,0 +1,185 @@ +import { + SecretReference, + type EnvironmentId, + type SecretProviderConfig, + type SecretProviderId, + type SecretProviderKind, +} from "@t3tools/contracts"; +import { builtInSecretProviderDefinitions } from "@t3tools/secret-provider-definitions/registry"; +import { useAtomValue } from "@effect/atom-react"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { KeyRoundIcon, TypeIcon } from "lucide-react"; +import { useState, type ComponentProps, type ComponentType, type ReactNode } from "react"; + +import { isSecretsFeatureEnabled } from "../../env"; +import { useEnvironmentSettings } from "../../hooks/useSettings"; +import { OnePasswordIcon } from "../Icons"; +import { serverEnvironment } from "../../state/server"; +import { DraftInput } from "../ui/draft-input"; +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; +import { SecretInput } from "./SecretInput"; +import { classifySecretProvider } from "./SecretProviderDialog.logic"; + +export interface SecretSource { + readonly id: SecretProviderId; + readonly kind: SecretProviderKind; + readonly label: string; +} + +const NO_SECRET_SOURCES: ReadonlyArray = []; +const STORED_VALUE = "__stored__"; +const decodeSecretReference = Schema.decodeUnknownOption(SecretReference); + +/** Secret providers a setting can point at; empty when the environment cannot read references. */ +export function useSecretSources(environmentId: EnvironmentId): ReadonlyArray { + const providers = useEnvironmentSettings(environmentId, (settings) => settings.secretProviders); + const supported = useAtomValue( + serverEnvironment.configValueAtom(environmentId), + (config) => config?.secretReferences === true, + ); + if (!isSecretsFeatureEnabled || !supported) return NO_SECRET_SOURCES; + return (Object.entries(providers) as Array<[SecretProviderId, SecretProviderConfig]>) + .filter(([, config]) => classifySecretProvider(config).kind !== "unsupported") + .map(([id, config]) => ({ id, kind: config.kind, label: config.label })); +} + +function SourceLabel(props: { + readonly icon: ComponentType<{ readonly className?: string }>; + readonly label: string; +}) { + return ( + + + {props.label} + + ); +} + +type SecretSourceInputProps = Omit, "value" | "children"> & { + readonly value: string | SecretReference; + readonly sources: ReadonlyArray; + readonly onSecretChange: (secret: SecretReference | undefined) => void; + /** Places the parts in the caller's layout; `source` is null when there is nothing to pick. */ + readonly children?: (parts: { + readonly source: ReactNode; + readonly value: ReactNode; + }) => ReactNode; +}; + +/** A secret entered here, or read from a secret provider when one is configured. */ +export function SecretSourceInput({ + value, + sources, + onSecretChange, + children, + ...rest +}: SecretSourceInputProps) { + // A provider picked here applies once a reference for it is committed. + const [pendingProvider, setPendingProvider] = useState(null); + const [rejectedReference, setRejectedReference] = useState(null); + const secret = typeof value === "string" ? undefined : value; + const input = { ...rest, value: typeof value === "string" ? value : "" }; + const provider = pendingProvider ?? secret?.provider ?? null; + const reference = secret !== undefined && secret.provider === provider ? secret.reference : ""; + const providerSource = sources.find((source) => source.id === provider); + const definition = providerSource && builtInSecretProviderDefinitions.find(providerSource.kind); + const label = input["aria-label"] ?? "Secret"; + const providerLabel = (id: SecretProviderId) => + sources.find((source) => source.id === id)?.label ?? id; + const providerIcon = (id: SecretProviderId) => + sources.some((source) => source.id === id) ? OnePasswordIcon : KeyRoundIcon; + + const source = + sources.length === 0 && provider === null ? null : ( + + ); + + const valueField = + provider === null ? ( + + ) : ( +
+ { + if (next.trim().length === 0) { + setRejectedReference(null); + if (pendingProvider === null && secret !== undefined) onSecretChange(undefined); + return; + } + const decoded = decodeSecretReference({ provider, reference: next }); + if ( + Option.isNone(decoded) || + definition?.isReference(decoded.value.reference) === false + ) { + setRejectedReference(next); + return; + } + setRejectedReference(null); + setPendingProvider(null); + onSecretChange(decoded.value); + }} + autoComplete="off" + placeholder={definition?.referencePlaceholder} + spellCheck={false} + aria-label={`${label} reference`} + aria-invalid={rejectedReference !== null} + /> + {rejectedReference !== null ? ( +

+ {definition?.invalidReferenceMessage ?? "Enter a valid secret reference."} +

+ ) : null} +
+ ); + + if (children !== undefined) return children({ source, value: valueField }); + if (source === null) return valueField; + return ( +
+
{source}
+ {valueField} +
+ ); +} diff --git a/apps/web/src/components/sidebar/SidebarChrome.tsx b/apps/web/src/components/sidebar/SidebarChrome.tsx index e975af3b56dd6..ccd54c8827065 100644 --- a/apps/web/src/components/sidebar/SidebarChrome.tsx +++ b/apps/web/src/components/sidebar/SidebarChrome.tsx @@ -105,7 +105,7 @@ export function SidebarBrandWidthProbe({ function SidebarBrand({ onBackdrop }: { onBackdrop: boolean }) { return ( ` or set `CHECK_CHANGED_CONCURRENCY` to +change the limit (`1` runs the steps one by one). A step that runs past `--timeout ` or +`CHECK_CHANGED_TIMEOUT` (default 900) is killed with every process it started and reported as +`TIMEOUT`, not as a failing check. When the files are exactly those of the last passing run, it prints that run's summary instead of running again. Any change to a file, the install, or a generated file runs the checks again. diff --git a/package.json b/package.json index 1c4ffa679029c..c78b879301b9e 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "report:renders": "node scripts/render-metrics.ts", "lint": "vp lint --report-unused-disable-directives", "knip": "knip --preprocessor ./scripts/knip-schemas.ts", - "knip:check": "knip --include files,dependencies --no-config-hints && knip --workspace apps/server --workspace apps/desktop --workspace apps/web --workspace packages/client-runtime --workspace packages/contracts --workspace packages/effect-acp --workspace packages/effect-codex-app-server --workspace packages/shared --workspace packages/ssh --workspace packages/tailscale --workspace packages/provider-core --workspace packages/provider-acp --workspace packages/provider-acp-registry --workspace packages/provider-testing --workspace packages/provider-pi --workspace packages/provider-muse --workspace packages/provider-opencode --workspace packages/provider-cursor --workspace packages/provider-fx --workspace packages/provider-grok --workspace packages/source-control-core --workspace packages/source-control-definitions --workspace packages/source-control-azure-devops --workspace packages/source-control-bitbucket --workspace packages/source-control-forgejo --workspace packages/source-control-gitcafe --workspace packages/source-control-github --workspace packages/source-control-gitlab --workspace packages/source-control-testing --exports --preprocessor ./scripts/knip-schemas.ts --no-config-hints", + "knip:check": "knip --include files,dependencies --no-config-hints && knip --workspace apps/server --workspace apps/desktop --workspace apps/web --workspace packages/client-runtime --workspace packages/contracts --workspace packages/effect-acp --workspace packages/effect-codex-app-server --workspace packages/shared --workspace packages/ssh --workspace packages/tailscale --workspace packages/provider-core --workspace packages/provider-acp --workspace packages/provider-acp-registry --workspace packages/provider-testing --workspace packages/provider-pi --workspace packages/provider-muse --workspace packages/provider-opencode --workspace packages/provider-cursor --workspace packages/provider-fx --workspace packages/provider-grok --workspace packages/secret-provider-core --workspace packages/secret-provider-definitions --workspace packages/secret-provider-onepassword --workspace packages/source-control-core --workspace packages/source-control-definitions --workspace packages/source-control-azure-devops --workspace packages/source-control-bitbucket --workspace packages/source-control-forgejo --workspace packages/source-control-gitcafe --workspace packages/source-control-github --workspace packages/source-control-gitlab --workspace packages/source-control-testing --exports --preprocessor ./scripts/knip-schemas.ts --no-config-hints", "knip:production": "knip --production --preprocessor ./scripts/knip-schemas.ts", "lint:mobile": "node scripts/mobile-native-static-check.ts", "mobile:install": "node scripts/mobile-native-client.ts ensure", diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index bac573bdca85b..11fe1cc695ad5 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -207,6 +207,10 @@ "types": "./src/state/orchestration.ts", "default": "./src/state/orchestration.ts" }, + "./state/thread-history-merge": { + "types": "./src/state/threadHistoryMerge.ts", + "default": "./src/state/threadHistoryMerge.ts" + }, "./state/orchestration-v2-projection": { "types": "./src/state/orchestrationV2Projection.ts", "default": "./src/state/orchestrationV2Projection.ts" diff --git a/packages/client-runtime/src/state/orchestration.ts b/packages/client-runtime/src/state/orchestration.ts index 4e6d3022b4812..5f5c2ce3020ac 100644 --- a/packages/client-runtime/src/state/orchestration.ts +++ b/packages/client-runtime/src/state/orchestration.ts @@ -107,6 +107,10 @@ export function createOrchestrationEnvironmentAtoms( label: "environment-data:orchestration:stop-workflow", tag: ORCHESTRATION_V2_WS_METHODS.stopWorkflow, }), + forceStopThread: createEnvironmentRpcCommand(runtime, { + label: "environment-data:orchestration:force-stop-thread", + tag: ORCHESTRATION_V2_WS_METHODS.forceStopThread, + }), turnItem: createEnvironmentRpcQueryAtomFamily(runtime, { label: "environment-data:orchestration:turn-item", tag: ORCHESTRATION_V2_WS_METHODS.getTurnItem, diff --git a/packages/client-runtime/src/state/threadDetail.ts b/packages/client-runtime/src/state/threadDetail.ts index e1f30998caefb..3819c5d92ba5e 100644 --- a/packages/client-runtime/src/state/threadDetail.ts +++ b/packages/client-runtime/src/state/threadDetail.ts @@ -14,6 +14,7 @@ import { type ThreadQueueWorkflowState, } from "./threadWorkflows.ts"; import { deriveThreadTurnSubagents, type ThreadTurnSubagents } from "./threadSubagents.ts"; +import { presentRunStop, type RunStopPresentation } from "./threadExecution.ts"; import { deriveThreadCheckpointSummaries, type ThreadCheckpointSummary, @@ -294,6 +295,32 @@ export function createEnvironmentThreadDetailAtoms( return value; }).pipe(Atom.setIdleTTL(0), Atom.withLabel(`environment-thread-queue:${key}`)); }); + const runStopAtomFamily = Atom.family((key: string) => { + let previous: Pick | null = null; + let stop: RunStopPresentation | null = null; + return Atom.make((get) => { + const projection = Option.getOrNull(get(threadStateValueAtomFamily(key)).data); + if (projection === null) { + previous = null; + stop = null; + } else if ( + projection.runs !== previous?.runs || + projection.turnItems !== previous?.turnItems + ) { + const { runs, turnItems } = projection; + previous = { runs, turnItems }; + const next = presentRunStop(previous); + if ( + next?.state !== stop?.state || + next?.runId !== stop?.runId || + next?.requestedAt !== stop?.requestedAt || + next?.background !== stop?.background + ) + stop = next; + } + return stop; + }).pipe(Atom.setIdleTTL(0), Atom.withLabel(`environment-thread-run-stop:${key}`)); + }); const queuedCountAtomFamily = Atom.family((key: string) => { let previous: Pick | null = null; let count = 0; @@ -435,6 +462,7 @@ export function createEnvironmentThreadDetailAtoms( pendingRequestsAtom: (ref: ScopedThreadRef) => pendingRequestsAtomFamily(threadKey(ref)), queueWorkflowAtom: (ref: ScopedThreadRef) => queueWorkflowAtomFamily(threadKey(ref)), queuedCountAtom: (ref: ScopedThreadRef) => queuedCountAtomFamily(threadKey(ref)), + runStopAtom: (ref: ScopedThreadRef) => runStopAtomFamily(threadKey(ref)), turnSubagentsAtom: (ref: ScopedThreadRef) => turnSubagentsAtomFamily(threadKey(ref)), checkpointSummariesAtom: (ref: ScopedThreadRef) => checkpointSummariesAtomFamily(threadKey(ref)), diff --git a/packages/client-runtime/src/state/threadExecution.test.ts b/packages/client-runtime/src/state/threadExecution.test.ts index 8ef37f959cd40..3f8ef2a7f9d31 100644 --- a/packages/client-runtime/src/state/threadExecution.test.ts +++ b/packages/client-runtime/src/state/threadExecution.test.ts @@ -25,6 +25,7 @@ import { deriveRunlessWorkStartedAt, deriveThreadActivityRun, deriveThreadRuntime, + presentRunStop, threadRuntimeHasInterruptibleRun, } from "./threadExecution.ts"; @@ -259,6 +260,74 @@ describe("thread execution presentation", () => { }); }); +describe("presentRunStop", () => { + const signal = (type: "run_interrupt_request" | "run_interrupt_result", runId: RunId) => ({ + id: TurnItemId.make(`${type}:${runId}`), + threadId: v2Projection.thread.id, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: type === "run_interrupt_result" ? ("interrupted" as const) : ("completed" as const), + title: "Interrupt", + startedAt: now, + completedAt: now, + updatedAt: now, + type, + message: "Stopped by user", + }); + const running = run("run-stopping", 1, "running"); + const request = signal("run_interrupt_request", running.id); + + it.each([ + { name: "no Stop", runs: [running], turnItems: [], state: null }, + { + name: "a Stop the agent is winding down", + runs: [running], + turnItems: [request], + state: "stopping", + }, + { + name: "a Stop the provider never confirmed", + runs: [running], + turnItems: [{ ...request, unresponsiveAt: now }], + state: "unresponsive", + }, + { + name: "a Stop that ended the run", + runs: [running], + turnItems: [request, signal("run_interrupt_result", running.id)], + state: null, + }, + { + name: "a settled run's background work still stopping", + runs: [{ ...running, status: "completed" as const, completedAt: now }], + turnItems: [request], + state: null, + }, + { + name: "a settled run whose background work ignored Stop", + runs: [{ ...running, status: "completed" as const, completedAt: now }], + turnItems: [{ ...request, unresponsiveAt: now }], + state: "unresponsive", + }, + { + name: "a settled run whose background work ignored Stop, with a queued follow-up", + runs: [ + { ...running, status: "completed" as const, completedAt: now }, + run("run-queued", 2, "queued"), + ], + turnItems: [{ ...request, unresponsiveAt: now }], + state: "unresponsive", + }, + ])("presents $name", ({ runs, turnItems, state }) => { + expect(presentRunStop({ runs, turnItems })?.state ?? null).toBe(state); + }); +}); + describe("deriveRunlessWorkStartedAt", () => { const later = DateTime.makeUnsafe("2026-07-28T10:05:00.000Z"); const rootTurn = ( diff --git a/packages/client-runtime/src/state/threadExecution.ts b/packages/client-runtime/src/state/threadExecution.ts index ba87930bebbd5..52d07babfbb61 100644 --- a/packages/client-runtime/src/state/threadExecution.ts +++ b/packages/client-runtime/src/state/threadExecution.ts @@ -294,6 +294,55 @@ const BACKGROUND_WORK_KINDS: Record< background_task: { order: 3, singular: "background task", plural: "background tasks" }, }; +export interface RunStopPresentation { + /** `unresponsive`: the provider did not confirm the stop in time, so Force stop is offered. */ + readonly state: "stopping" | "unresponsive"; + readonly runId: ThreadStructure["runs"][number]["id"]; + readonly requestedAt: string; + /** The run already ended; the Stop targeted its background work. */ + readonly background: boolean; +} + +/** + * A Stop the provider has not finished yet. It stays `stopping` while the agent winds down, + * since some providers confirm the request before the turn actually ends. On a run that + * already ended, only an unanswered Stop of its background work counts. + */ +export function presentRunStop( + projection: Pick, +): RunStopPresentation | null { + let active: ThreadStructure["runs"][number] | null = null; + let latest: ThreadStructure["runs"][number] | null = null; + for (const candidate of projection.runs) { + // Queued follow-ups never ran, so they hold no Stop of their own. + if (candidate.status !== "queued" && candidate.ordinal > (latest?.ordinal ?? -1)) + latest = candidate; + if ( + INTERRUPTIBLE_RUN_STATUSES.has(candidate.status) && + candidate.ordinal > (active?.ordinal ?? -1) + ) + active = candidate; + } + const run = active ?? latest; + if (run === null) return null; + let request: + | Extract + | undefined; + for (const item of projection.turnItems) { + if (item.runId !== run.id) continue; + if (item.type === "run_interrupt_result") return null; + if (item.type === "run_interrupt_request") request = item; + } + if (request === undefined || (active === null && request.unresponsiveAt === undefined)) + return null; + return { + state: request.unresponsiveAt === undefined ? "stopping" : "unresponsive", + runId: run.id, + requestedAt: DateTime.formatIso(request.startedAt ?? request.updatedAt), + background: active === null, + }; +} + export interface PendingBackgroundWorkItem { readonly taskId: string; readonly kind: BackgroundWorkKind; diff --git a/packages/contracts/src/clientRpcPermissions.ts b/packages/contracts/src/clientRpcPermissions.ts index f9564b2e2441c..d3139f2eeff24 100644 --- a/packages/contracts/src/clientRpcPermissions.ts +++ b/packages/contracts/src/clientRpcPermissions.ts @@ -60,6 +60,8 @@ export const CLIENT_GUARDED_RPC_SCOPES = { [WS_METHODS.scheduledTasksRunNow]: AuthOrchestrationOperateScope, [WS_METHODS.scheduledTasksRotateWebhookToken]: AuthOrchestrationOperateScope, [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: AuthOrchestrationOperateScope, + // Force stop kills agent processes, as Stop on a terminal does. + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: AuthOrchestrationOperateScope, // A guest link hands thread access to someone outside, like a pairing link. [WS_METHODS.guestShareCreate]: AuthAccessWriteScope, [WS_METHODS.peerLinksLink]: AuthAccessWriteScope, diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index f93fa2b9fd06e..cb6023cf56e1d 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -99,6 +99,8 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ /** Hide the switch when absent; older servers silently drop per-project provider patches. */ projectProviderInstanceOverrides: Schema.optionalKey(Schema.Boolean), threadSnooze: Schema.optionalKey(Schema.Boolean), + /** Serves `orchestration.forceStopThread`; hide Force stop where it is absent. */ + threadForceStop: Schema.optionalKey(Schema.Boolean), /** Every protocol-2 server sends this; older mobile builds still read it. */ environmentThemes: Schema.optionalKey(Schema.Boolean), usageLimitSources: Schema.optionalKey(Schema.Boolean), diff --git a/packages/contracts/src/guestShare.ts b/packages/contracts/src/guestShare.ts index fd991fe1e4c3b..715a837daeae3 100644 --- a/packages/contracts/src/guestShare.ts +++ b/packages/contracts/src/guestShare.ts @@ -129,6 +129,11 @@ export const guestSessionCookieName = (shareId: string) => `t3_guest_${shareId}` export const GUEST_RENDER_ROUTE = `${GUEST_ROUTE_PREFIX}/render`; /** One item of the shared thread, as an expanded row loads it: `?share=&item=`. */ export const GUEST_ITEM_ROUTE = `${GUEST_ROUTE_PREFIX}/item`; +/** + * Older turns of the shared thread, as the owner's "Load earlier turns" pages them: + * `?share=&cursor=`. + */ +export const GUEST_HISTORY_ROUTE = `${GUEST_ROUTE_PREFIX}/history`; /** Server-sent output frames of one command in the shared thread: `?share=&item=`. */ export const GUEST_COMMAND_OUTPUT_ROUTE = `${GUEST_ROUTE_PREFIX}/command-output`; /** diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 59fd35159ba79..88ee595c5530d 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -70,4 +70,5 @@ export * from "./rpc.ts"; export * from "./worktreeSetup.ts"; export * from "./serverPush.ts"; export * from "./secretRequest.ts"; +export * from "./secretReference.ts"; export * from "./clientRpcPermissions.ts"; diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 026e74831233d..d20da5780255b 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1676,6 +1676,8 @@ export const OrchestrationV2TurnItem = Schema.Union([ ...OrchestrationV2TurnItemBaseFields, type: Schema.Literal("run_interrupt_request"), message: Schema.String, + /** Set when the provider did not confirm the stop in time; clients then offer Force stop. */ + unresponsiveAt: Schema.optionalKey(Schema.DateTimeUtc), }), Schema.Struct({ ...OrchestrationV2TurnItemBaseFields, @@ -2500,6 +2502,7 @@ export const OrchestrationV2TurnItemJson = Schema.Union([ ...OrchestrationV2TurnItemJsonBaseFields, type: Schema.Literal("run_interrupt_request"), message: Schema.String, + unresponsiveAt: Schema.optionalKey(Schema.DateTimeUtcFromString), }), Schema.Struct({ ...OrchestrationV2TurnItemJsonBaseFields, @@ -3436,12 +3439,22 @@ const OrchestrationV2InternalCommand = Schema.Union([ threadId: ThreadId, requestId: RuntimeRequestId, }), + /** The provider did not confirm a Stop of this turn in time. */ + Schema.Struct({ + type: Schema.Literal("thread.interrupt.unresponsive"), + commandId: CommandId, + threadId: ThreadId, + providerThreadId: ProviderThreadId, + providerTurnId: ProviderTurnId, + }), Schema.Struct({ type: Schema.Literal("thread.background-work.settle"), commandId: CommandId, threadId: ThreadId, providerThreadId: ProviderThreadId, providerTurnId: ProviderTurnId, + /** Replaces the Stop's own reason on the result, as Force stop does. */ + reason: Schema.optional(Schema.String), subagent: Schema.optional( Schema.Struct({ id: NodeId, @@ -3521,6 +3534,7 @@ export const ORCHESTRATION_V2_WS_METHODS = { getThreadProjection: "orchestration.getThreadProjection", getWorkflowScript: "orchestration.getWorkflowScript", stopWorkflow: "orchestration.stopWorkflow", + forceStopThread: "orchestration.forceStopThread", getTurnItem: "orchestration.getTurnItem", launchThread: "orchestration.launchThread", subscribeShell: "orchestration.subscribeShell", @@ -3908,6 +3922,41 @@ export class OrchestrationV2StopWorkflowError extends Schema.TaggedError()( + "OrchestrationV2ForceStopThreadError", + { + threadId: ThreadId, + /** `superseded`: a newer run started since the user asked; nothing was stopped. */ + reason: Schema.optional(Schema.Literal("superseded")), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return this.reason === "superseded" + ? "A new turn started, so nothing was force stopped." + : "Failed to force stop the agent."; + } +} + export const OrchestrationV2GetWorkflowScriptInput = Schema.Struct({ threadId: ThreadId, scriptPath: TrimmedNonEmptyString, @@ -4094,6 +4143,10 @@ export const OrchestrationV2RpcSchemas = { input: OrchestrationV2StopWorkflowInput, output: Schema.Void, }, + forceStopThread: { + input: OrchestrationV2ForceStopThreadInput, + output: OrchestrationV2ForceStopThreadResult, + }, getWorkflowScript: { input: OrchestrationV2GetWorkflowScriptInput, output: OrchestrationV2GetWorkflowScriptResult, diff --git a/packages/contracts/src/providerInstance.ts b/packages/contracts/src/providerInstance.ts index 9cc0a3f52375d..61fee930c3193 100644 --- a/packages/contracts/src/providerInstance.ts +++ b/packages/contracts/src/providerInstance.ts @@ -1,6 +1,8 @@ import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; +import * as SchemaTransformation from "effect/SchemaTransformation"; import { TrimmedNonEmptyString } from "./baseSchemas.ts"; +import { SecretReference } from "./secretReference.ts"; const PROVIDER_SLUG_MAX_CHARS = 64; const PROVIDER_SLUG_PATTERN = /^[a-zA-Z][a-zA-Z0-9_-]*$/; @@ -27,17 +29,48 @@ export const ProviderInstanceEnvironmentVariableName = TrimmedNonEmptyString.che export type ProviderInstanceEnvironmentVariableName = typeof ProviderInstanceEnvironmentVariableName.Type; -export const ProviderInstanceEnvironmentVariable = Schema.Struct({ +const ProviderInstanceEnvironmentVariableValue = Schema.Struct({ + name: ProviderInstanceEnvironmentVariableName, + value: Schema.Union([Schema.String, SecretReference]), + sensitive: Schema.Boolean, + valueRedacted: Schema.optionalKey(Schema.Boolean), +}); + +// A reference is encoded beside an empty value so older servers and clients still decode it. +const StoredProviderInstanceEnvironmentVariable = Schema.Struct({ name: ProviderInstanceEnvironmentVariableName, value: Schema.String.pipe(Schema.withDecodingDefault(Effect.succeed(""))), sensitive: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), valueRedacted: Schema.optionalKey(Schema.Boolean), + secret: Schema.optionalKey(SecretReference), }); + +export const ProviderInstanceEnvironmentVariable = StoredProviderInstanceEnvironmentVariable.pipe( + Schema.decodeTo( + Schema.toType(ProviderInstanceEnvironmentVariableValue), + SchemaTransformation.transform< + typeof ProviderInstanceEnvironmentVariableValue.Type, + typeof StoredProviderInstanceEnvironmentVariable.Type + >({ + decode: ({ secret, ...variable }) => + secret === undefined ? variable : { ...variable, value: secret }, + encode: ({ value, ...variable }) => + typeof value === "string" + ? { ...variable, value } + : { ...variable, value: "", secret: value }, + }), + ), +); export type ProviderInstanceEnvironmentVariable = typeof ProviderInstanceEnvironmentVariable.Type; export const ProviderInstanceEnvironment = Schema.Array(ProviderInstanceEnvironmentVariable); export type ProviderInstanceEnvironment = typeof ProviderInstanceEnvironment.Type; +/** What drivers receive: every secret reference has been read into a plain value. */ +export type ResolvedProviderInstanceEnvironment = ReadonlyArray< + Omit & { readonly value: string } +>; + /** Drivers decode their own config; unknown-driver envelopes round-trip verbatim across versions. */ export const ProviderInstanceConfig = Schema.Struct({ driver: ProviderDriverKind, diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 5a7a33c3c4982..f9ab82d790d6a 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -228,6 +228,7 @@ import { ORCHESTRATION_V2_WS_METHODS, OrchestrationGetWorkflowScriptError, OrchestrationV2StopWorkflowError, + OrchestrationV2ForceStopThreadError, OrchestrationV2DispatchCommandError, OrchestrationV2GetShellSnapshotError, OrchestrationV2CommandOutputError, @@ -649,6 +650,8 @@ export const WsServerRefreshProvidersRpc = Rpc.make(WS_METHODS.serverRefreshProv fresh: Schema.optional(Schema.Boolean), /** Bypass model caches; explicit rediscovery may open sessions, background refreshes must not. */ refreshModels: Schema.optional(Schema.Boolean), + /** Read secret references again; unlocking a vault can prompt on the host, so only explicit user refreshes ask. */ + reloadSecrets: Schema.optional(Schema.Boolean), /** * Reply without snapshots, or with only the requested instance's cwd snapshot. Snapshots then * reach the client only through these replies, so another client's scan shows on its next @@ -1802,6 +1805,12 @@ const WsOrchestrationV2StopWorkflowRpc = Rpc.make(ORCHESTRATION_V2_WS_METHODS.st error: Schema.Union([OrchestrationV2StopWorkflowError, EnvironmentAuthorizationError]), }); +const WsOrchestrationV2ForceStopThreadRpc = Rpc.make(ORCHESTRATION_V2_WS_METHODS.forceStopThread, { + payload: OrchestrationV2RpcSchemas.forceStopThread.input, + success: OrchestrationV2RpcSchemas.forceStopThread.output, + error: Schema.Union([OrchestrationV2ForceStopThreadError, EnvironmentAuthorizationError]), +}); + const WsOrchestrationV2GetWorkflowScriptRpc = Rpc.make( ORCHESTRATION_V2_WS_METHODS.getWorkflowScript, { @@ -2156,6 +2165,7 @@ export const WsRpcGroup = RpcGroup.make( WsOrchestrationV2DispatchCommandRpc, WsOrchestrationV2GetWorkflowScriptRpc, WsOrchestrationV2StopWorkflowRpc, + WsOrchestrationV2ForceStopThreadRpc, WsOrchestrationV2GetTurnItemRpc, WsOrchestrationV2GetTurnDiffRpc, WsOrchestrationV2GetFullThreadDiffRpc, diff --git a/packages/contracts/src/secretReference.ts b/packages/contracts/src/secretReference.ts new file mode 100644 index 0000000000000..4e03f8a675688 --- /dev/null +++ b/packages/contracts/src/secretReference.ts @@ -0,0 +1,27 @@ +import * as Schema from "effect/Schema"; + +import { TrimmedNonEmptyString } from "./baseSchemas.ts"; + +// Separate module avoids an import cycle between settings and provider instance contracts. +// Final once shipped: older builds drop entries whose key fails here, and erase them on their next save. +// Unlike provider slugs it may start with a digit, so a provider named "1Password" fits. +export const SecretProviderSlug = TrimmedNonEmptyString.check( + Schema.isMaxLength(64), + Schema.isPattern(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), +); + +export const SecretProviderId = SecretProviderSlug.pipe(Schema.brand("SecretProviderId")); +export type SecretProviderId = typeof SecretProviderId.Type; + +// Provider-specific syntax is the secret provider driver's to check. +export const SecretReferencePath = TrimmedNonEmptyString.check( + Schema.isMaxLength(1024), + Schema.isPattern(/^[^\r\n]+$/), +); + +/** Points at a secret held by a configured secret provider; the secret itself is never stored. */ +export const SecretReference = Schema.Struct({ + provider: SecretProviderId, + reference: SecretReferencePath, +}); +export type SecretReference = typeof SecretReference.Type; diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 9fd3813abf5fa..ab79a9dea60f5 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -640,6 +640,8 @@ export const ServerConfig = Schema.Struct({ threadFindProgressive: Schema.optionalKey(Schema.Boolean), /** Settings patches may carry `secretProviders`; older servers drop the key without an error. */ secretProviderSettings: Schema.optionalKey(Schema.Boolean), + /** Secret settings may point at a secret provider; older servers drop the reference. */ + secretReferences: Schema.optionalKey(Schema.Boolean), scratchWorkspaceRoot: Schema.optionalKey(TrimmedNonEmptyString), newProjectsRoot: Schema.optionalKey(TrimmedNonEmptyString), /** diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts index fcf7020e0e5d9..e93fa27e86da1 100644 --- a/packages/contracts/src/settings.test.ts +++ b/packages/contracts/src/settings.test.ts @@ -2,7 +2,11 @@ import { describe, expect, it } from "vite-plus/test"; import * as Schema from "effect/Schema"; import { type CommandId, ProjectId } from "./baseSchemas.ts"; -import { ProviderDriverKind, ProviderInstanceId } from "./providerInstance.ts"; +import { + ProviderDriverKind, + ProviderInstanceEnvironment, + ProviderInstanceId, +} from "./providerInstance.ts"; import { ClientSettingsSchema, ClientSettingsPatch, @@ -23,6 +27,14 @@ const decodeClientSettings = Schema.decodeUnknownSync(ClientSettingsSchema); const decodeClientSettingsPatch = Schema.decodeUnknownSync(ClientSettingsPatch); const encodeClientSettings = Schema.encodeSync(ClientSettingsSchema); const decodeServerSettings = Schema.decodeUnknownSync(ServerSettings); +const encodeProviderInstanceEnvironment = Schema.encodeSync(ProviderInstanceEnvironment); +const decodeProviderInstanceEnvironment = Schema.decodeSync(ProviderInstanceEnvironment); +// The environment variable shape builds released before secret references decode. +const decodeStringOnlyEnvironment = Schema.decodeUnknownSync( + Schema.Array( + Schema.Struct({ name: Schema.String, value: Schema.String, sensitive: Schema.Boolean }), + ), +); const decodeServerSettingsPatch = Schema.decodeUnknownSync(ServerSettingsPatch); const encodeServerSettings = Schema.encodeSync(ServerSettings); const encodeServerSettingsPatch = Schema.encodeSync(ServerSettingsPatch); @@ -154,14 +166,45 @@ describe("ServerSettings secret providers", () => { ).toThrow(); }); - it("rejects writes of an unknown kind or an account op could misread", () => { - const write = (entry: unknown) => () => - decodeServerSettingsPatch({ secretProviders: { personal: entry } }); - for (const account of ["", "my account", "--help"]) { - expect(write(onePassword(account))).toThrow(); + it("stores a provider environment secret reference in a layout older builds still decode", () => { + const providerInstances = (secret: unknown) => ({ + codex: { + driver: "codex", + environment: [ + { name: "OPENAI_API_KEY", sensitive: true, value: "", secret }, + { name: "BASE_URL", value: "https://example.test" }, + ], + }, + }); + const secret = { provider: "personal", reference: "op://Private/OpenAI/credential" }; + const environment = decodeServerSettings({ providerInstances: providerInstances(secret) }) + .providerInstances[ProviderInstanceId.make("codex")]?.environment; + expect(environment).toEqual([ + { name: "OPENAI_API_KEY", value: secret, sensitive: true }, + { name: "BASE_URL", value: "https://example.test", sensitive: false }, + ]); + + const encoded = encodeProviderInstanceEnvironment(environment ?? []); + expect(encoded).toEqual([ + { name: "OPENAI_API_KEY", value: "", sensitive: true, secret }, + { name: "BASE_URL", value: "https://example.test", sensitive: false }, + ]); + expect(decodeProviderInstanceEnvironment(encoded)).toEqual(environment); + expect(decodeStringOnlyEnvironment(encoded)).toEqual([ + { name: "OPENAI_API_KEY", value: "", sensitive: true }, + { name: "BASE_URL", value: "https://example.test", sensitive: false }, + ]); + + for (const invalid of [ + { provider: "My Vault", reference: "op://a/b/c" }, + { provider: "personal", reference: " " }, + { provider: "personal", reference: "op://Private/OpenAI\n/credential" }, + { provider: "personal", reference: "x".repeat(1025) }, + ]) { + expect(() => + decodeServerSettings({ providerInstances: providerInstances(invalid) }), + ).toThrow(); } - expect(write({ kind: "one_password", label: "Personal" })).toThrow(); - expect(write({ kind: "openbao", label: "Vault", config: { address: "https://x" } })).toThrow(); }); }); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 3d2a72ba2a987..73633d276c994 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -20,6 +20,7 @@ import { TrimmedString, } from "./baseSchemas.ts"; import { UsageLimitSourceId } from "./usageLimitSourceId.ts"; +import { SecretProviderId, SecretProviderSlug } from "./secretReference.ts"; import { EnvironmentMachineKind, ThreadEnvMode, WorktreeSubmodules } from "./environment.ts"; import { KeybindingShortcut } from "./keybindings.ts"; import { @@ -957,42 +958,11 @@ export const DictationSettings = Schema.Struct({ }); export type DictationSettings = typeof DictationSettings.Type; -// Final once shipped: older builds drop entries whose key fails here, and erase them on their next save. -// Unlike provider slugs it may start with a digit, so a provider named "1Password" fits. -const SecretProviderSlug = TrimmedNonEmptyString.check( - Schema.isMaxLength(64), - Schema.isPattern(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), -); - -export const SecretProviderId = SecretProviderSlug.pipe(Schema.brand("SecretProviderId")); -export type SecretProviderId = typeof SecretProviderId.Type; - export const SecretProviderKind = SecretProviderSlug.pipe(Schema.brand("SecretProviderKind")); export type SecretProviderKind = typeof SecretProviderKind.Type; -const ONE_PASSWORD_ACCOUNT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; - -/** - * Any value `op --account` accepts: shorthand, sign-in address, account ID, or user ID. - * It is passed to the CLI as an argument, so it can never start with `-`. - */ -export const OnePasswordAccount = Schema.String.check( - Schema.isMaxLength(253), - Schema.makeFilter((value: string) => - ONE_PASSWORD_ACCOUNT_PATTERN.test(value) - ? true - : "1Password account must be a shorthand, sign-in address, or ID with no spaces.", - ), -).pipe(Schema.brand("OnePasswordAccount")); -export type OnePasswordAccount = typeof OnePasswordAccount.Type; - export const SecretProviderLabel = TrimmedNonEmptyString.check(Schema.isMaxLength(64)); -export const OnePasswordSecretProviderConfig = Schema.Struct({ - account: OnePasswordAccount, -}); -export type OnePasswordSecretProviderConfig = typeof OnePasswordSecretProviderConfig.Type; - /** Kinds decode their own config; unknown-kind envelopes round-trip verbatim across versions. */ export const SecretProviderConfig = Schema.Struct({ kind: SecretProviderKind, @@ -1001,31 +971,12 @@ export const SecretProviderConfig = Schema.Struct({ }); export type SecretProviderConfig = typeof SecretProviderConfig.Type; -/** Kinds this build can resolve. Their `config` holds no credentials, so clients may read it. */ -const SECRET_PROVIDER_KIND_CONFIG_GUARDS: Readonly boolean>> = { - one_password: Schema.is(OnePasswordSecretProviderConfig), -}; - -export function isKnownSecretProviderKind(kind: string): boolean { - return Object.hasOwn(SECRET_PROVIDER_KIND_CONFIG_GUARDS, kind); -} - -// Limits that may change later are enforced on writes only, never on the stored envelope. -// Writes must be a kind this build knows with a config it accepts, so an older server -// rejects a newer kind instead of storing what it cannot check. +// Limits that may change later are enforced on writes only, never on the stored envelope. The +// server also checks the kind and its config against the secret providers it has loaded. const SecretProviderConfigWrite = Schema.Struct({ ...SecretProviderConfig.fields, label: SecretProviderLabel, -}).check( - Schema.makeFilter((entry) => { - if (!isKnownSecretProviderKind(entry.kind)) { - return `Unsupported secret provider kind: ${entry.kind}.`; - } - return SECRET_PROVIDER_KIND_CONFIG_GUARDS[entry.kind]!(entry.config) - ? true - : `Invalid ${entry.kind} secret provider config.`; - }), -); +}); export const SourceControlHostSettings = Schema.Record(Schema.String, Schema.Unknown); export type SourceControlHostSettings = typeof SourceControlHostSettings.Type; @@ -1628,6 +1579,7 @@ export const ServerSettingsOperation = Schema.Literals([ "remove-secret", "remove-stale-secret", "write-secret", + "write-secret-provider", "write-file", "prepare-directory", ]); diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index 13f0cbbc240d5..c895abe3f5ef8 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -1324,6 +1324,8 @@ interface SnapshotMessageState { loadingIndex: number; } +/** How long a Stop waits for the agent to confirm before the user is offered Force stop. */ +const ACP_STOP_CONFIRM_TIMEOUT_MS = 10_000; const TOOL_UPDATE_PERSIST_EVERY = 10; // Agents can resend whole calls while streaming arguments. @@ -7989,7 +7991,9 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( return yield* Effect.uninterruptibleMask((restore) => Effect.gen(function* () { yield* restore(runtimeTransitionPermit.take(1)); - return yield* Effect.gen(function* () { + // Stop's caller times out on this call. The transition runs in its own fiber so + // a stalled hard teardown keeps its state and permit without holding the caller. + const transitionFiber = yield* Effect.gen(function* () { const transition = yield* Effect.gen(function* () { const interruptContext = yield* Ref.get(activeTurn); if ( @@ -8000,8 +8004,12 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ) { // ACP cancel leaves native children alive; close confirms session-wide Stop. pendingStopCloseTurn = interruptContext; - const closed = yield* runtime.closeSession().pipe( - Effect.timeoutOption("10 seconds"), + // Interruptible, or neither this timeout nor the caller's could end a hung close. + const closed = yield* restore( + runtime + .closeSession() + .pipe(Effect.timeoutOption(ACP_STOP_CONFIRM_TIMEOUT_MS)), + ).pipe( Effect.mapError( (cause) => new ProviderAdapter.ProviderAdapterProtocolError({ @@ -8013,10 +8021,11 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ), ); if (Option.isNone(closed)) { - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({ driver, - detail: - "The agent did not confirm session close; background work may still run", + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + timeoutMs: ACP_STOP_CONFIRM_TIMEOUT_MS, }); } yield* runtimeCallbackPermit.withPermit( @@ -8246,15 +8255,16 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( } } const stopped = yield* Deferred.await(context.completed).pipe( - Effect.timeoutOption("10 seconds"), + Effect.timeoutOption(ACP_STOP_CONFIRM_TIMEOUT_MS), ); + // The turn stays open: the agent may still be working, and the user + // decides whether to Force stop it. if (Option.isNone(stopped)) { - if (!context.finalized) { - yield* finalizeTurn(context, "interrupted"); - } - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({ driver, - detail: `ACP provider turn ${turnInput.providerTurnId} did not acknowledge cancellation before the interrupt timeout`, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + timeoutMs: ACP_STOP_CONFIRM_TIMEOUT_MS, }); } }); @@ -8275,20 +8285,29 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( }), ), ); - }).pipe(Effect.ensuring(runtimeTransitionPermit.release(1))); + }).pipe( + Effect.ensuring(runtimeTransitionPermit.release(1)), + Effect.forkIn(sessionScope, { startImmediately: true, uninterruptible: true }), + ); + return yield* restore(Fiber.join(transitionFiber)).pipe( + Effect.onInterrupt(() => + Fiber.interrupt(transitionFiber).pipe(Effect.forkDetach, Effect.asVoid), + ), + ); }), ); }, (effect, turnInput) => effect.pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterInterruptError({ - driver, - providerThreadId: turnInput.providerThread.id, - providerTurnId: turnInput.providerTurnId, - cause, - }), + Effect.mapError((cause) => + cause._tag === "ProviderAdapterInterruptTimeoutError" + ? cause + : new ProviderAdapter.ProviderAdapterInterruptError({ + driver, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + cause, + }), ), ), ), diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index e168512736c7c..0b5459f647f8d 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -327,6 +327,21 @@ export class ProviderAdapterInterruptError extends Schema.TaggedError()( + "ProviderAdapterInterruptTimeoutError", + { + driver: ProviderDriverKind, + providerThreadId: ProviderThreadId, + providerTurnId: ProviderTurnId, + timeoutMs: Schema.Number, + }, +) { + override get message(): string { + return `The ${this.driver} provider did not confirm stopping turn ${this.providerTurnId} within ${this.timeoutMs} ms.`; + } +} + export class ProviderAdapterRuntimeRequestResponseError extends Schema.TaggedError()( "ProviderAdapterRuntimeRequestResponseError", { @@ -388,6 +403,7 @@ export const ProviderAdapterV2Error = Schema.Union([ ProviderAdapterSteerRunUnsupportedError, ProviderAdapterSteerRunError, ProviderAdapterInterruptError, + ProviderAdapterInterruptTimeoutError, ProviderAdapterRuntimeRequestResponseError, ProviderAdapterEventStreamError, ProviderAdapterProtocolError, diff --git a/packages/provider-core/src/server/driver.ts b/packages/provider-core/src/server/driver.ts index d236a98fdcba2..a1771e2e748e9 100644 --- a/packages/provider-core/src/server/driver.ts +++ b/packages/provider-core/src/server/driver.ts @@ -6,7 +6,7 @@ import type { AcpRegistryOperationError, AcpRegistrySetProviderInput, ProviderDriverKind, - ProviderInstanceEnvironment, + ResolvedProviderInstanceEnvironment, ProviderInstanceId, ServerProvider, ServerProviderWorkspaceSnapshot, @@ -96,7 +96,7 @@ export interface ProviderDriverCreateInput { readonly instanceId: ProviderInstanceId; readonly displayName: string | undefined; readonly accentColor?: string | undefined; - readonly environment: ProviderInstanceEnvironment; + readonly environment: ResolvedProviderInstanceEnvironment; readonly enabled: boolean; readonly config: Config; } diff --git a/packages/provider-core/src/server/instanceEnvironment.test.ts b/packages/provider-core/src/server/instanceEnvironment.test.ts index 1899d1f9d5c3f..7f70cdddad573 100644 --- a/packages/provider-core/src/server/instanceEnvironment.test.ts +++ b/packages/provider-core/src/server/instanceEnvironment.test.ts @@ -1,5 +1,6 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; +import { SecretProviderId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Path from "effect/Path"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -61,12 +62,21 @@ describe("mergeProviderInstanceEnvironment", () => { [ { name: "OPENROUTER_API_KEY", value: "sk-or-test", sensitive: true }, { name: "ANTHROPIC_API_KEY", value: "", sensitive: false }, + { + name: "OPENAI_API_KEY", + value: { + provider: SecretProviderId.make("personal"), + reference: "op://Private/OpenAI/credential", + }, + sensitive: true, + }, ], - { ANTHROPIC_API_KEY: "inherited", PATH: "/bin" }, + { ANTHROPIC_API_KEY: "inherited", OPENAI_API_KEY: "inherited", PATH: "/bin" }, ), ).toMatchObject({ OPENROUTER_API_KEY: "sk-or-test", ANTHROPIC_API_KEY: "", + OPENAI_API_KEY: "inherited", PATH: "/bin", }); }), diff --git a/packages/provider-core/src/server/instanceEnvironment.ts b/packages/provider-core/src/server/instanceEnvironment.ts index d0cddd119a13f..4a73f678da0c0 100644 --- a/packages/provider-core/src/server/instanceEnvironment.ts +++ b/packages/provider-core/src/server/instanceEnvironment.ts @@ -15,6 +15,8 @@ export const mergeProviderInstanceEnvironment = Effect.fn(function* ( const home = yield* HostProcess.HomeDirectory; const next: NodeJS.ProcessEnv = { ...baseEnv }; for (const variable of environment) { + // An unresolved secret reference carries no value; the registry resolves it for the provider. + if (typeof variable.value !== "string") continue; next[variable.name] = variable.name === "CODEX_HOME" || variable.name === "CLAUDE_CONFIG_DIR" ? expandHomePath(variable.value, home) diff --git a/packages/secret-provider-core/package.json b/packages/secret-provider-core/package.json new file mode 100644 index 0000000000000..3edea7a879b97 --- /dev/null +++ b/packages/secret-provider-core/package.json @@ -0,0 +1,33 @@ +{ + "name": "@t3tools/secret-provider-core", + "private": true, + "type": "module", + "exports": { + "./shared/definition": { + "types": "./src/shared/definition.ts", + "import": "./src/shared/definition.ts" + }, + "./server/driver": { + "types": "./src/server/driver.ts", + "import": "./src/server/driver.ts" + }, + "./server/SecretResolver": { + "types": "./src/server/SecretResolver.ts", + "import": "./src/server/SecretResolver.ts" + } + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vp test run --config ../../vite.config.ts --dir ." + }, + "dependencies": { + "@t3tools/contracts": "workspace:*", + "effect": "catalog:" + }, + "devDependencies": { + "@effect/platform-node": "catalog:", + "@effect/vitest": "catalog:", + "@types/node": "catalog:", + "vite-plus": "catalog:" + } +} diff --git a/packages/secret-provider-core/src/server/SecretResolver.test.ts b/packages/secret-provider-core/src/server/SecretResolver.test.ts new file mode 100644 index 0000000000000..d3c44aa14eaf7 --- /dev/null +++ b/packages/secret-provider-core/src/server/SecretResolver.test.ts @@ -0,0 +1,170 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import { + SecretProviderId, + SecretProviderKind, + type SecretReference, + type ServerSettings, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import { defineSecretProviderDriver } from "./driver.ts"; +import * as SecretResolver from "./SecretResolver.ts"; +import { defineSecretProvider } from "../shared/definition.ts"; + +const FAKE_KIND = SecretProviderKind.make("fake"); +const FakeConfig = Schema.Struct({ account: Schema.String }); + +const personal = SecretProviderId.make("personal"); +const work = SecretProviderId.make("work"); +const fake = (account: string) => ({ + kind: FAKE_KIND, + label: account, + config: { account }, +}); +const ref = (reference: string, provider = personal): SecretReference => ({ provider, reference }); + +function fakeDriver(answer: (account: string, reference: string) => string | undefined) { + const reads: Array<{ readonly account: string; readonly references: ReadonlyArray }> = []; + const driver = defineSecretProviderDriver({ + kind: FAKE_KIND, + definition: defineSecretProvider({ + kind: FAKE_KIND, + label: "Fake", + configSchema: FakeConfig, + isReference: (reference) => reference.startsWith("fake://"), + referencePlaceholder: "fake://item", + invalidReferenceMessage: "Use fake://item.", + }), + read: ({ account }, references) => + Effect.sync(() => { + reads.push({ account, references }); + return references.map((reference) => answer(account, reference)); + }), + }); + return { driver, reads }; +} + +const resolverLayer = ( + driver: ReturnType["driver"], + secretProviders: () => ServerSettings["secretProviders"] = () => ({ [personal]: fake("a") }), +) => + SecretResolver.layer({ + drivers: [driver], + secretProviders: Effect.sync(secretProviders), + }).pipe(Layer.provide(NodeServices.layer)); + +describe("SecretResolver", () => { + it.effect("reads a reference once and holds it", () => { + const { driver, reads } = fakeDriver(() => "sk-live"); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + const token = ref("fake://openai"); + + assert.deepStrictEqual(yield* resolver.read([token]), ["sk-live"]); + assert.deepStrictEqual(yield* resolver.read([token]), ["sk-live"]); + assert.deepStrictEqual(reads, [{ account: "a", references: ["fake://openai"] }]); + }).pipe(Effect.provide(resolverLayer(driver))); + }); + + it.effect("holds a failed read until invalidated, then reads again", () => { + let locked = true; + const { driver, reads } = fakeDriver(() => (locked ? undefined : "sk-live")); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + const token = ref("fake://openai"); + + assert.deepStrictEqual(yield* resolver.read([token]), [undefined]); + locked = false; + assert.deepStrictEqual(yield* resolver.read([token]), [undefined]); + assert.strictEqual(reads.length, 1); + + yield* resolver.invalidate([token]); + assert.deepStrictEqual(yield* resolver.read([token]), ["sk-live"]); + assert.strictEqual(reads.length, 2); + }).pipe(Effect.provide(resolverLayer(driver))); + }); + + it.effect("invalidates every reference when none are named", () => { + const { driver, reads } = fakeDriver(() => "sk-live"); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + const token = ref("fake://openai"); + + yield* resolver.read([token]); + yield* resolver.invalidate(); + yield* resolver.read([token]); + assert.strictEqual(reads.length, 2); + }).pipe(Effect.provide(resolverLayer(driver))); + }); + + it.effect("reads each distinct reference of a provider in one call, in input order", () => { + const { driver, reads } = fakeDriver((_, reference) => reference.toUpperCase()); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + + const values = yield* resolver.read([ref("fake://a"), ref("fake://b"), ref("fake://a")]); + + assert.deepStrictEqual(values, ["FAKE://A", "FAKE://B", "FAKE://A"]); + assert.deepStrictEqual(reads, [{ account: "a", references: ["fake://a", "fake://b"] }]); + }).pipe(Effect.provide(resolverLayer(driver))); + }); + + it.effect("reads each provider separately", () => { + const { driver, reads } = fakeDriver((account, reference) => `${account}:${reference}`); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + + const values = yield* resolver.read([ref("fake://a"), ref("fake://a", work)]); + + assert.deepStrictEqual(values, ["a:fake://a", "b:fake://a"]); + assert.strictEqual(reads.length, 2); + }).pipe( + Effect.provide(resolverLayer(driver, () => ({ [personal]: fake("a"), [work]: fake("b") }))), + ); + }); + + it.effect("reads again when the provider's config changes", () => { + let account = "a"; + const { driver, reads } = fakeDriver((account, reference) => `${account}:${reference}`); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + const token = ref("fake://openai"); + + assert.deepStrictEqual(yield* resolver.read([token]), ["a:fake://openai"]); + account = "b"; + assert.deepStrictEqual(yield* resolver.read([token]), ["b:fake://openai"]); + assert.deepStrictEqual( + reads.map((read) => read.account), + ["a", "b"], + ); + }).pipe(Effect.provide(resolverLayer(driver, () => ({ [personal]: fake(account) })))); + }); + + it.effect("does not call a driver for what it cannot read, and reads it later", () => { + const { driver, reads } = fakeDriver(() => "sk-live"); + const providers: ServerSettings["secretProviders"] = { + [personal]: fake("a"), + [SecretProviderId.make("vault")]: { + kind: SecretProviderKind.make("openbao"), + label: "Vault", + }, + [SecretProviderId.make("broken")]: { kind: FAKE_KIND, label: "Broken", config: {} }, + }; + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + + const values = yield* resolver.read([ + ref("fake://openai", SecretProviderId.make("missing")), + ref("fake://openai", SecretProviderId.make("vault")), + ref("fake://openai", SecretProviderId.make("broken")), + ref("openai"), + ]); + + assert.deepStrictEqual(values, [undefined, undefined, undefined, undefined]); + assert.strictEqual(reads.length, 0); + }).pipe(Effect.provide(resolverLayer(driver, () => providers))); + }); +}); diff --git a/packages/secret-provider-core/src/server/SecretResolver.ts b/packages/secret-provider-core/src/server/SecretResolver.ts new file mode 100644 index 0000000000000..28f5816f953b9 --- /dev/null +++ b/packages/secret-provider-core/src/server/SecretResolver.ts @@ -0,0 +1,125 @@ +// Reads secrets that settings point at through a configured secret provider. Each read, including +// a failed one, is held for the server's life so rebuilds never re-prompt an unlock; `invalidate` +// (the Settings refresh) is the only way to read again. +import type { SecretReference, ServerSettings } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import type { SecretProviderDriver, SecretProviderDriverServices } from "./driver.ts"; + +export class SecretResolver extends Context.Service< + SecretResolver, + { + /** Values in input order; `undefined` marks a reference that could not be read. */ + readonly read: ( + references: ReadonlyArray, + ) => Effect.Effect>; + /** Forgets the given references, or every reference, so the next read asks the provider again. */ + readonly invalidate: (references?: ReadonlyArray) => Effect.Effect; + } +>()("@t3tools/secret-provider-core/server/SecretResolver") {} + +const cacheKey = (reference: SecretReference) => + `${reference.provider}\u0000${reference.reference}`; + +interface CachedSecret { + // A provider whose config changed, such as another account, reads again. + readonly config: string; + readonly value: string | undefined; +} + +const decodeConfigWith = Schema.decodeUnknownOption>; + +interface ReadablePlan { + readonly driver: SecretProviderDriver; + readonly config: unknown; + readonly configKey: string; +} + +export const make = Effect.fn("SecretResolver.make")(function* (options: { + readonly drivers: ReadonlyArray; + readonly secretProviders: Effect.Effect; +}) { + const context = yield* Effect.context(); + const driversByKind = new Map( + options.drivers.map((driver) => [ + driver.kind, + { driver, decodeConfig: decodeConfigWith(driver.definition.configSchema) }, + ]), + ); + const cache = new Map(); + const semaphore = yield* Semaphore.make(1); + + // Only a provider this build can read is cached; anything else fails fast and is checked again + // on the next read. + const planFor = ( + providers: ServerSettings["secretProviders"], + reference: SecretReference, + ): ReadablePlan | undefined => { + const provider = providers[reference.provider]; + const entry = provider === undefined ? undefined : driversByKind.get(provider.kind); + if (provider === undefined || entry === undefined) return undefined; + const { driver, decodeConfig } = entry; + if (!driver.definition.isReference(reference.reference)) return undefined; + const config = decodeConfig(provider.config); + return Option.isNone(config) + ? undefined + : { driver, config: config.value, configKey: JSON.stringify(config.value) }; + }; + + const read: SecretResolver["Service"]["read"] = Effect.fn("SecretResolver.read")( + function* (references) { + const providers = yield* options.secretProviders; + const plans = references.map((reference) => planFor(providers, reference)); + + const wanted = new Map< + string, + { readonly plan: ReadablePlan; readonly byReference: Map> } + >(); + references.forEach((reference, index) => { + const plan = plans[index]; + if (plan === undefined || cache.get(cacheKey(reference))?.config === plan.configKey) return; + const group = wanted.get(reference.provider) ?? { plan, byReference: new Map() }; + group.byReference.set(reference.reference, [ + ...(group.byReference.get(reference.reference) ?? []), + reference, + ]); + wanted.set(reference.provider, group); + }); + for (const { plan, byReference } of wanted.values()) { + const values = yield* plan.driver.read(plan.config, [...byReference.keys()]); + [...byReference.values()].forEach((sameReference, index) => { + for (const reference of sameReference) { + cache.set(cacheKey(reference), { config: plan.configKey, value: values[index] }); + } + }); + } + + return references.map((reference, index) => { + const cached = cache.get(cacheKey(reference)); + return plans[index] !== undefined && cached?.config === plans[index].configKey + ? cached.value + : undefined; + }); + }, + (effect) => semaphore.withPermits(1)(effect), + Effect.provideContext(context), + ); + + const invalidate: SecretResolver["Service"]["invalidate"] = (references) => + semaphore.withPermits(1)( + Effect.sync(() => { + if (references === undefined) cache.clear(); + else for (const reference of references) cache.delete(cacheKey(reference)); + }), + ); + + return SecretResolver.of({ read, invalidate }); +}); + +export const layer = (options: Parameters[0]) => + Layer.effect(SecretResolver, make(options)); diff --git a/packages/secret-provider-core/src/server/driver.ts b/packages/secret-provider-core/src/server/driver.ts new file mode 100644 index 0000000000000..85deb93d8938b --- /dev/null +++ b/packages/secret-provider-core/src/server/driver.ts @@ -0,0 +1,46 @@ +import type { SecretProviderKind } from "@t3tools/contracts"; +import type * as Crypto from "effect/Crypto"; +import * as Effect from "effect/Effect"; +import type * as FileSystem from "effect/FileSystem"; +import * as Schema from "effect/Schema"; +import type { ChildProcessSpawner } from "effect/process"; + +import type { SecretProviderDefinition } from "../shared/definition.ts"; + +/** What the server gives every driver. A driver builds anything else it needs itself. */ +export type SecretProviderDriverServices = + | ChildProcessSpawner.ChildProcessSpawner + | FileSystem.FileSystem + | Crypto.Crypto; + +export interface SecretProviderDriver { + readonly kind: SecretProviderKind; + readonly definition: SecretProviderDefinition; + /** + * Values in input order; `undefined` marks a reference that could not be read. `config` has + * decoded with the definition's `configSchema` and every reference passed its `isReference`. + */ + readonly read: ( + config: unknown, + references: ReadonlyArray, + ) => Effect.Effect, never, SecretProviderDriverServices>; +} + +export const defineSecretProviderDriver = (driver: { + readonly kind: SecretProviderKind; + readonly definition: SecretProviderDefinition; + readonly read: ( + config: Config, + references: ReadonlyArray, + ) => Effect.Effect, never, SecretProviderDriverServices>; +}): SecretProviderDriver => { + const isConfig = Schema.is(driver.definition.configSchema); + return { + kind: driver.kind, + definition: driver.definition, + read: (config, references) => + isConfig(config) + ? driver.read(config, references) + : Effect.succeed(references.map(() => undefined)), + }; +}; diff --git a/packages/secret-provider-core/src/shared/definition.ts b/packages/secret-provider-core/src/shared/definition.ts new file mode 100644 index 0000000000000..7f4320dd0c3ea --- /dev/null +++ b/packages/secret-provider-core/src/shared/definition.ts @@ -0,0 +1,35 @@ +import type { SecretProviderKind } from "@t3tools/contracts"; +import type * as Schema from "effect/Schema"; + +/** What clients and the server both need to know about one kind of secret provider. */ +export interface SecretProviderDefinition { + readonly kind: SecretProviderKind; + readonly label: string; + readonly configSchema: Schema.Codec; + /** Whether `reference` is syntax this kind can read. */ + readonly isReference: (reference: string) => boolean; + readonly referencePlaceholder: string; + readonly invalidReferenceMessage: string; +} + +export const defineSecretProvider = ( + definition: SecretProviderDefinition, +): SecretProviderDefinition => definition; + +export interface SecretProviderDefinitionRegistry { + readonly definitions: ReadonlyArray; + readonly find: (kind: string) => SecretProviderDefinition | undefined; +} + +export function makeSecretProviderDefinitionRegistry( + definitions: ReadonlyArray, +): SecretProviderDefinitionRegistry { + const byKind = new Map(); + for (const definition of definitions) { + if (byKind.has(definition.kind)) { + throw new Error(`Secret provider '${definition.kind}' is defined more than once.`); + } + byKind.set(definition.kind, definition); + } + return { definitions, find: (kind) => byKind.get(kind) }; +} diff --git a/packages/secret-provider-core/tsconfig.json b/packages/secret-provider-core/tsconfig.json new file mode 100644 index 0000000000000..374bac55202dd --- /dev/null +++ b/packages/secret-provider-core/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src"] +} diff --git a/packages/secret-provider-definitions/package.json b/packages/secret-provider-definitions/package.json new file mode 100644 index 0000000000000..5e3592a276030 --- /dev/null +++ b/packages/secret-provider-definitions/package.json @@ -0,0 +1,18 @@ +{ + "name": "@t3tools/secret-provider-definitions", + "private": true, + "type": "module", + "exports": { + "./registry": { + "types": "./src/registry.ts", + "import": "./src/registry.ts" + } + }, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@t3tools/secret-provider-core": "workspace:*", + "@t3tools/secret-provider-onepassword": "workspace:*" + } +} diff --git a/packages/secret-provider-definitions/src/registry.ts b/packages/secret-provider-definitions/src/registry.ts new file mode 100644 index 0000000000000..70df74e113810 --- /dev/null +++ b/packages/secret-provider-definitions/src/registry.ts @@ -0,0 +1,6 @@ +import { makeSecretProviderDefinitionRegistry } from "@t3tools/secret-provider-core/shared/definition"; +import * as OnePassword from "@t3tools/secret-provider-onepassword/shared/definition"; + +export const builtInSecretProviderDefinitions = makeSecretProviderDefinitionRegistry([ + OnePassword.definition, +]); diff --git a/packages/secret-provider-definitions/tsconfig.json b/packages/secret-provider-definitions/tsconfig.json new file mode 100644 index 0000000000000..564a5990051fd --- /dev/null +++ b/packages/secret-provider-definitions/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"] +} diff --git a/packages/secret-provider-onepassword/package.json b/packages/secret-provider-onepassword/package.json new file mode 100644 index 0000000000000..062495c90895f --- /dev/null +++ b/packages/secret-provider-onepassword/package.json @@ -0,0 +1,32 @@ +{ + "name": "@t3tools/secret-provider-onepassword", + "private": true, + "type": "module", + "exports": { + "./shared/definition": { + "types": "./src/shared/definition.ts", + "import": "./src/shared/definition.ts" + }, + "./server/driver": { + "types": "./src/server/driver.ts", + "import": "./src/server/driver.ts" + } + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vp test run --config ../../vite.config.ts --dir ." + }, + "dependencies": { + "@t3tools/contracts": "workspace:*", + "@t3tools/provider-core": "workspace:*", + "@t3tools/secret-provider-core": "workspace:*", + "@t3tools/shared": "workspace:*", + "effect": "catalog:" + }, + "devDependencies": { + "@effect/platform-node": "catalog:", + "@effect/vitest": "catalog:", + "@types/node": "catalog:", + "vite-plus": "catalog:" + } +} diff --git a/packages/secret-provider-onepassword/src/server/driver.test.ts b/packages/secret-provider-onepassword/src/server/driver.test.ts new file mode 100644 index 0000000000000..62675b1d0cbfb --- /dev/null +++ b/packages/secret-provider-onepassword/src/server/driver.test.ts @@ -0,0 +1,155 @@ +// @effect-diagnostics nodeBuiltinImport:off -- the fake `op` reads the template the driver wrote. +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeFileSystem from "@effect/platform-node/NodeFileSystem"; +import { assert, describe, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; +import { ChildProcessSpawner } from "effect/process"; +import * as NodeFS from "node:fs"; + +import { OnePasswordAccount } from "../shared/definition.ts"; +import { driver } from "./driver.ts"; + +const encoder = new TextEncoder(); +const account = OnePasswordAccount.make("my.1password.com"); + +type OpResult = { + readonly stdout?: string; + readonly stderr?: string; + readonly code?: number; + readonly hang?: boolean; +}; + +function fakeOp(answer: (args: ReadonlyArray) => OpResult) { + const invocations: Array> = []; + const spawned = Deferred.makeUnsafe(); + const layer = Layer.mergeAll( + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.sync(() => { + const { args } = command as unknown as { readonly args: ReadonlyArray }; + invocations.push(args); + Deferred.doneUnsafe(spawned, Effect.void); + const result = answer(args); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: result.hang + ? Effect.never + : Effect.succeed(ChildProcessSpawner.ExitCode(result.code ?? 0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.make(encoder.encode(result.stdout ?? "")), + stderr: Stream.make(encoder.encode(result.stderr ?? "")), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ), + ), + NodeFileSystem.layer, + NodeCrypto.layer, + ); + return { layer, invocations, spawned }; +} + +// Answers `op inject` by substituting each `{{ reference }}` in the template file it was handed. +const injectFrom = + (secrets: Readonly>) => + (args: ReadonlyArray): OpResult => { + const template = NodeFS.readFileSync(args[args.indexOf("-i") + 1]!, "utf8"); + return { stdout: template.replace(/\{\{ (\S+) \}\}/g, (_, reference) => secrets[reference]!) }; + }; + +describe("1Password driver", () => { + it.effect("reads one reference from the account", () => { + const op = fakeOp(() => ({ stdout: "sk-live" })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* driver.read({ account }, ["op://Private/OpenAI/credential"]), [ + "sk-live", + ]); + assert.deepStrictEqual(op.invocations, [ + [ + "--cache=false", + "read", + "--account=my.1password.com", + "--no-newline", + "op://Private/OpenAI/credential", + ], + ]); + }).pipe(Effect.provide(op.layer)); + }); + + it.effect("reads a failed reference as undefined", () => { + const op = fakeOp(() => ({ code: 1, stderr: "[ERROR] account is locked" })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* driver.read({ account }, ["op://Private/OpenAI/credential"]), [ + undefined, + ]); + }).pipe(Effect.provide(op.layer)); + }); + + it.effect("reads every reference in a single call", () => { + const op = fakeOp( + injectFrom({ + "op://Private/OpenAI/credential": "sk-a", + "op://Private/Anthropic/key": " sk-b\n", + }), + ); + return Effect.gen(function* () { + const values = yield* driver.read({ account }, [ + "op://Private/OpenAI/credential", + "op://Private/Anthropic/key", + ]); + + assert.deepStrictEqual(values, ["sk-a", " sk-b\n"]); + assert.strictEqual(op.invocations.length, 1); + assert.deepStrictEqual(op.invocations[0]?.slice(0, 3), [ + "--cache=false", + "inject", + "--account=my.1password.com", + ]); + }).pipe(Effect.provide(op.layer)); + }); + + it.effect("falls back to one read at a time when the batch fails", () => { + const op = fakeOp((args) => + args.includes("inject") + ? { code: 1, stderr: '[ERROR] could not resolve "op://Private/Gone/key"' } + : args.at(-1) === "op://Private/Gone/key" + ? { code: 1, stderr: "[ERROR] item not found" } + : { stdout: "sk-a" }, + ); + return Effect.gen(function* () { + const values = yield* driver.read({ account }, [ + "op://Private/OpenAI/credential", + "op://Private/Gone/key", + ]); + + assert.deepStrictEqual(values, ["sk-a", undefined]); + assert.strictEqual(op.invocations.length, 3); + }).pipe(Effect.provide(op.layer)); + }); + + it.effect("does not read one at a time after the batch times out", () => { + const op = fakeOp(() => ({ hang: true })); + return Effect.gen(function* () { + const reading = yield* driver + .read({ account }, ["op://Private/OpenAI/credential", "op://Private/Anthropic/key"]) + .pipe(Effect.forkChild); + yield* Deferred.await(op.spawned); + yield* TestClock.adjust("1 minute"); + + assert.deepStrictEqual(yield* Fiber.join(reading), [undefined, undefined]); + assert.strictEqual(op.invocations.length, 1); + }).pipe(Effect.provide(op.layer)); + }); +}); diff --git a/packages/secret-provider-onepassword/src/server/driver.ts b/packages/secret-provider-onepassword/src/server/driver.ts new file mode 100644 index 0000000000000..abf9d6da5ed88 --- /dev/null +++ b/packages/secret-provider-onepassword/src/server/driver.ts @@ -0,0 +1,112 @@ +import { spawnAndCollect } from "@t3tools/provider-core/server/snapshotProbe"; +import { defineSecretProviderDriver } from "@t3tools/secret-provider-core/server/driver"; +import { resolveSpawnCommand } from "@t3tools/shared/shell"; +import * as Crypto from "effect/Crypto"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import { ChildProcess } from "effect/process"; + +import { definition, type OnePasswordAccount } from "../shared/definition.ts"; + +const ONE_PASSWORD_BINARY = "op"; +// Bounds a read long enough to reach for a fingerprint reader, short enough not to wedge a rebuild. +const READ_TIMEOUT = Duration.seconds(45); + +// `--cache=false` skips the `op daemon`, which hangs on prompts a background server cannot show. +const runOnePassword = Effect.fn("OnePasswordDriver.runOnePassword")(function* ( + args: ReadonlyArray, +) { + const spawnCommand = yield* resolveSpawnCommand(ONE_PASSWORD_BINARY, ["--cache=false", ...args]); + return yield* spawnAndCollect( + ONE_PASSWORD_BINARY, + ChildProcess.make(spawnCommand.command, spawnCommand.args, { shell: spawnCommand.shell }), + ); +}); + +// A secret is passed on verbatim; surrounding whitespace can be part of it. +const nonEmpty = (value: string) => (value.trim().length > 0 ? value : undefined); + +const readOne = Effect.fn("OnePasswordDriver.readOne")(function* ( + account: OnePasswordAccount, + reference: string, +) { + const result = yield* runOnePassword(["read", `--account=${account}`, "--no-newline", reference]); + if (result.code !== 0) { + yield* Effect.logWarning("Could not read secret from 1Password", { + stage: "read", + exitCode: result.code, + }); + return undefined; + } + return nonEmpty(result.stdout); +}); + +// One `op inject` costs one unlock for every reference in the template. `op` ignores piped stdin +// from Node, so the template goes through a temp file. A random separator after every reference +// keeps one secret from shifting the others, and leaves any trailing newline outside the values. +const readTogether = Effect.fn("OnePasswordDriver.readTogether")(function* ( + account: OnePasswordAccount, + references: ReadonlyArray, +) { + const fileSystem = yield* FileSystem.FileSystem; + const separator = `__t3-secret-${yield* (yield* Crypto.Crypto).randomUUIDv4()}__`; + const templatePath = yield* fileSystem.makeTempFileScoped({ prefix: "t3code-op-inject-" }); + yield* fileSystem.writeFileString( + templatePath, + references.map((reference) => `{{ ${reference} }}${separator}`).join(""), + ); + const result = yield* runOnePassword(["inject", `--account=${account}`, "-i", templatePath]); + const values = result.stdout.split(separator); + const trailing = values.pop(); + if ( + result.code !== 0 || + values.length !== references.length || + trailing === undefined || + trailing.trim().length > 0 + ) { + yield* Effect.logWarning("Could not batch-read secrets from 1Password", { + stage: "inject", + exitCode: result.code, + references: references.length, + }); + return undefined; + } + return values.map(nonEmpty); +}, Effect.scoped); + +/** `None` when 1Password did not answer in time; a failed run reads as `Some(undefined)`. */ +const bounded = ( + stage: "read" | "inject", + effect: Effect.Effect, +) => + effect.pipe( + Effect.timeoutOption(READ_TIMEOUT), + Effect.tap((outcome) => + Option.isNone(outcome) + ? Effect.logWarning("1Password did not answer in time", { stage }) + : Effect.void, + ), + Effect.catch((error) => + Effect.logWarning("Could not run 1Password", { stage, error: error._tag }).pipe( + Effect.as(Option.some(undefined)), + ), + ), + ); + +export const driver = defineSecretProviderDriver({ + kind: definition.kind, + definition, + read: Effect.fn("OnePasswordDriver.read")(function* ({ account }, references) { + if (references.length > 1) { + const together = yield* bounded("inject", readTogether(account, references)); + // Reading one by one after a timeout would only wait that long again for each reference. + if (Option.isNone(together)) return references.map(() => undefined); + if (together.value !== undefined) return together.value; + } + return yield* Effect.forEach(references, (reference) => + bounded("read", readOne(account, reference)).pipe(Effect.map(Option.getOrUndefined)), + ); + }), +}); diff --git a/packages/secret-provider-onepassword/src/shared/definition.test.ts b/packages/secret-provider-onepassword/src/shared/definition.test.ts new file mode 100644 index 0000000000000..0daefcf6acca1 --- /dev/null +++ b/packages/secret-provider-onepassword/src/shared/definition.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { definition } from "./definition.ts"; + +describe("1Password definition", () => { + it("accepts vault, item, optional section, and field references", () => { + for (const reference of [ + "op://Private/OpenAI/credential", + "op://Private/OpenAI/Section 1/credential", + "op://Private/OpenAI/credential?attribute=otp", + ]) { + expect(definition.isReference(reference), reference).toBe(true); + } + }); + + it("rejects anything that is not a plain reference", () => { + for (const reference of [ + "--help", + "Private/OpenAI/credential", + "op://Private/OpenAI", + "op://Private/OpenAI/credential }} {{ op://Private/B/c", + "op://Private/OpenAI\n/credential", + `op://Private/OpenAI/${"x".repeat(1024)}`, + ]) { + expect(definition.isReference(reference), reference).toBe(false); + } + }); +}); diff --git a/packages/secret-provider-onepassword/src/shared/definition.ts b/packages/secret-provider-onepassword/src/shared/definition.ts new file mode 100644 index 0000000000000..ff5164b1e00b2 --- /dev/null +++ b/packages/secret-provider-onepassword/src/shared/definition.ts @@ -0,0 +1,41 @@ +import { SecretProviderKind } from "@t3tools/contracts"; +import { defineSecretProvider } from "@t3tools/secret-provider-core/shared/definition"; +import * as Schema from "effect/Schema"; + +const ACCOUNT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +/** + * Any value `op --account` accepts: shorthand, sign-in address, account ID, or user ID. + * It is passed to the CLI as an argument, so it can never start with `-`. + */ +export const OnePasswordAccount = Schema.String.check( + Schema.isMaxLength(253), + Schema.makeFilter((value: string) => + ACCOUNT_PATTERN.test(value) + ? true + : "1Password account must be a shorthand, sign-in address, or ID with no spaces.", + ), +).pipe(Schema.brand("OnePasswordAccount")); +export type OnePasswordAccount = typeof OnePasswordAccount.Type; + +export const OnePasswordSecretProviderConfig = Schema.Struct({ + account: OnePasswordAccount, +}); +export type OnePasswordSecretProviderConfig = typeof OnePasswordSecretProviderConfig.Type; + +// 1Password secret reference syntax: names outside this charset must be written as item ids. +// It also keeps `{{`, `}}`, and line breaks out of the `op inject` template the server builds. +const NAME = String.raw`[A-Za-z0-9._-](?:[A-Za-z0-9 ._-]*[A-Za-z0-9._-])?`; +const REFERENCE = new RegExp( + String.raw`^op://${NAME}(?:/${NAME}){2,3}(?:\?[a-z-]+=[A-Za-z0-9_-]+)?$`, +); +const MAX_REFERENCE_LENGTH = 1024; + +export const definition = defineSecretProvider({ + kind: SecretProviderKind.make("one_password"), + label: "1Password", + configSchema: OnePasswordSecretProviderConfig, + isReference: (reference) => reference.length <= MAX_REFERENCE_LENGTH && REFERENCE.test(reference), + referencePlaceholder: "op://vault/item/field", + invalidReferenceMessage: "Use a 1Password reference like op://vault/item/field.", +}); diff --git a/packages/secret-provider-onepassword/tsconfig.json b/packages/secret-provider-onepassword/tsconfig.json new file mode 100644 index 0000000000000..374bac55202dd --- /dev/null +++ b/packages/secret-provider-onepassword/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src"] +} diff --git a/packages/shared/src/AgentScope.ts b/packages/shared/src/AgentScope.ts index ce2d58c017b25..5f0ffca217172 100644 --- a/packages/shared/src/AgentScope.ts +++ b/packages/shared/src/AgentScope.ts @@ -22,6 +22,12 @@ export interface AgentScopeShape { * descendants outside its process group. Preserves systemd OOM diagnosis. */ readonly stop: (unit: string) => Effect.Effect; + /** + * SIGKILLs every process in the thread's own scope, which works even while + * the kernel throttles them. Shared scopes serve other threads and are never + * killed here. Succeeds with whether a scope was killed. + */ + readonly kill: (threadId: string) => Effect.Effect; /** * Forgets the thread's scope. The session manager calls it before it opens * a provider session, so a scope only answers for the session that launched @@ -42,6 +48,7 @@ export const AgentScope = Context.Reference("@t3tools/shared/Ag wrap: ({ command, args }) => Effect.succeed({ command, args }), oomKilled: () => Effect.succeed(false), stop: () => Effect.void, + kill: () => Effect.succeed(false), clear: () => Effect.void, }), }); diff --git a/packages/shared/src/orchestrationTiming.ts b/packages/shared/src/orchestrationTiming.ts index 187db6c6d97d4..1f43f99453456 100644 --- a/packages/shared/src/orchestrationTiming.ts +++ b/packages/shared/src/orchestrationTiming.ts @@ -22,10 +22,7 @@ export function deriveSubagentElapsedMs( export function formatDuration(durationMs: number): string { if (!Number.isFinite(durationMs) || durationMs < 0) return "0ms"; if (durationMs < 1_000) return `${Math.max(1, Math.round(durationMs))}ms`; - if (durationMs < 10_000) { - const tenths = Math.round(durationMs / 100) / 10; - return tenths >= 10 ? "10s" : `${tenths.toFixed(1)}s`; - } + if (durationMs < 10_000) return `${Math.round(durationMs / 100) / 10}s`; const totalSeconds = Math.round(durationMs / 1_000); // Choose the unit after rounding so 59.5 seconds never renders as "60s". if (totalSeconds < 60) return `${totalSeconds}s`; diff --git a/packages/shared/src/serverSettings.test.ts b/packages/shared/src/serverSettings.test.ts index cb9f6520a6c9d..7decee0ec87c8 100644 --- a/packages/shared/src/serverSettings.test.ts +++ b/packages/shared/src/serverSettings.test.ts @@ -1,6 +1,5 @@ import { DEFAULT_SERVER_SETTINGS, - OnePasswordAccount, ProjectId, ProviderDriverKind, ProviderInstanceId, @@ -590,7 +589,7 @@ describe("serverSettings helpers", () => { const onePassword = (label: string, account: string) => ({ kind: SecretProviderKind.make("one_password"), label, - config: { account: OnePasswordAccount.make(account) }, + config: { account }, }); const current = { ...DEFAULT_SERVER_SETTINGS, diff --git a/packages/source-control-core/src/server/PullRequestProvider.ts b/packages/source-control-core/src/server/PullRequestProvider.ts index 46fc8fd48b0dc..63dfe8db7a85b 100644 --- a/packages/source-control-core/src/server/PullRequestProvider.ts +++ b/packages/source-control-core/src/server/PullRequestProvider.ts @@ -353,6 +353,16 @@ export interface PullRequestProviderApi { input: ProviderRepositoryRef & { readonly number: number }, ) => Effect.Effect; + /** + * For a provider that holds change request reads of its own: called when the service drops its + * reads, so a refresh or a finished turn reaches the host. No scope forgets everything; a scope + * without a number forgets the repository's change requests. + */ + readonly forgetChangeRequestReads?: (scope?: { + readonly repository: string; + readonly number?: number; + }) => Effect.Effect; + readonly getChangeRequestPreview?: ( input: ProviderRepositoryRef & { readonly number: number }, ) => Effect.Effect< diff --git a/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.test.ts b/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.test.ts index 6eec9ccd09142..cc6bab6c53f42 100644 --- a/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.test.ts +++ b/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.test.ts @@ -1,6 +1,7 @@ import { assert, describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as TestClock from "effect/testing/TestClock"; import * as GitLabCli from "./GitLabCli.ts"; import * as GitLabPullRequestCli from "./GitLabPullRequestCli.ts"; @@ -122,6 +123,157 @@ describe("getChangeRequest base freshness", () => { ); }); +describe("merge request reads", () => { + const reference = { cwd: "/w", repository: "acme/web", host: "gitlab.com", number: 7 }; + const detail = { + number: 7, + title: "Merge request 7", + url: "https://gitlab.com/acme/web/-/merge_requests/7", + author: null, + headBranch: "feat/page", + baseBranch: "main", + state: "open" as const, + isDraft: false, + mergeability: "mergeable" as const, + additions: 0, + deletions: 0, + createdAt: "2026-07-01T00:00:00Z", + updatedAt: "2026-07-02T00:00:00Z", + reviewRequestLogins: [], + labels: [], + body: "", + changedFiles: 1, + mergedAt: null, + closedAt: null, + reviewers: [], + checks: [], + viewerCanMerge: true, + reviewerIds: [], + }; + + const countingProvider = Effect.gen(function* () { + const calls = { detail: 0, mergeCapabilities: 0 }; + const host = { viewerCanMerge: true }; + const provider = yield* make.pipe( + Effect.provide( + Layer.mock(GitLabPullRequestCli.GitLabPullRequestCli)({ + getMergeRequestDetail: (input) => + Effect.sync(() => { + calls.detail += 1; + return { ...detail, number: input.number, viewerCanMerge: host.viewerCanMerge }; + }), + getProjectMergeCapabilities: () => + Effect.sync(() => { + calls.mergeCapabilities += 1; + return { merge: true, squash: true, rebase: true }; + }), + runMergeRequestAction: () => Effect.void, + }), + ), + ); + const checks = provider.getChangeRequestChecks; + const forget = provider.forgetChangeRequestReads; + if (checks === undefined || forget === undefined) { + return yield* Effect.die("GitLab answers checks and forgets its reads"); + } + return { provider, checks, forget, calls, host }; + }); + + it.effect("asks for a project's merge methods once across its merge requests", () => + Effect.gen(function* () { + const { provider, calls } = yield* countingProvider; + + yield* provider.getChangeRequest(reference); + yield* provider.getChangeRequest(reference); + yield* provider.getChangeRequest({ ...reference, number: 8 }); + + expect(calls.mergeCapabilities).toBe(1); + }), + ); + + it.effect("answers checks from the merge request the detail just read", () => + Effect.gen(function* () { + const { provider, checks, calls } = yield* countingProvider; + + yield* provider.getChangeRequest(reference); + yield* checks(reference); + + expect(calls.detail).toBe(1); + }), + ); + + it.effect("reads a merge request again after a write to it", () => + Effect.gen(function* () { + const { provider, checks, calls } = yield* countingProvider; + + yield* provider.getChangeRequest(reference); + yield* provider.runAction({ ...reference, action: "ready" }); + yield* checks(reference); + + expect(calls.detail).toBe(2); + }), + ); + + it.effect("does not share reads between checkouts", () => + Effect.gen(function* () { + const { provider, calls } = yield* countingProvider; + + yield* provider.getChangeRequest(reference); + yield* provider.getChangeRequest({ ...reference, cwd: "/other" }); + + expect(calls).toEqual({ detail: 2, mergeCapabilities: 2 }); + }), + ); + + it.effect("reads a merge request again once a minute has passed", () => + Effect.gen(function* () { + const { provider, calls } = yield* countingProvider; + + yield* provider.getChangeRequest(reference); + yield* TestClock.adjust("61 seconds"); + yield* provider.getChangeRequest(reference); + + expect(calls.detail).toBe(2); + }), + ); + + it.effect("answers viewer permissions from the host, not the page's read", () => + Effect.gen(function* () { + const { provider, host } = yield* countingProvider; + + host.viewerCanMerge = false; + yield* provider.getChangeRequest(reference); + host.viewerCanMerge = true; + const granted = yield* provider.getViewerPermissions(reference); + host.viewerCanMerge = false; + const revoked = yield* provider.getViewerPermissions(reference); + + expect(granted.actions).toContain("merge"); + expect(revoked.actions).not.toContain("merge"); + }), + ); + + it.effect("reads a merge request again once its reads are forgotten", () => + Effect.gen(function* () { + const { provider, checks, forget, calls } = yield* countingProvider; + const other = { ...reference, repository: "acme/api" }; + + yield* provider.getChangeRequest(reference); + yield* provider.getChangeRequest(other); + yield* forget({ repository: "ACME/web", number: 7 }); + yield* checks(reference); + yield* checks(other); + expect(calls.detail).toBe(3); + + yield* forget({ repository: "acme/web" }); + yield* checks(reference); + yield* forget(); + yield* checks(other); + expect(calls.detail).toBe(5); + }), + ); +}); + describe("optional activity reads", () => { const reference = { cwd: "/w", repository: "acme/web", host: "gitlab.com", number: 7 }; const comment = { diff --git a/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.ts b/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.ts index a9856f907ebb8..79c3d43c01966 100644 --- a/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.ts +++ b/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.ts @@ -1,5 +1,8 @@ import { SourceControlProviderKind } from "@t3tools/contracts"; +import * as Cache from "effect/Cache"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import type { PullRequestCapabilities, PullRequestReaction, @@ -72,6 +75,16 @@ export function gitLabViewerPermissions(input: { }; } +/** + * One page view asks for the detail and then its checks within a second, and the poll that + * follows asks again. A minute keeps those to one `glab` call without hiding a pipeline for long; + * a refresh or a finished turn drops it sooner through `forgetChangeRequestReads`. + */ +const MERGE_REQUEST_DETAIL_TTL = Duration.minutes(1); +/** A project's merge methods change in its settings, which nobody does between page views. */ +const PROJECT_MERGE_CAPABILITIES_TTL = Duration.minutes(15); +const READ_CACHE_CAPACITY = 256; + function gitLabProviderFailure( error: GitLabPullRequestCli.GitLabPullRequestCliError, ): PullRequestProviderFailure { @@ -85,6 +98,55 @@ function gitLabProviderFailure( export const make = Effect.gen(function* () { const cli = yield* GitLabPullRequestCli.GitLabPullRequestCli; + // Keyed on the checkout as well: `glab` picks the host and the account to answer as from the + // checkout's remote and config, so two checkouts may read the same project as different people. + const mergeRequestDetails = yield* Cache.makeWith( + (key: string) => { + const [cwd, repository, number] = JSON.parse(key) as [string, string, number]; + return cli.getMergeRequestDetail({ cwd, repository, number }); + }, + { + capacity: READ_CACHE_CAPACITY, + timeToLive: (exit) => (Exit.isSuccess(exit) ? MERGE_REQUEST_DETAIL_TTL : Duration.zero), + }, + ); + const projectMergeCapabilities = yield* Cache.makeWith( + (key: string) => { + const [cwd, repository] = JSON.parse(key) as [string, string]; + return cli.getProjectMergeCapabilities({ cwd, repository }); + }, + { + capacity: READ_CACHE_CAPACITY, + timeToLive: (exit) => (Exit.isSuccess(exit) ? PROJECT_MERGE_CAPABILITIES_TTL : Duration.zero), + }, + ); + + const detailKey = (input: { + readonly cwd: string; + readonly repository: string; + readonly number: number; + }) => JSON.stringify([input.cwd, input.repository, input.number]); + const mergeRequestDetail = (input: { + readonly cwd: string; + readonly repository: string; + readonly number: number; + }) => Cache.get(mergeRequestDetails, detailKey(input)); + + // Dropped on both sides, as the service does with its own reads: a read that starts while the + // write is in flight must not outlive it. + const writing = + < + I extends { readonly cwd: string; readonly repository: string; readonly number: number }, + A, + E, + >( + write: (input: I) => Effect.Effect, + ) => + (input: I): Effect.Effect => { + const invalidate = Cache.invalidate(mergeRequestDetails, detailKey(input)); + return invalidate.pipe(Effect.andThen(write(input)), Effect.ensuring(invalidate)); + }; + const fail = (operation: string) => (error: GitLabPullRequestCli.GitLabPullRequestCliError) => new PullRequestProviderError({ provider: SourceControlProviderKind.make("gitlab"), @@ -138,7 +200,7 @@ export const make = Effect.gen(function* () { ), getChangeRequestChecks: (input) => - cli.getMergeRequestDetail(input).pipe( + mergeRequestDetail(input).pipe( Effect.map(({ state, checks }) => ({ state, checks })), Effect.mapError(fail("getChangeRequestChecks")), ), @@ -153,8 +215,8 @@ export const make = Effect.gen(function* () { getChangeRequest: (input) => Effect.all( [ - cli.getMergeRequestDetail(input), - cli.getProjectMergeCapabilities({ cwd: input.cwd, repository: input.repository }), + mergeRequestDetail(input), + Cache.get(projectMergeCapabilities, JSON.stringify([input.cwd, input.repository])), ], { concurrency: 2 }, ).pipe( @@ -209,11 +271,31 @@ export const make = Effect.gen(function* () { })), ), + // Straight from the host, never the shared read: this is what a write is refused or let + // through on, and access granted or withdrawn within the minute must count. getViewerPermissions: (input) => cli .getMergeRequestDetail(input) .pipe(Effect.mapError(fail("getViewerPermissions")), Effect.map(gitLabViewerPermissions)), + forgetChangeRequestReads: (scope) => + Cache.keys(mergeRequestDetails).pipe( + Effect.flatMap((keys) => + Effect.forEach( + Array.from(keys).filter((key) => { + if (scope === undefined) return true; + const [, repository, number] = JSON.parse(key) as [string, string, number]; + return ( + repository.toLowerCase() === scope.repository.toLowerCase() && + (scope.number === undefined || number === scope.number) + ); + }), + (key) => Cache.invalidate(mergeRequestDetails, key), + { discard: true }, + ), + ), + ), + getDiff: (input) => cli.getMergeRequestDiff(input).pipe(Effect.mapError(fail("getDiff"))), getDiffFileContents: (input) => @@ -245,7 +327,7 @@ export const make = Effect.gen(function* () { .listProjectUsers({ cwd: input.cwd, repository: input.repository, search: input.query }) .pipe(Effect.mapError(fail("searchRepositoryAuthors"))), - setReviewerRequest: (input) => + setReviewerRequest: writing((input) => cli .setReviewerRequest({ cwd: input.cwd, @@ -255,8 +337,9 @@ export const make = Effect.gen(function* () { requested: input.requested, }) .pipe(Effect.mapError(fail("setReviewerRequest"))), + ), - runAction: (input) => + runAction: writing((input) => cli .runMergeRequestAction({ cwd: input.cwd, @@ -266,8 +349,9 @@ export const make = Effect.gen(function* () { ...(input.mergeMethod === undefined ? {} : { mergeMethod: input.mergeMethod }), }) .pipe(Effect.mapError(fail("runAction"))), + ), - updateChangeRequest: (input) => + updateChangeRequest: writing((input) => cli .updateMergeRequest({ cwd: input.cwd, @@ -277,10 +361,13 @@ export const make = Effect.gen(function* () { ...(input.body === undefined ? {} : { description: input.body }), }) .pipe(Effect.mapError(fail("updateChangeRequest"))), + ), - comment: (input) => cli.commentOnMergeRequest(input).pipe(Effect.mapError(fail("comment"))), + comment: writing((input) => + cli.commentOnMergeRequest(input).pipe(Effect.mapError(fail("comment"))), + ), - updateComment: (input) => + updateComment: writing((input) => cli .updateNote({ cwd: input.cwd, @@ -290,10 +377,13 @@ export const make = Effect.gen(function* () { body: input.body, }) .pipe(Effect.mapError(fail("updateComment"))), + ), - submitReview: (input) => cli.submitReview(input).pipe(Effect.mapError(fail("submitReview"))), + submitReview: writing((input) => + cli.submitReview(input).pipe(Effect.mapError(fail("submitReview"))), + ), - replyToThread: (input) => + replyToThread: writing((input) => cli .replyToDiscussion({ cwd: input.cwd, @@ -303,8 +393,9 @@ export const make = Effect.gen(function* () { body: input.body, }) .pipe(Effect.mapError(fail("replyToThread"))), + ), - setReaction: (input) => + setReaction: writing((input) => cli .setReaction({ cwd: input.cwd, @@ -315,8 +406,9 @@ export const make = Effect.gen(function* () { reacted: input.reacted, }) .pipe(Effect.mapError(fail("setReaction"))), + ), - setThreadResolution: (input) => + setThreadResolution: writing((input) => cli .setDiscussionResolution({ cwd: input.cwd, @@ -326,6 +418,7 @@ export const make = Effect.gen(function* () { resolved: input.resolved, }) .pipe(Effect.mapError(fail("setThreadResolution"))), + ), }; return provider; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fdf26e22678ff..0bcc79b8e86d9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -970,6 +970,15 @@ importers: '@t3tools/provider-testing': specifier: workspace:* version: link:../../packages/provider-testing + '@t3tools/secret-provider-core': + specifier: workspace:* + version: link:../../packages/secret-provider-core + '@t3tools/secret-provider-definitions': + specifier: workspace:* + version: link:../../packages/secret-provider-definitions + '@t3tools/secret-provider-onepassword': + specifier: workspace:* + version: link:../../packages/secret-provider-onepassword '@t3tools/shared': specifier: workspace:* version: link:../../packages/shared @@ -1081,6 +1090,12 @@ importers: '@t3tools/provider-core': specifier: workspace:* version: link:../../packages/provider-core + '@t3tools/secret-provider-definitions': + specifier: workspace:* + version: link:../../packages/secret-provider-definitions + '@t3tools/secret-provider-onepassword': + specifier: workspace:* + version: link:../../packages/secret-provider-onepassword '@t3tools/shared': specifier: workspace:* version: link:../../packages/shared @@ -1917,6 +1932,68 @@ importers: specifier: 24.12.4 version: 24.12.4 + packages/secret-provider-core: + dependencies: + '@t3tools/contracts': + specifier: workspace:* + version: link:../contracts + effect: + specifier: 4.0.5 + version: 4.0.5 + devDependencies: + '@effect/platform-node': + specifier: 4.0.5 + version: 4.0.5(bufferutil@4.1.0)(effect@4.0.5)(redis@6.2.1)(utf-8-validate@6.0.6) + '@effect/vitest': + specifier: 4.0.5 + version: 4.0.5(patch_hash=2731c9c3761c595f25ddc625a7d61ca376b389d5b3049ecb837c8940f21cfa02)(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(effect@4.0.5)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + '@types/node': + specifier: 24.12.4 + version: 24.12.4 + vite-plus: + specifier: 1.4.0 + version: 1.4.0(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + + packages/secret-provider-definitions: + dependencies: + '@t3tools/secret-provider-core': + specifier: workspace:* + version: link:../secret-provider-core + '@t3tools/secret-provider-onepassword': + specifier: workspace:* + version: link:../secret-provider-onepassword + + packages/secret-provider-onepassword: + dependencies: + '@t3tools/contracts': + specifier: workspace:* + version: link:../contracts + '@t3tools/provider-core': + specifier: workspace:* + version: link:../provider-core + '@t3tools/secret-provider-core': + specifier: workspace:* + version: link:../secret-provider-core + '@t3tools/shared': + specifier: workspace:* + version: link:../shared + effect: + specifier: 4.0.5 + version: 4.0.5 + devDependencies: + '@effect/platform-node': + specifier: 4.0.5 + version: 4.0.5(bufferutil@4.1.0)(effect@4.0.5)(redis@6.2.1)(utf-8-validate@6.0.6) + '@effect/vitest': + specifier: 4.0.5 + version: 4.0.5(patch_hash=2731c9c3761c595f25ddc625a7d61ca376b389d5b3049ecb837c8940f21cfa02)(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(effect@4.0.5)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + '@types/node': + specifier: 24.12.4 + version: 24.12.4 + vite-plus: + specifier: 1.4.0 + version: 1.4.0(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + packages/shared: dependencies: '@noble/curves': diff --git a/scripts/check-changed.test.ts b/scripts/check-changed.test.ts new file mode 100644 index 0000000000000..2bcd623a0b4b6 --- /dev/null +++ b/scripts/check-changed.test.ts @@ -0,0 +1,154 @@ +import { assert, describe, it } from "@effect/vitest"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { ChildProcessSpawner } from "effect/process"; +import * as TestClock from "effect/testing/TestClock"; + +import { defaultConcurrency, gitLines, runSteps, type Step } from "./check-changed.ts"; + +const GiB = 1024 ** 3; + +describe("check:changed", () => { + it("bounds the default concurrency by the scarcer of cores and available memory", () => { + assert.strictEqual(defaultConcurrency(32, 128 * GiB), 16); + assert.strictEqual(defaultConcurrency(32, 20 * GiB), 3); + assert.strictEqual(defaultConcurrency(4, 15 * GiB), 2); + assert.strictEqual(defaultConcurrency(4, 2 * GiB), 1); + assert.strictEqual(defaultConcurrency(1, 64 * GiB), 1); + }); + + it.effect( + "keeps weighted steps within the budget, in order, and serial steps one at a time", + () => + Effect.gen(function* () { + const startedAt: Record = {}; + let active = 0; + let maxActive = 0; + let serialActive = 0; + let maxSerial = 0; + const step = ( + name: string, + options: { weight?: number; serial?: boolean } = {}, + ): Step => { + const slots = Math.min(options.weight ?? 1, 3); + return { + name, + ...options, + run: Effect.gen(function* () { + startedAt[name] = (yield* Clock.currentTimeMillis) / 1000; + active += slots; + maxActive = Math.max(maxActive, active); + if (options.serial) maxSerial = Math.max(maxSerial, ++serialActive); + yield* Effect.sleep("1 second"); + active -= slots; + if (options.serial) serialActive--; + return { ok: true, output: "" }; + }), + }; + }; + const fiber = yield* runSteps( + [ + step("light 1"), + // Above the budget, so it holds every slot rather than never starting. + step("heavy", { weight: 5 }), + step("pair", { weight: 2 }), + step("light 2"), + step("test 1", { serial: true }), + step("test 2", { serial: true }), + ], + { concurrency: 3, timeoutSeconds: 60 }, + ).pipe(Effect.forkChild); + yield* TestClock.adjust("10 seconds"); + const results = yield* Fiber.join(fiber); + + assert.deepStrictEqual( + results.map((result) => result.outcome), + ["pass", "pass", "pass", "pass", "pass", "pass"], + ); + assert.strictEqual(maxActive, 3); + // Lighter steps queued behind the heavy one never start ahead of it. + assert.deepStrictEqual(startedAt, { + "light 1": 0, + heavy: 1, + pair: 2, + "light 2": 2, + "test 1": 3, + "test 2": 4, + }); + assert.strictEqual(maxSerial, 1); + }), + ); + + it.effect("interrupts a step at its timeout and reports TIMEOUT, apart from FAIL", () => + Effect.gen(function* () { + let interrupted = false; + const fiber = yield* runSteps( + [ + { + name: "hangs", + run: Effect.never.pipe( + Effect.onInterrupt(() => Effect.sync(() => (interrupted = true))), + ), + }, + { name: "fails", run: Effect.succeed({ ok: false, output: "error" }) }, + { name: "passes", run: Effect.succeed({ ok: true, output: "" }) }, + ], + { concurrency: 3, timeoutSeconds: 10 }, + ).pipe(Effect.forkChild); + yield* TestClock.adjust("9 seconds"); + assert.isFalse(interrupted); + yield* TestClock.adjust("1 second"); + const results = yield* Fiber.join(fiber); + + assert.isTrue(interrupted); + assert.deepStrictEqual( + results.map(({ name, outcome, time }) => [name, outcome, time]), + [ + ["hangs", "TIMEOUT", "killed after 10s"], + ["fails", "FAIL", "0.0s"], + ["passes", "pass", "0.0s"], + ], + ); + }), + ); + + it.effect("fails instead of returning an empty file list when git exits nonzero", () => { + const spawner = (exitCode: number) => + ChildProcessSpawner.make(() => + Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.encodeText(Stream.make(exitCode === 0 ? "a.ts\nb.ts\n" : "")), + stderr: Stream.encodeText( + Stream.make(exitCode === 0 ? "" : "fatal: index file corrupt"), + ), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ), + ); + return Effect.gen(function* () { + const listed = yield* gitLines("/repo", ["ls-files"]).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner(0)), + ); + assert.deepStrictEqual(listed, ["a.ts", "b.ts"]); + const error = yield* gitLines("/repo", ["ls-files"]).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner(128)), + Effect.flip, + ); + assert.strictEqual( + error, + "check:changed: git ls-files exited with 128: fatal: index file corrupt", + ); + }); + }); +}); diff --git a/scripts/check-changed.ts b/scripts/check-changed.ts index b854cf0a4e539..dfa4f3a833bbe 100644 --- a/scripts/check-changed.ts +++ b/scripts/check-changed.ts @@ -1,4 +1,4 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off +// @effect-diagnostics nodeBuiltinImport:off - hashes inputs with node:crypto and sizes the budget with node:os. // Runs CI's fast checks for this branch's changes: format and lint of the // changed files, knip, generated GraphQL operations, typecheck of the affected // packages and their dependents, the test files beside the changed files, and @@ -7,369 +7,551 @@ // A dependency change also gets the unused-patch install check below. CI still // runs the full test suites and the jobs listed in the summary. // -// vp run check:changed compare against the merge base with origin/main -// vp run check:changed compare against the merge base with +// vp run check:changed compare against the merge base with origin/main +// vp run check:changed compare against the merge base with +// vp run check:changed --concurrency 1 run the steps one at a time // // Uncommitted and untracked files count. Affected packages come from // .github/scripts/ci-changes.cjs, the same mapping CI uses to gate its jobs. -// The steps run side by side; each prints its output when it finishes. A -// rerun on exactly the files of the last passing run prints that run's summary. -import * as NodeChildProcess from "node:child_process"; +// The steps run side by side within a budget derived from the machine's cores +// and available memory, and each prints its output when it finishes. A step +// running past the timeout is killed with its process group and reported as +// TIMEOUT. A rerun on exactly the files of the last passing run prints that +// run's summary. import * as NodeCrypto from "node:crypto"; -import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; -import * as NodePath from "node:path"; + +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; +import * as Config from "effect/Config"; +import * as Console from "effect/Console"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import type * as PlatformError from "effect/PlatformError"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import { Argument, Command, Flag } from "effect/cli"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + import { decide, readWorkspace } from "../.github/scripts/ci-changes.cjs"; -const root = NodePath.resolve(import.meta.dirname, ".."); +const GiB = 1024 ** 3; + +// A typecheck or test step can take several cores and gigabytes of memory, so +// allow one step per two cores and per 6 GiB of available memory. +export const defaultConcurrency = (cores: number, availableBytes: number) => + Math.max(1, Math.min(Math.floor(cores / 2), Math.floor(availableBytes / (6 * GiB)))); + +export type Outcome = "pass" | "FAIL" | "TIMEOUT"; + +export interface Step { + readonly name: string; + readonly run: Effect.Effect<{ readonly ok: boolean; readonly output: string }, never, R>; + // Budget slots the step holds: one per worker it runs in parallel. + readonly weight?: number; + // Vitest runs started together race to rebuild node_modules/.vitest-cache (ENOTEMPTY). + readonly serial?: boolean; +} + +// Interrupting a step at its timeout closes the scopes of the processes it +// spawned, which kills each one's process group. +export const runSteps = Effect.fn("checkChanged.runSteps")(function* ( + steps: ReadonlyArray>, + options: { readonly concurrency: number; readonly timeoutSeconds: number }, +) { + const budget = yield* Semaphore.make(options.concurrency); + // Steps claim budget one at a time, in order: Semaphore lets a light request + // take free slots ahead of a waiting heavier one, which would delay typecheck. + const admission = yield* Semaphore.make(1); + const serial = yield* Semaphore.make(1); + return yield* Effect.forEach( + steps, + (step) => { + const timed = Effect.gen(function* () { + const started = yield* Clock.currentTimeMillis; + const result = yield* Effect.timeoutOption(step.run, `${options.timeoutSeconds} seconds`); + const seconds = ((yield* Clock.currentTimeMillis) - started) / 1000; + const outcome: Outcome = Option.match(result, { + onNone: () => "TIMEOUT", + onSome: ({ ok }) => (ok ? "pass" : "FAIL"), + }); + const time = + outcome === "TIMEOUT" + ? `killed after ${options.timeoutSeconds}s` + : `${seconds.toFixed(1)}s`; + yield* Console.log(`\n> vp ${step.name} ${outcome} ${time}`); + const output = Option.getOrUndefined(result)?.output.trimEnd(); + if (output) yield* Console.log(output); + return { name: step.name, outcome, time }; + }); + const slots = Math.min(step.weight ?? 1, options.concurrency); + const limited = Effect.acquireUseRelease( + admission.withPermit(budget.take(slots)), + () => timed, + () => budget.release(slots), + ); + return step.serial ? serial.withPermit(limited) : limited; + }, + { concurrency: "unbounded" }, + ); +}); + +const CacheEntry = Schema.fromJsonString( + Schema.Struct({ key: Schema.String, summary: Schema.Array(Schema.String) }), +); +const decodeCacheEntry = Schema.decodeEffect(CacheEntry); +const encodeCacheEntry = Schema.encodeEffect(CacheEntry); +const decodeRootPackageJson = Schema.decodeEffect( + Schema.fromJsonString(Schema.Struct({ scripts: Schema.Struct({ "knip:check": Schema.String }) })), +); + +const collectBytes = (stream: Stream.Stream) => + Stream.runCollect(stream).pipe(Effect.map((chunks) => Buffer.concat(chunks))); + +// Fails on a nonzero exit: an empty file list from a broken index would +// otherwise pass as "no changes". quotePath=false keeps non-ASCII paths as real +// paths instead of quoted escapes. +export const git = Effect.fn("checkChanged.git")(function* ( + cwd: string, + args: ReadonlyArray, +) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* Effect.scoped( + Effect.gen(function* () { + const handle = yield* spawner.spawn( + ChildProcess.make("git", ["-c", "core.quotePath=false", ...args], { cwd }), + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [collectBytes(handle.stdout), collectBytes(handle.stderr), handle.exitCode], + { concurrency: "unbounded" }, + ); + if (exitCode !== 0) { + return yield* Effect.fail( + `check:changed: git ${args.join(" ")} exited with ${exitCode}: ${stderr.toString().trim()}`, + ); + } + return stdout; + }), + ); +}); +export const gitLines = (cwd: string, args: ReadonlyArray) => + git(cwd, args).pipe(Effect.map((stdout) => stdout.toString().split("\n").filter(Boolean))); + const CODE = /\.[cm]?[jt]sx?$/; +const TEST = /\.test\.[cm]?[jt]sx?$/; +const escapeRegExp = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -// quotePath=false keeps non-ASCII paths as real paths instead of quoted escapes. -const git = (...args: Array) => - NodeChildProcess.execFileSync("git", ["-c", "core.quotePath=false", ...args], { - cwd: root, - encoding: "utf8", - maxBuffer: 64 * 1024 * 1024, - }) - .split("\n") - .filter(Boolean); - -const ref = process.argv[2] ?? "origin/main"; -const mergeBase = NodeChildProcess.spawnSync("git", ["merge-base", "HEAD", ref], { - cwd: root, - encoding: "utf8", +// A flag wins over its environment variable; either must be a positive integer. +const override = Effect.fn("checkChanged.override")(function* ( + flagName: string, + flag: Option.Option, + envName: string, +) { + const [name, value] = Option.isSome(flag) + ? [`--${flagName}`, flag] + : [envName, yield* Config.option(Config.Int(envName))]; + if (Option.isSome(value) && value.value < 1) { + return yield* Effect.fail(`check:changed: ${name} must be a positive integer.`); + } + return value; }); -if (mergeBase.status !== 0) { - console.error( - `check:changed: no merge base with ${ref}. Run \`git fetch origin\` or pass a ref.`, + +const checkChanged = Effect.fn("checkChanged")(function* (input: { + readonly ref: string; + readonly concurrency: Option.Option; + readonly timeout: Option.Option; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* path.fromFileUrl(new URL("..", import.meta.url)); + const inRoot = (file: string) => path.join(root, file); + + // macOS counts its file cache as used, so available memory there would + // starve a large Mac. Elsewhere availableMemory() honors a cgroup limit. + const platform = yield* HostProcess.Platform; + const concurrency = Option.getOrElse( + yield* override("concurrency", input.concurrency, "CHECK_CHANGED_CONCURRENCY"), + () => + defaultConcurrency( + NodeOS.availableParallelism(), + platform === "darwin" ? NodeOS.totalmem() : process.availableMemory(), + ), ); - process.exit(1); -} -const base = mergeBase.stdout.trim(); -// --no-renames lists a rename as its old and new path, as CI does. -const untracked = git("ls-files", "--others", "--exclude-standard"); -const files = [ - ...new Set([...git("diff", "--name-only", "--no-renames", base, "--"), ...untracked]), -].sort(); - -if (files.length === 0) { - console.log(`check:changed: no changes since ${base.slice(0, 12)}.`); - process.exit(0); -} -console.log(`check:changed: ${files.length} files since ${base.slice(0, 12)}`); - -// Ignored generated sources, declarations, env files, and install state affect check results. -// Include them in the key, but skip large ignored folders that no check reads. -// Recompute after a passing run so edits made during it are never cached as passing. -const inputKey = () => { - const hash = NodeCrypto.createHash("sha256"); - // CI changes test behavior (vite.config.ts rejects .only under CI). - hash.update(`${process.version}\0${base}\0${process.env.CI ?? ""}\0`); - hash.update( - NodeChildProcess.execFileSync( - "git", - ["diff", "--binary", "--no-renames", "--no-ext-diff", "--no-textconv", base, "--"], - { cwd: root, maxBuffer: 1024 * 1024 * 1024 }, - ), + const timeoutSeconds = Option.getOrElse( + yield* override("timeout", input.timeout, "CHECK_CHANGED_TIMEOUT"), + () => 900, ); - const ignored = git( - "ls-files", - "--others", - "--ignored", - "--exclude-standard", - "--", - ":(glob)**/.generated/**", - ":(glob)**/.astro/**", - ":(glob)**/*.d.ts", - ":(glob)**/.env*", - ":(exclude,glob)**/node_modules/**", + + const lines = (...args: Array) => gitLines(root, args); + + const base = yield* git(root, ["merge-base", "HEAD", input.ref]).pipe( + Effect.map((stdout) => stdout.toString().trim()), + Effect.mapError( + () => + `check:changed: no merge base with ${input.ref}. Run \`git fetch origin\` or pass a ref.`, + ), ); - for (const file of [ - // Raw bytes of changed files too: .gitattributes normalizes line endings in - // the diff, but the formatter reads the file as it is on disk. - ...git("diff", "--name-only", "--no-renames", base, "--"), - ...git("ls-files", "--others", "--exclude-standard"), - ...ignored, - "node_modules/.modules.yaml", - "node_modules/.pnpm/lock.yaml", - ]) { - hash.update(`\0${file}\0`); - const path = NodePath.join(root, file); - // Untracked nested repositories are listed as directories. - if (NodeFS.statSync(path, { throwIfNoEntry: false })?.isFile()) { - hash.update(NodeFS.readFileSync(path)); - } + // --no-renames lists a rename as its old and new path, as CI does. + const changedFiles = lines("diff", "--name-only", "--no-renames", base, "--"); + const untrackedFiles = lines("ls-files", "--others", "--exclude-standard"); + const files = [...new Set([...(yield* changedFiles), ...(yield* untrackedFiles)])].sort(); + + if (files.length === 0) { + yield* Console.log(`check:changed: no changes since ${base.slice(0, 12)}.`); + return true; } - return hash.digest("hex"); -}; -const key = inputKey(); -const cachePath = NodePath.join(root, "node_modules/.cache/check-changed.json"); -// A missing or unreadable cache is a miss, never a failure. -const readCache = () => { - try { - return JSON.parse(NodeFS.readFileSync(cachePath, "utf8")) as { - key: string; - summary: Array; - }; - } catch { - return undefined; + yield* Console.log(`check:changed: ${files.length} files since ${base.slice(0, 12)}`); + + // Ignored generated sources, declarations, env files, and install state affect check results. + // Include them in the key, but skip large ignored folders that no check reads. + // Recompute after a passing run so edits made during it are never cached as passing. + const ci = Option.getOrElse(yield* Config.option(Config.String("CI")), () => ""); + const inputKey = Effect.gen(function* () { + const hash = NodeCrypto.createHash("sha256"); + // CI changes test behavior (vite.config.ts rejects .only under CI). + hash.update(`${process.version}\0${base}\0${ci}\0`); + hash.update( + yield* git(root, [ + "diff", + "--binary", + "--no-renames", + "--no-ext-diff", + "--no-textconv", + base, + "--", + ]), + ); + const ignored = yield* lines( + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + "--", + ":(glob)**/.generated/**", + ":(glob)**/.astro/**", + ":(glob)**/*.d.ts", + ":(glob)**/.env*", + ":(exclude,glob)**/node_modules/**", + ); + for (const file of [ + // Raw bytes of changed files too: .gitattributes normalizes line endings in + // the diff, but the formatter reads the file as it is on disk. + ...(yield* changedFiles), + ...(yield* untrackedFiles), + ...ignored, + "node_modules/.modules.yaml", + "node_modules/.pnpm/lock.yaml", + ]) { + hash.update(`\0${file}\0`); + // Untracked nested repositories are listed as directories. + const info = yield* Effect.option(fs.stat(inRoot(file))); + if (Option.isSome(info) && info.value.type === "File") { + hash.update(yield* fs.readFile(inRoot(file))); + } + } + return hash.digest("hex"); + }); + const key = yield* inputKey; + const cachePath = inRoot("node_modules/.cache/check-changed.json"); + // A missing or unreadable cache is a miss, never a failure. + const cached = yield* fs + .readFileString(cachePath) + .pipe(Effect.flatMap(decodeCacheEntry), Effect.option); + if (Option.isSome(cached) && cached.value.key === key) { + yield* Console.log("Nothing changed since the last passing run.\n"); + for (const line of cached.value.summary) yield* Console.log(line); + return true; } -}; -const cached = readCache(); -if (cached?.key === key) { - console.log("Nothing changed since the last passing run.\n"); - for (const line of cached.summary) console.log(line); - process.exit(0); -} -const present = files.filter((file) => NodeFS.existsSync(NodePath.join(root, file))); -const code = present.filter((file) => CODE.test(file)); -const packages = readWorkspace(); -const { affected, outputs } = decide(files, packages, false); + const present = yield* Effect.filter(files, (file) => fs.exists(inRoot(file))); + const code = present.filter((file) => CODE.test(file)); + const packages = readWorkspace(); + const { affected, outputs } = decide(files, packages, false); -// Changed test files, and the test files beside a changed source file -// (foo.ts -> foo.test.ts, foo.browser.test.tsx). Vitest's import-graph -// `related` reaches most of a package: one leaf server file pulled in 252 of -// 406 server test files and took 7 minutes. CI runs the rest. -const TEST = /\.test\.[cm]?[jt]sx?$/; -const tests = new Set(code.filter((file) => TEST.test(file))); -for (const file of code) { - const dir = NodePath.dirname(file); - const prefix = `${NodePath.basename(file).replace(CODE, "")}.`; - for (const entry of NodeFS.readdirSync(NodePath.join(root, dir))) { - if (entry.startsWith(prefix) && TEST.test(entry)) tests.add(NodePath.join(dir, entry)); + // Changed test files, and the test files beside a changed source file + // (foo.ts -> foo.test.ts, foo.browser.test.tsx). Vitest's import-graph + // `related` reaches most of a package: one leaf server file pulled in 252 of + // 406 server test files and took 7 minutes. CI runs the rest. + const tests = new Set(code.filter((file) => TEST.test(file))); + for (const file of code) { + const dir = path.dirname(file); + const prefix = `${path.basename(file).replace(CODE, "")}.`; + for (const entry of yield* fs.readDirectory(inRoot(dir))) { + if (entry.startsWith(prefix) && TEST.test(entry)) tests.add(path.join(dir, entry)); + } } -} -// It reads files other than the ones beside it, so no changed file selects it. -if (affected.includes("t3")) tests.add("apps/server/src/provider/AgentContextBudget.test.ts"); + // It reads files other than the ones beside it, so no changed file selects it. + if (affected.includes("t3")) tests.add("apps/server/src/provider/AgentContextBudget.test.ts"); -// React Compiler coverage compiles every client source file, so run only the -// changed files' cases, in one extra step: a -t filter in the main test step -// would drop every other test. An app is skipped when the main step already -// runs its coverage file unfiltered. File filters mirror each coverage test. -const escapeRegExp = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -const compilerApps = [ - { - test: "apps/web/src/reactCompilerCoverage.test.ts", - root: "apps/web/src/", - // Test names keep the repo-relative path. - name: (file: string) => file, - compiled: (file: string) => - /\.tsx?$/.test(file) && !/\.(test|browser)\.tsx?$/.test(file) && !file.endsWith(".gen.ts"), - // Web has only per-file cases. - always: [], - }, - { - test: "apps/mobile/src/reactCompilerCoverage.test.ts", - root: "apps/mobile/src/", - name: (file: string) => file.slice("apps/mobile/".length), - compiled: (file: string) => /\.tsx?$/.test(file) && !/\.(test\.tsx?|d\.ts)$/.test(file), - // The Metro, cache, and stale-allowlist cases check more than one file, deletions included. - always: ["^(?!.*compiles )"], - }, -].flatMap(({ test, root: appRoot, name, compiled, always }) => { - const changed = files.filter((file) => file.startsWith(appRoot) && compiled(file)); - if (changed.length === 0 || tests.has(test)) return []; - const names = changed - .filter((file) => present.includes(file)) - .map((file) => `compiles ${escapeRegExp(name(file))}$`); - const pattern = [...names, ...always]; - return pattern.length > 0 ? [{ test, names, pattern }] : []; -}); -const compilerCoverage = compilerApps.length > 0 && { - name: `react compiler (${compilerApps.reduce((count, app) => count + app.names.length, 0)} files)`, - args: [ - "test", - "run", - ...compilerApps.map((app) => app.test), - "-t", - compilerApps.flatMap((app) => app.pattern).join("|"), - ], -}; - -// knip:check cannot be scoped to a package. Any code change can leave a file, -// export, or dependency unused (removing the last import of a helper does), so -// it runs whenever code or a manifest changes. -const runKnip = files.some( - (file) => CODE.test(file) || NodePath.basename(file) === "package.json" || file === "knip.jsonc", -); -// CI runs knip:check as one script of `knip ... && knip ...` passes. The passes -// do not depend on each other, so they run side by side here. -const knipCheck: string = JSON.parse( - NodeFS.readFileSync(NodePath.join(root, "package.json"), "utf8"), -).scripts["knip:check"]; -const knipPasses = knipCheck.split(" && ").map((command, index, all) => { - const [bin, ...args] = command.trim().split(/\s+/); - if (bin !== "knip" || /["'$\\|;&<>]/.test(command)) { - throw new Error(`check:changed cannot split knip:check: ${command}`); + // React Compiler coverage compiles every client source file, so run only the + // changed files' cases, in one extra step: a -t filter in the main test step + // would drop every other test. An app is skipped when the main step already + // runs its coverage file unfiltered. File filters mirror each coverage test. + const compilerApps = [ + { + test: "apps/web/src/reactCompilerCoverage.test.ts", + root: "apps/web/src/", + // Test names keep the repo-relative path. + name: (file: string) => file, + compiled: (file: string) => + /\.tsx?$/.test(file) && !/\.(test|browser)\.tsx?$/.test(file) && !file.endsWith(".gen.ts"), + // Web has only per-file cases. + always: [], + }, + { + test: "apps/mobile/src/reactCompilerCoverage.test.ts", + root: "apps/mobile/src/", + name: (file: string) => file.slice("apps/mobile/".length), + compiled: (file: string) => /\.tsx?$/.test(file) && !/\.(test\.tsx?|d\.ts)$/.test(file), + // The Metro, cache, and stale-allowlist cases check more than one file, deletions included. + always: ["^(?!.*compiles )"], + }, + ].flatMap(({ test, root: appRoot, name, compiled, always }) => { + const changed = files.filter((file) => file.startsWith(appRoot) && compiled(file)); + if (changed.length === 0 || tests.has(test)) return []; + const names = changed + .filter((file) => present.includes(file)) + .map((file) => `compiles ${escapeRegExp(name(file))}$`); + const pattern = [...names, ...always]; + return pattern.length > 0 ? [{ test, names, pattern }] : []; + }); + + // knip:check cannot be scoped to a package. Any code change can leave a file, + // export, or dependency unused (removing the last import of a helper does), so + // it runs whenever code or a manifest changes. + const runKnip = files.some( + (file) => CODE.test(file) || path.basename(file) === "package.json" || file === "knip.jsonc", + ); + // CI runs knip:check as one script of `knip ... && knip ...` passes. The passes + // do not depend on each other, so they run side by side here. + const rootPackage = yield* fs + .readFileString(inRoot("package.json")) + .pipe(Effect.flatMap(decodeRootPackageJson), Effect.orDie); + const knipCheck = rootPackage.scripts["knip:check"]; + const knipPasses: Array> = []; + for (const command of knipCheck.split(" && ")) { + const [bin, ...args] = command.trim().split(/\s+/); + if (bin !== "knip" || /["'$\\|;&<>]/.test(command)) { + return yield* Effect.fail(`check:changed cannot split knip:check: ${command}`); + } + knipPasses.push(["exec", "knip", ...args]); } - return { name: `knip ${index + 1}/${all.length}`, args: ["exec", "knip", ...args] }; -}); -// A version bump can leave a patch unused, causing pnpm to reject the install -// with ERR_PNPM_UNUSED_PATCH. Check it here as Release Smoke does in CI. -const runPatchCheck = files.some( - (file) => - file === "pnpm-workspace.yaml" || - file === "pnpm-lock.yaml" || - file === ".pnpmfile.cjs" || - file.startsWith("patches/") || - file.startsWith("apps/mobile/deps/") || - NodePath.basename(file) === "package.json", -); + // A version bump can leave a patch unused, causing pnpm to reject the install + // with ERR_PNPM_UNUSED_PATCH. Check it here as Release Smoke does in CI. + const runPatchCheck = files.some( + (file) => + file === "pnpm-workspace.yaml" || + file === "pnpm-lock.yaml" || + file === ".pnpmfile.cjs" || + file.startsWith("patches/") || + file.startsWith("apps/mobile/deps/") || + path.basename(file) === "package.json", + ); + + // Output is piped, so keep the colors a terminal would get. + const env = process.stdout.isTTY ? { FORCE_COLOR: "1" } : {}; + const vp = (args: ReadonlyArray, cwd = root) => + Effect.scoped( + Effect.gen(function* () { + const handle = yield* spawner.spawn( + ChildProcess.make("vp", args, { cwd, env, extendEnv: true, forceKillAfter: "5 seconds" }), + ); + const [output, exitCode] = yield* Effect.all( + [Stream.mkString(Stream.decodeText(handle.all)), handle.exitCode], + { concurrency: 2 }, + ); + return { ok: exitCode === 0, output }; + }), + ).pipe(Effect.catch((error) => Effect.succeed({ ok: false, output: error.message }))); -// Resolve without the lockfile in a temp copy to preserve the working tree and lockfile. -// Copy mobile modules too: they are file: dependencies, not workspace packages. -// Cached registry metadata limits this to patches stranded by local changes, not registry drift. -const checkPatches = (): boolean => { - const temp = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-patch-check-")); - try { - const copy = (relative: string) => { - const dest = NodePath.join(temp, relative); - NodeFS.mkdirSync(NodePath.dirname(dest), { recursive: true }); - NodeFS.cpSync(NodePath.join(root, relative), dest, { recursive: true }); - }; - copy("package.json"); - copy("pnpm-workspace.yaml"); - for (const pkg of packages) copy(`${pkg.dir}/package.json`); - const modulesDir = NodePath.join(root, "apps/mobile/modules"); - if (NodeFS.existsSync(modulesDir)) { - for (const entry of NodeFS.readdirSync(modulesDir)) { - const manifest = `apps/mobile/modules/${entry}/package.json`; - if (NodeFS.existsSync(NodePath.join(root, manifest))) copy(manifest); + // Resolve without the lockfile in a temp copy to preserve the working tree and lockfile. + // Copy mobile modules too: they are file: dependencies, not workspace packages. + // Cached registry metadata limits this to patches stranded by local changes, not registry drift. + const checkPatches = Effect.scoped( + Effect.gen(function* () { + const temp = yield* fs.makeTempDirectoryScoped({ prefix: "t3-patch-check-" }); + const copy = Effect.fnUntraced(function* (relative: string) { + const dest = path.join(temp, relative); + yield* fs.makeDirectory(path.dirname(dest), { recursive: true }); + yield* fs.copy(inRoot(relative), dest); + }); + yield* copy("package.json"); + yield* copy("pnpm-workspace.yaml"); + for (const pkg of packages) yield* copy(`${pkg.dir}/package.json`); + const modulesDir = "apps/mobile/modules"; + if (yield* fs.exists(inRoot(modulesDir))) { + for (const entry of yield* fs.readDirectory(inRoot(modulesDir))) { + const manifest = `${modulesDir}/${entry}/package.json`; + if (yield* fs.exists(inRoot(manifest))) yield* copy(manifest); + } } - } - for (const extra of ["apps/mobile/deps", "patches", ".pnpmfile.cjs"]) { - if (NodeFS.existsSync(NodePath.join(root, extra))) copy(extra); - } - const { status, stdout, stderr, error } = NodeChildProcess.spawnSync( - "vp", - ["install", "--lockfile-only", "--ignore-scripts"], - { cwd: temp, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }, - ); - if (status === 0) return true; - if (stdout) console.log(stdout); - if (stderr) console.error(stderr); - if (error) console.error(error.message); - if (`${stdout ?? ""}${stderr ?? ""}`.includes("ERR_PNPM_UNUSED_PATCH")) { - console.error( - "Unused dependency patches: remove each patch named above from " + + for (const extra of ["apps/mobile/deps", "patches", ".pnpmfile.cjs"]) { + if (yield* fs.exists(inRoot(extra))) yield* copy(extra); + } + const result = yield* vp(["install", "--lockfile-only", "--ignore-scripts"], temp); + if (result.ok || !result.output.includes("ERR_PNPM_UNUSED_PATCH")) return result; + return { + ok: false, + output: + `${result.output}\nUnused dependency patches: remove each patch named above from ` + "patchedDependencies in pnpm-workspace.yaml and from patches/, " + "update it to the version the install resolves, " + "or pin the dependency that pulls it in to the patched version.", - ); - } - return false; - } finally { - NodeFS.rmSync(temp, { recursive: true, force: true }); - } -}; + }; + }), + ).pipe(Effect.catch((error) => Effect.succeed({ ok: false, output: error.message }))); -// CI's Lint job checks that generated GraphQL operations match their sources. -const graphql = ["apps/server", "packages/source-control-github"].filter((dir) => - files.some((file) => file.startsWith(`${dir}/`)), -); + // CI's Lint job checks that generated GraphQL operations match their sources. + const graphql = ["apps/server", "packages/source-control-github"].filter((dir) => + files.some((file) => file.startsWith(`${dir}/`)), + ); + const filters = (names: ReadonlyArray) => names.flatMap((name) => ["--filter", name]); + const typecheckWorkers = Math.min(concurrency, affected.length); + const command = (name: string, args: ReadonlyArray) => ({ name, run: vp(args) }); + const compilerFiles = compilerApps.reduce((count, app) => count + app.names.length, 0); -const filters = (names: ReadonlyArray) => names.flatMap((name) => ["--filter", name]); -const steps = [ - // Runs first; it takes about a second. - runPatchCheck && { name: "unused patches", run: checkPatches }, - present.length > 0 && { - name: "format", - args: ["fmt", "--check", "--no-error-on-unmatched-pattern", ...present], - }, - code.length > 0 && { name: "lint", args: ["lint", ...code] }, - ...(runKnip ? knipPasses : []), - ...graphql.map((dir) => ({ - name: `graphql (${dir})`, - args: ["run", "--filter", `./${dir}`, "generate", "--check"], - })), - affected.length > 0 && { - name: `typecheck (${affected.length} packages)`, - // `tsc --noEmit` writes nothing another package reads, so dependency order - // only made dependents wait: --parallel cut a contracts change from 13 s to 9 s. - args: [ - "run", - "--cache", - "--parallel", - "--concurrency-limit", - "4", - ...filters(affected), - "typecheck", - ], - }, - // From the repo root, the root vite.config.ts hands each file to its package's config. - tests.size > 0 && { - name: `test (${tests.size} files)`, - args: ["test", "run", "--passWithNoTests", ...tests], - }, - compilerCoverage, -].filter((step) => step !== false); - -// Output is piped, so keep the colors a terminal would get. -const env = process.stdout.isTTY ? { ...process.env, FORCE_COLOR: "1" } : process.env; -const runStep = (step: (typeof steps)[number]) => - new Promise<{ name: string; ok: boolean; seconds: string }>((resolve) => { - const started = performance.now(); - const finish = (ok: boolean, output?: Buffer) => { - const seconds = ((performance.now() - started) / 1000).toFixed(1); - console.log(`\n> vp ${step.name} ${ok ? "pass" : "FAIL"} ${seconds}s`); - if (output) process.stdout.write(output); - resolve({ name: step.name, ok, seconds }); - }; - // In-process steps print their own output. The patch check is first and - // takes about a second, so the commands start right after it. - if ("run" in step) return finish(step.run()); - const output: Array = []; - const child = NodeChildProcess.spawn("vp", step.args, { cwd: root, env }); - child.stdout.on("data", (chunk: Buffer) => output.push(chunk)); - child.stderr.on("data", (chunk: Buffer) => output.push(chunk)); - // A failed spawn emits "error", then "close" with a nonzero code. - child.on("error", (error) => output.push(Buffer.from(`${error.message}\n`))); - child.on("close", (status) => finish(status === 0, Buffer.concat(output))); - }); + const steps: Array> = [ + ...(runPatchCheck ? [{ name: "unused patches", run: checkPatches }] : []), + ...(present.length > 0 + ? [command("format", ["fmt", "--check", "--no-error-on-unmatched-pattern", ...present])] + : []), + ...(code.length > 0 ? [command("lint", ["lint", ...code])] : []), + ...(runKnip + ? knipPasses.map((args, index) => command(`knip ${index + 1}/${knipPasses.length}`, args)) + : []), + ...graphql.map((dir) => + command(`graphql (${dir})`, ["run", "--filter", `./${dir}`, "generate", "--check"]), + ), + ...(affected.length > 0 + ? [ + { + // `tsc --noEmit` writes nothing another package reads, so dependency order + // only made dependents wait: --parallel cut a contracts change from 13 s to 9 s. + ...command(`typecheck (${affected.length} packages)`, [ + "run", + "--cache", + "--parallel", + "--concurrency-limit", + String(typecheckWorkers), + ...filters(affected), + "typecheck", + ]), + weight: typecheckWorkers, + }, + ] + : []), + // From the repo root, the root vite.config.ts hands each file to its package's config. + ...(tests.size > 0 + ? [ + { + ...command(`test (${tests.size} files)`, [ + "test", + "run", + "--passWithNoTests", + ...tests, + ]), + serial: true, + }, + ] + : []), + ...(compilerApps.length > 0 + ? [ + { + ...command(`react compiler (${compilerFiles} files)`, [ + "test", + "run", + ...compilerApps.map((app) => app.test), + "-t", + compilerApps.flatMap((app) => app.pattern).join("|"), + ]), + serial: true, + }, + ] + : []), + ]; -// Vitest runs started together race to rebuild node_modules/.vitest-cache after -// an install (ENOTEMPTY), so they run one after another. -let previousVitest: Promise = Promise.resolve(); -const results = await Promise.all( - steps.map((step) => { - if (!("args" in step) || step.args[0] !== "test") return runStep(step); - const result = previousVitest.then(() => runStep(step)); - previousVitest = result; - return result; - }), -); + const results = yield* runSteps(steps, { concurrency, timeoutSeconds }); -const summary = ["check:changed summary"]; -for (const result of results) { - summary.push(` ${result.ok ? "pass" : "FAIL"} ${result.name} ${result.seconds}s`); -} -const ciOnly = Object.entries({ - "full test suites": outputs.test || outputs.test_web || outputs.test_server, - build: outputs.build, - rust: outputs.rust, - "mobile native lint": outputs.mobile_native, - "release smoke": outputs.release_smoke, -}).flatMap(([job, runs]) => (runs ? [job] : [])); -if (ciOnly.length > 0) summary.push(` CI also runs: ${ciOnly.join(", ")}`); -if (!runPatchCheck) summary.push(" unused patch check skipped: no dependency files changed"); -if (!runKnip) summary.push(" knip skipped: no code or manifest changed"); -console.log(`\n${summary.join("\n")}`); -// exitCode, not exit(): piped stdout is asynchronous on macOS, and exit() -// would cut off a failing step's output. -if (results.some((result) => !result.ok)) { - process.exitCode = 1; -} else if (inputKey() !== key) { - console.log(" pass not cached: files changed during the run"); -} else { - try { - // Write then rename, so a concurrent run never reads half a file. - const temp = `${cachePath}.${process.pid}`; - NodeFS.mkdirSync(NodePath.dirname(cachePath), { recursive: true }); - NodeFS.writeFileSync(temp, JSON.stringify({ key, summary })); - NodeFS.renameSync(temp, cachePath); - } catch (error) { - console.warn(` pass not cached: ${error instanceof Error ? error.message : String(error)}`); + const summary = ["check:changed summary"]; + for (const result of results) { + summary.push(` ${result.outcome} ${result.name} ${result.time}`); + } + if (results.some((result) => result.outcome === "TIMEOUT")) { + summary.push( + ` TIMEOUT: ran past ${timeoutSeconds}s and was killed; this is not a check failure. ` + + "Raise --timeout or lower --concurrency.", + ); + } + const ciOnly = Object.entries({ + "full test suites": outputs.test || outputs.test_web || outputs.test_server, + build: outputs.build, + rust: outputs.rust, + "mobile native lint": outputs.mobile_native, + "release smoke": outputs.release_smoke, + }).flatMap(([job, runs]) => (runs ? [job] : [])); + if (ciOnly.length > 0) summary.push(` CI also runs: ${ciOnly.join(", ")}`); + if (!runPatchCheck) summary.push(" unused patch check skipped: no dependency files changed"); + if (!runKnip) summary.push(" knip skipped: no code or manifest changed"); + yield* Console.log(`\n${summary.join("\n")}`); + + if (results.some((result) => result.outcome !== "pass")) return false; + if ((yield* inputKey) !== key) { + yield* Console.log(" pass not cached: files changed during the run"); + return true; } + // Write then rename, so a concurrent run never reads half a file. + const temp = `${cachePath}.${process.pid}`; + yield* Effect.gen(function* () { + yield* fs.makeDirectory(path.dirname(cachePath), { recursive: true }); + yield* fs.writeFileString(temp, yield* encodeCacheEntry({ key, summary })); + yield* fs.rename(temp, cachePath); + }).pipe(Effect.catch((error) => Console.warn(` pass not cached: ${error.message}`))); + return true; +}); + +export const checkChangedCommand = Command.make( + "check-changed", + { + ref: Argument.String("ref").pipe( + Argument.withDescription("Compare against the merge base with this ref."), + Argument.withDefault("origin/main"), + ), + concurrency: Flag.Int("concurrency").pipe( + Flag.withDescription( + "Budget of parallel steps and typecheck workers; also CHECK_CHANGED_CONCURRENCY. Defaults to one per two cores and per 6 GiB of available memory.", + ), + Flag.optional, + ), + timeout: Flag.Int("timeout").pipe( + Flag.withDescription( + "Seconds before a step is killed; also CHECK_CHANGED_TIMEOUT. Defaults to 900.", + ), + Flag.optional, + ), + }, + (input) => + checkChanged(input).pipe( + Effect.catch((message) => Console.error(message).pipe(Effect.as(false))), + // exitCode, not a failed exit: runMain calls process.exit() on failure, and + // piped stdout is asynchronous on macOS, so it would cut off step output. + Effect.flatMap((ok) => + Effect.sync(() => { + if (!ok) process.exitCode = 1; + }), + ), + ), +).pipe(Command.withDescription("Run CI's fast checks for this branch's changes.")); + +if (import.meta.main) { + Command.run(checkChangedCommand, { version: "0.0.0" }).pipe( + Effect.provide(NodeServices.layer), + NodeRuntime.runMain, + ); } diff --git a/scripts/release-smoke.ts b/scripts/release-smoke.ts index 12ef21824014d..8f300533523ae 100644 --- a/scripts/release-smoke.ts +++ b/scripts/release-smoke.ts @@ -39,6 +39,9 @@ const workspaceFiles = [ "packages/provider-grok/package.json", "packages/provider-muse/package.json", "packages/provider-opencode/package.json", + "packages/secret-provider-core/package.json", + "packages/secret-provider-definitions/package.json", + "packages/secret-provider-onepassword/package.json", "packages/source-control-azure-devops/package.json", "packages/source-control-bitbucket/package.json", "packages/source-control-core/package.json",