From 9370e2c2d329ab622b7cb21c05bb2362a5008761 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:46:42 +0100 Subject: [PATCH 1/5] feat(supervisor): reclaim a run's checkpoint storage when it finishes --- apps/supervisor/src/env.ts | 1 + apps/supervisor/src/workloadServer/index.ts | 92 ++++++++++++++++++- .../src/v3/serverOnly/checkpointClient.ts | 38 ++++++++ 3 files changed, 130 insertions(+), 1 deletion(-) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 670a00ab582..9733f56bc3d 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -22,6 +22,7 @@ export const Env = z // also reject invalid tokens. WORKLOAD_TOKEN_SECRET: z.string().optional(), WORKLOAD_TOKEN_ENFORCEMENT: z.enum(["disabled", "log", "enforce"]).default("disabled"), + DELETE_CHECKPOINTS_ON_COMPLETION: BoolEnv.default(false), // irreversible; enable per cluster // Absolute expiry for minted deployment tokens. Deterministic (no wall-clock issued-at) so every // pod of a deployment carries an identical token; bump before this date. Must outlive any run. WORKLOAD_TOKEN_EXP: z.string().datetime().default("2032-01-01T00:00:00.000Z"), diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index 86717438c72..d28eaedf775 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -23,6 +23,8 @@ import EventEmitter from "node:events"; import type { IncomingMessage, ServerResponse } from "node:http"; import { type Namespace, Server, type Socket } from "socket.io"; import { z } from "zod"; +import { tryCatch } from "@trigger.dev/core/utils"; +import { Counter } from "prom-client"; import { env } from "../env.js"; import { register } from "../metrics.js"; import { @@ -30,6 +32,7 @@ import { workloadTokenEnforced, workloadTokensEnabled, } from "../workloadToken.js"; +import type { WorkloadDeploymentTokenClaims } from "@trigger.dev/core/v3"; import { ComputeSnapshotService, type RunTraceContext, @@ -50,6 +53,19 @@ interface DefaultEventsMap { [event: string]: (...args: any[]) => void; } +/** + * checkpointDeleteRequests counts the delete requests this supervisor makes, and every reason it + * decides not to: `sent`, `disabled`, `not_terminal`, `no_claims`, `no_project_ref`, `http_error`. + * Without the negative outcomes, "no deletes are happening" is indistinguishable from the feature + * being switched off - and with no lifecycle expiry, that difference is leaked storage. + */ +const checkpointDeleteRequests = new Counter({ + name: "checkpoint_delete_requests_total", + help: "Checkpoint delete requests attempted at run completion, by outcome", + labelNames: ["result"], + registers: [register], +}); + const WorkloadActionParams = z.object({ runFriendlyId: z.string(), snapshotFriendlyId: z.string(), @@ -181,10 +197,16 @@ export class WorkloadServer extends EventEmitter { * environment_id to forward upstream. The env id is only forwarded in enforce mode: in log mode * we still verify + record metrics but attach no header (so the platform never scopes). Only * enforce fails a request, and only for a present-but-invalid token; absent and legacy ids pass. + * + * `claims` are returned whenever the token verifies, in either mode. They are for addressing a + * run's own resources locally (e.g. its checkpoint storage) - never for scoping the platform, + * which is why environmentId above stays gated on enforce. */ private async authorizeWorkloadRequest( req: IncomingMessage - ): Promise<{ ok: true; environmentId?: string } | { ok: false }> { + ): Promise< + { ok: true; environmentId?: string; claims?: WorkloadDeploymentTokenClaims } | { ok: false } + > { if (!workloadTokensEnabled) { return { ok: true }; } @@ -201,9 +223,70 @@ export class WorkloadServer extends EventEmitter { workloadTokenEnforced && result.outcome === "jwt_valid" ? result.claims.environment_id : undefined, + claims: result.outcome === "jwt_valid" ? result.claims : undefined, }; } + /** + * reclaimCheckpoints asks the checkpoint service to delete a finished run's checkpoint storage. + * + * Called only after the reply has been sent, so it never delays the runner - the same shape the + * suspend route uses. Every early return is counted: with no lifecycle expiry behind this, a + * silently skipped delete is storage leaked forever, and silence must not look like success. + * + * `RUN_PENDING_CANCEL` is terminal too - a run cancelled mid-execution never restores - so it is + * reclaimed alongside `RUN_FINISHED`. Retries are deliberately excluded: the prefix is run-level, + * so a retry's checkpoints are cleaned by the final completion. + */ + private async reclaimCheckpoints( + req: IncomingMessage, + runFriendlyId: string, + attemptStatus: string, + claims: WorkloadDeploymentTokenClaims | undefined + ): Promise { + if (!env.DELETE_CHECKPOINTS_ON_COMPLETION || !this.checkpointClient || this.snapshotService) { + checkpointDeleteRequests.inc({ result: "disabled" }); + return; + } + + if (attemptStatus !== "RUN_FINISHED" && attemptStatus !== "RUN_PENDING_CANCEL") { + checkpointDeleteRequests.inc({ result: "not_terminal" }); + return; + } + + if (!claims) { + checkpointDeleteRequests.inc({ result: "no_claims" }); + return; + } + + const projectRef = this.projectRefFromRequest(req); + if (!projectRef) { + checkpointDeleteRequests.inc({ result: "no_project_ref" }); + this.logger.error("Cannot reclaim checkpoints without a project ref", { runFriendlyId }); + return; + } + + const [error, accepted] = await tryCatch( + this.checkpointClient.deleteCheckpoints({ + runFriendlyId, + body: { + orgId: claims.org_id, + envId: claims.environment_id, + deploymentVersion: claims.deployment_version, + projectRef, + }, + }) + ); + + if (error || !accepted) { + checkpointDeleteRequests.inc({ result: "http_error" }); + this.logger.error("Failed to request checkpoint reclaim", { runFriendlyId, error }); + return; + } + + checkpointDeleteRequests.inc({ result: "sent" }); + } + /** * Sets common route meta on the wide-event state from URL params. */ @@ -364,6 +447,13 @@ export class WorkloadServer extends EventEmitter { } reply.json(completeResponse.data satisfies WorkloadRunAttemptCompleteResponseBody); + + await this.reclaimCheckpoints( + req, + params.runFriendlyId, + completeResponse.data.result.attemptStatus, + auth.claims + ); return; } ), diff --git a/packages/core/src/v3/serverOnly/checkpointClient.ts b/packages/core/src/v3/serverOnly/checkpointClient.ts index 0250b441912..936557c0acf 100644 --- a/packages/core/src/v3/serverOnly/checkpointClient.ts +++ b/packages/core/src/v3/serverOnly/checkpointClient.ts @@ -119,4 +119,42 @@ export class CheckpointClient { return true; } + + /** + * Ask the checkpoint service to reclaim a finished run's checkpoint storage. Best-effort: the + * service enqueues and returns 202, so a `true` here means accepted, not deleted. + */ + async deleteCheckpoints({ + runFriendlyId, + body, + }: { + runFriendlyId: string; + body: { + orgId: string; + envId: string; + projectRef: string; + deploymentVersion: string; + }; + }): Promise { + const res = await fetch( + new URL(`/api/v1/runs/${runFriendlyId}/checkpoints/delete`, this.opts.apiUrl), + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + } + ); + + if (!res.ok) { + this.logger.error("[CheckpointClient] Delete checkpoints request failed", { + runFriendlyId, + status: res.status, + }); + return false; + } + + return true; + } } From b4d6f670fb2cd0f678566f5a94c4fff9af6e4c62 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 3 Aug 2026 18:01:11 +0100 Subject: [PATCH 2/5] docs(supervisor): scope the reclaim comment to runner-driven completion --- apps/supervisor/src/workloadServer/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index d28eaedf775..05cc9353499 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -231,8 +231,12 @@ export class WorkloadServer extends EventEmitter { * reclaimCheckpoints asks the checkpoint service to delete a finished run's checkpoint storage. * * Called only after the reply has been sent, so it never delays the runner - the same shape the - * suspend route uses. Every early return is counted: with no lifecycle expiry behind this, a - * silently skipped delete is storage leaked forever, and silence must not look like success. + * suspend route uses. Every early return is counted: nothing reclaims storage behind this, so a + * silently skipped request leaks it, and silence must not look like success. + * + * This covers runner-driven completion only. A run that dies without posting one - killed pod, + * OOM, node loss, platform-side expiry - is finalised on the platform, which the worker never + * hears about, so those are not reclaimed here and are not reclaimable from this side. * * `RUN_PENDING_CANCEL` is terminal too - a run cancelled mid-execution never restores - so it is * reclaimed alongside `RUN_FINISHED`. Retries are deliberately excluded: the prefix is run-level, From 3388dd619c8524f87b498801c61de177e1d8538d Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 3 Aug 2026 18:28:48 +0100 Subject: [PATCH 3/5] fix(supervisor): split the skipped-reclaim metric by reason --- apps/supervisor/src/workloadServer/index.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index 05cc9353499..18099d37f1e 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -55,7 +55,8 @@ interface DefaultEventsMap { /** * checkpointDeleteRequests counts the delete requests this supervisor makes, and every reason it - * decides not to: `sent`, `disabled`, `not_terminal`, `no_claims`, `no_project_ref`, `http_error`. + * decides not to: `sent`, `disabled`, `no_client`, `not_applicable`, `not_terminal`, `no_claims`, + * `no_project_ref`, `http_error`. * Without the negative outcomes, "no deletes are happening" is indistinguishable from the feature * being switched off - and with no lifecycle expiry, that difference is leaked storage. */ @@ -248,11 +249,21 @@ export class WorkloadServer extends EventEmitter { attemptStatus: string, claims: WorkloadDeploymentTokenClaims | undefined ): Promise { - if (!env.DELETE_CHECKPOINTS_ON_COMPLETION || !this.checkpointClient || this.snapshotService) { + if (!env.DELETE_CHECKPOINTS_ON_COMPLETION) { checkpointDeleteRequests.inc({ result: "disabled" }); return; } + if (!this.checkpointClient) { + checkpointDeleteRequests.inc({ result: "no_client" }); + return; + } + + if (this.snapshotService) { + checkpointDeleteRequests.inc({ result: "not_applicable" }); + return; + } + if (attemptStatus !== "RUN_FINISHED" && attemptStatus !== "RUN_PENDING_CANCEL") { checkpointDeleteRequests.inc({ result: "not_terminal" }); return; From d12394014246c9e0000c722a9bc880e34b0e6cc3 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:58:35 +0100 Subject: [PATCH 4/5] fix(supervisor): refuse to start if reclaim is on without deployment tokens --- apps/supervisor/src/env.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 9733f56bc3d..6830d5b8642 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -327,6 +327,14 @@ export const Env = z path: ["WORKLOAD_TOKEN_SECRET"], }); } + if (data.DELETE_CHECKPOINTS_ON_COMPLETION && data.WORKLOAD_TOKEN_ENFORCEMENT === "disabled") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "DELETE_CHECKPOINTS_ON_COMPLETION needs WORKLOAD_TOKEN_ENFORCEMENT set to log or enforce: the tenancy it deletes by comes from the deployment token, so with tokens disabled it would silently reclaim nothing", + path: ["DELETE_CHECKPOINTS_ON_COMPLETION"], + }); + } if ( data.TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED && !data.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST From 98b7f2df555f9800254340d550f5da74f9e2b85f Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:10:37 +0100 Subject: [PATCH 5/5] chore(supervisor): cut comments back to contracts --- apps/supervisor/src/workloadServer/index.ts | 25 +++------------------ 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index 18099d37f1e..e14060e609a 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -53,13 +53,6 @@ interface DefaultEventsMap { [event: string]: (...args: any[]) => void; } -/** - * checkpointDeleteRequests counts the delete requests this supervisor makes, and every reason it - * decides not to: `sent`, `disabled`, `no_client`, `not_applicable`, `not_terminal`, `no_claims`, - * `no_project_ref`, `http_error`. - * Without the negative outcomes, "no deletes are happening" is indistinguishable from the feature - * being switched off - and with no lifecycle expiry, that difference is leaked storage. - */ const checkpointDeleteRequests = new Counter({ name: "checkpoint_delete_requests_total", help: "Checkpoint delete requests attempted at run completion, by outcome", @@ -199,9 +192,8 @@ export class WorkloadServer extends EventEmitter { * we still verify + record metrics but attach no header (so the platform never scopes). Only * enforce fails a request, and only for a present-but-invalid token; absent and legacy ids pass. * - * `claims` are returned whenever the token verifies, in either mode. They are for addressing a - * run's own resources locally (e.g. its checkpoint storage) - never for scoping the platform, - * which is why environmentId above stays gated on enforce. + * `claims` are returned on any valid token, for local use only - never to scope the platform, + * which is why environmentId stays gated on enforce. */ private async authorizeWorkloadRequest( req: IncomingMessage @@ -230,18 +222,7 @@ export class WorkloadServer extends EventEmitter { /** * reclaimCheckpoints asks the checkpoint service to delete a finished run's checkpoint storage. - * - * Called only after the reply has been sent, so it never delays the runner - the same shape the - * suspend route uses. Every early return is counted: nothing reclaims storage behind this, so a - * silently skipped request leaks it, and silence must not look like success. - * - * This covers runner-driven completion only. A run that dies without posting one - killed pod, - * OOM, node loss, platform-side expiry - is finalised on the platform, which the worker never - * hears about, so those are not reclaimed here and are not reclaimable from this side. - * - * `RUN_PENDING_CANCEL` is terminal too - a run cancelled mid-execution never restores - so it is - * reclaimed alongside `RUN_FINISHED`. Retries are deliberately excluded: the prefix is run-level, - * so a retry's checkpoints are cleaned by the final completion. + * Must be called after the reply is sent: it never delays the runner. */ private async reclaimCheckpoints( req: IncomingMessage,