Skip to content

Commit 2af68e0

Browse files
authored
Merge branch 'main' into samejr/org-menu-update
2 parents ac3bdfb + 325b906 commit 2af68e0

145 files changed

Lines changed: 3957 additions & 359 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.env.example

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
SESSION_SECRET=abcdef1234
33
MAGIC_LINK_SECRET=abcdef1234
44
ENCRYPTION_KEY=ae13021afef0819c3a307ad487071c06 # Must be a random 16 byte hex string. You can generate an encryption key by running `openssl rand -hex 16` in your terminal
5+
MANAGED_WORKER_SECRET=abcdef1234 # Must match the supervisor's MANAGED_WORKER_SECRET
56
LOGIN_ORIGIN=http://localhost:3030
67
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres?schema=public
78
# This sets the URL used for direct connections to the database and should only be needed in limited circumstances

.github/workflows/helm-prerelease.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,14 @@ jobs:
5252
5353
- name: Lint Helm Chart
5454
run: |
55-
helm lint ./hosting/k8s/helm/
55+
helm lint ./hosting/k8s/helm/ \
56+
--values ./hosting/k8s/helm/ci/lint-values.yaml
5657
5758
- name: Render templates
5859
run: |
5960
helm template test-release ./hosting/k8s/helm/ \
6061
--values ./hosting/k8s/helm/values.yaml \
62+
--values ./hosting/k8s/helm/ci/lint-values.yaml \
6163
--output-dir ./helm-output
6264
6365
- name: Validate manifests

.github/workflows/release-helm.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,14 @@ jobs:
4747
4848
- name: Lint Helm Chart
4949
run: |
50-
helm lint ./hosting/k8s/helm/
50+
helm lint ./hosting/k8s/helm/ \
51+
--values ./hosting/k8s/helm/ci/lint-values.yaml
5152
5253
- name: Render templates
5354
run: |
5455
helm template test-release ./hosting/k8s/helm/ \
5556
--values ./hosting/k8s/helm/values.yaml \
57+
--values ./hosting/k8s/helm/ci/lint-values.yaml \
5658
--output-dir ./helm-output
5759
5860
- name: Validate manifests

apps/supervisor/.env.example

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# This needs to match the token of the worker group you want to connect to
22
TRIGGER_WORKER_TOKEN=
33

4-
# This needs to match the MANAGED_WORKER_SECRET env var on the webapp
5-
MANAGED_WORKER_SECRET=managed-secret
4+
# Must match the webapp's MANAGED_WORKER_SECRET. Generate with: openssl rand -hex 16
5+
MANAGED_WORKER_SECRET=
66

77
# Point this at the webapp in prod
88
TRIGGER_API_URL=http://localhost:3030

apps/supervisor/src/env.ts

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,17 @@ export const Env = z
1414

1515
// Required settings
1616
TRIGGER_API_URL: z.string().url(),
17-
TRIGGER_WORKER_TOKEN: z.string(), // accepts file:// path to read from a file
17+
TRIGGER_WORKER_TOKEN: z.string().min(1), // accepts file:// path to read from a file
1818
MANAGED_WORKER_SECRET: z.string(),
19+
20+
// Deployment token: sign a token into TRIGGER_DEPLOYMENT_ID at pod creation and verify it on
21+
// inbound workload calls. "disabled" = off; "log" = mint + verify + metrics only; "enforce" =
22+
// also reject invalid tokens.
23+
WORKLOAD_TOKEN_SECRET: z.string().optional(),
24+
WORKLOAD_TOKEN_ENFORCEMENT: z.enum(["disabled", "log", "enforce"]).default("disabled"),
25+
// Absolute expiry for minted deployment tokens. Deterministic (no wall-clock issued-at) so every
26+
// pod of a deployment carries an identical token; bump before this date. Must outlive any run.
27+
WORKLOAD_TOKEN_EXP: z.string().datetime().default("2032-01-01T00:00:00.000Z"),
1928
OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), // set on the runners
2029

2130
// Workload API settings (coordinator mode) - the workload API is what the run controller connects to
@@ -365,6 +374,14 @@ export const Env = z
365374
path: ["TRIGGER_WORKLOAD_API_DOMAIN"],
366375
});
367376
}
377+
if (data.WORKLOAD_TOKEN_ENFORCEMENT !== "disabled" && !data.WORKLOAD_TOKEN_SECRET) {
378+
ctx.addIssue({
379+
code: z.ZodIssueCode.custom,
380+
message:
381+
"WORKLOAD_TOKEN_SECRET is required when WORKLOAD_TOKEN_ENFORCEMENT is not disabled",
382+
path: ["WORKLOAD_TOKEN_SECRET"],
383+
});
384+
}
368385
if (
369386
data.TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED &&
370387
!data.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST

apps/supervisor/src/index.ts

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import { register } from "./metrics.js";
2727
import { PodCleaner } from "./services/podCleaner.js";
2828
import { FailedPodHandler } from "./services/failedPodHandler.js";
2929
import { getWorkerToken } from "./workerToken.js";
30+
import { mintDeploymentToken } from "./workloadToken.js";
3031
import { OtlpTraceService } from "./services/otlpTraceService.js";
3132
import {
3233
WarmStartVerificationService,
@@ -96,6 +97,7 @@ class ManagedSupervisor {
9697
COMPUTE_GATEWAY_AUTH_TOKEN,
9798
DOCKER_REGISTRY_PASSWORD,
9899
TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD,
100+
WORKLOAD_TOKEN_SECRET,
99101
...envWithoutSecrets
100102
} = env;
101103

@@ -290,8 +292,10 @@ class ManagedSupervisor {
290292
});
291293
}
292294

295+
const workerToken = getWorkerToken();
296+
293297
this.workerSession = new SupervisorSession({
294-
workerToken: getWorkerToken(),
298+
workerToken,
295299
apiUrl: env.TRIGGER_API_URL,
296300
instanceName: env.TRIGGER_WORKER_INSTANCE_NAME,
297301
managedWorkerSecret: env.MANAGED_WORKER_SECRET,
@@ -569,6 +573,7 @@ class ManagedSupervisor {
569573
checkpointClient: this.checkpointClient,
570574
computeManager: this.computeManager,
571575
tracing: this.tracing,
576+
snapshotCallbackSecret: workerToken,
572577
wideEventOpts: this.wideEventOpts,
573578
wideEventsNoisyRoutes: this.wideEventsNoisyRoutes,
574579
});
@@ -603,6 +608,15 @@ class ManagedSupervisor {
603608
throw new Error("Image is missing");
604609
}
605610

611+
const deploymentToken = await mintDeploymentToken({
612+
deployment: message.deployment.friendlyId,
613+
deployment_version: message.backgroundWorker.version,
614+
environment_id: message.environment.id,
615+
environment_type: message.environment.type,
616+
org_id: message.organization.id,
617+
project_id: message.project.id,
618+
});
619+
606620
await this.workloadManager.create({
607621
dequeuedAt: message.dequeuedAt,
608622
dequeueResponseMs: timings.dequeueResponseMs,
@@ -617,6 +631,7 @@ class ManagedSupervisor {
617631
deploymentFriendlyId: message.deployment.friendlyId,
618632
deploymentVersion: message.backgroundWorker.version,
619633
runtime: message.backgroundWorker.runtime,
634+
deploymentToken,
620635
runId: message.run.id,
621636
runFriendlyId: message.run.friendlyId,
622637
version: message.version,

apps/supervisor/src/services/computeSnapshotService.test.ts

Lines changed: 116 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,26 @@ function createService() {
2020
snapshot,
2121
} as unknown as ComputeWorkloadManager;
2222

23+
const submitSuspendCompletion = vi.fn(async () => ({ success: true }));
24+
2325
const service = new ComputeSnapshotService({
2426
computeManager,
25-
workerClient: {} as SupervisorHttpClient,
27+
workerClient: { submitSuspendCompletion } as unknown as SupervisorHttpClient,
2628
wideEventOpts: { service: "supervisor-test", env: {}, enabled: false },
29+
snapshotCallbackSecret: "test-secret",
2730
});
2831

29-
return { service, snapshot };
32+
return { service, snapshot, submitSuspendCompletion };
33+
}
34+
35+
function dispatchedMetadata(snapshot: {
36+
mock: { calls: Array<Array<{ metadata?: Record<string, string> }>> };
37+
}) {
38+
const metadata = snapshot.mock.calls[0]?.[0]?.metadata;
39+
if (!metadata) {
40+
throw new Error("Snapshot was not dispatched");
41+
}
42+
return metadata;
3043
}
3144

3245
function delayedSnapshot(runnerId = "runner-1") {
@@ -38,6 +51,24 @@ function delayedSnapshot(runnerId = "runner-1") {
3851
}
3952

4053
describe("ComputeSnapshotService", () => {
54+
it("refuses to construct with an empty callback secret", () => {
55+
const computeManager = {
56+
snapshotDelayMs: DELAY_MS,
57+
snapshotDispatchLimit: 1,
58+
snapshot: vi.fn(async () => true),
59+
} as unknown as ComputeWorkloadManager;
60+
61+
expect(
62+
() =>
63+
new ComputeSnapshotService({
64+
computeManager,
65+
workerClient: {} as SupervisorHttpClient,
66+
wideEventOpts: { service: "supervisor-test", env: {}, enabled: false },
67+
snapshotCallbackSecret: "",
68+
})
69+
).toThrow();
70+
});
71+
4172
it("dispatches a scheduled snapshot after the delay", async () => {
4273
const { service, snapshot } = createService();
4374
try {
@@ -46,7 +77,12 @@ describe("ComputeSnapshotService", () => {
4677
await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 });
4778
expect(snapshot).toHaveBeenCalledWith({
4879
runnerId: "runner-1",
49-
metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_1" },
80+
metadata: expect.objectContaining({
81+
runId: "run_1",
82+
snapshotFriendlyId: "snapshot_1",
83+
snapshotCallbackNonce: expect.any(String),
84+
snapshotCallbackToken: expect.any(String),
85+
}),
5086
});
5187
} finally {
5288
service.stop();
@@ -121,8 +157,84 @@ describe("ComputeSnapshotService", () => {
121157
expect(snapshot).toHaveBeenCalledTimes(1);
122158
expect(snapshot).toHaveBeenCalledWith({
123159
runnerId: "runner-1",
124-
metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_2" },
160+
metadata: expect.objectContaining({
161+
runId: "run_1",
162+
snapshotFriendlyId: "snapshot_2",
163+
snapshotCallbackNonce: expect.any(String),
164+
snapshotCallbackToken: expect.any(String),
165+
}),
166+
});
167+
} finally {
168+
service.stop();
169+
}
170+
});
171+
172+
it("accepts a snapshot callback with the dispatched token", async () => {
173+
const { service, snapshot, submitSuspendCompletion } = createService();
174+
try {
175+
service.schedule("run_1", delayedSnapshot());
176+
177+
await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 });
178+
const metadata = dispatchedMetadata(snapshot);
179+
180+
const result = await service.handleCallback({
181+
status: "completed",
182+
instance_id: "instance_1",
183+
snapshot_id: "compute_snapshot_1",
184+
metadata,
185+
});
186+
187+
expect(result).toEqual({ ok: true, status: 200 });
188+
expect(submitSuspendCompletion).toHaveBeenCalledWith({
189+
runId: "run_1",
190+
snapshotId: "snapshot_1",
191+
body: {
192+
success: true,
193+
checkpoint: {
194+
type: "COMPUTE",
195+
location: "compute_snapshot_1",
196+
},
197+
},
198+
});
199+
} finally {
200+
service.stop();
201+
}
202+
});
203+
204+
it("rejects a snapshot callback without a valid token", async () => {
205+
const { service, submitSuspendCompletion } = createService();
206+
try {
207+
const result = await service.handleCallback({
208+
status: "completed",
209+
instance_id: "instance_1",
210+
snapshot_id: "compute_snapshot_1",
211+
metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_1" },
212+
});
213+
214+
expect(result).toEqual({ ok: false, status: 401 });
215+
expect(submitSuspendCompletion).not.toHaveBeenCalled();
216+
} finally {
217+
service.stop();
218+
}
219+
});
220+
221+
it("rejects a snapshot callback whose token is for a different snapshot", async () => {
222+
const { service, snapshot, submitSuspendCompletion } = createService();
223+
try {
224+
service.schedule("run_1", delayedSnapshot());
225+
226+
await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 });
227+
const metadata = dispatchedMetadata(snapshot);
228+
229+
const result = await service.handleCallback({
230+
status: "completed",
231+
instance_id: "instance_1",
232+
snapshot_id: "compute_snapshot_1",
233+
metadata: { ...metadata, snapshotFriendlyId: "snapshot_2" },
125234
});
235+
236+
expect(result).toEqual({ ok: false, status: 401 });
237+
expect(submitSuspendCompletion).not.toHaveBeenCalled();
126238
} finally {
127239
service.stop();
128240
}

0 commit comments

Comments
 (0)