From 1ba96c3db03182c05b0b716a4b66de45f2a11929 Mon Sep 17 00:00:00 2001 From: TomasPalsson Date: Thu, 8 Oct 2026 10:34:15 +0000 Subject: [PATCH 1/3] test(service): failing tests for reused MicroVM launch tokens A stateful runtime session idle longer than the rtsx:gen TTL (8h10m) gets the fixed config-derived seed generation again, so its next launch resends sess--, the token its first launch used. AWS still remembers that token and rejects it with "The provided clientToken was used with different request parameters" (HTTP 400, non-transient). The PENDING intent then keeps that token and every later request replays it. Red run (bun 1.3.14, the CI version) on unfixed ad213ba: $ bun test src/sandbox-backend/lambda-microvm.test.ts -t "never reissues|stops replaying" expect(secondToken).not.toBe(firstToken) Expected: not "sess-rt_session_1-3972529254911613" expect(tokens[2]).not.toBe(poisoned) Expected: not "sess-rt_session_1-3972529254911613" 0 pass, 2 fail exit=1 --- .../sandbox-backend/lambda-microvm.test.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/service/src/sandbox-backend/lambda-microvm.test.ts b/service/src/sandbox-backend/lambda-microvm.test.ts index 6d20f1c5..a964ad7c 100644 --- a/service/src/sandbox-backend/lambda-microvm.test.ts +++ b/service/src/sandbox-backend/lambda-microvm.test.ts @@ -875,6 +875,53 @@ describe('LambdaMicrovmSandboxBackend session execution', () => { expect((await readRuntimeSessionRecord('rt_session_1'))?.image_version).toBe('4'); }); + test('never reissues a launch token after registry loss on an unchanged config', async () => { + const fake = fakeClient(); + await makeBackend(fake).execute(request(), sessionContext()); + const firstToken = (fake.callsFor('runMicrovm')[0].args as { clientToken?: string }).clientToken; + + /* The rtsx:gen counter expires after an idle night while AWS still + * remembers the token and rejects its reuse ("The provided clientToken was + * used with different request parameters"), even for an identical body. + * A relaunch on the same config must therefore never resend the token. */ + await fake.terminateMicrovm([...fake.vms.keys()][0]); + await mock.del('rtsx:sess:{rt_session_1}', 'rtsx:gen:{rt_session_1}'); + + const second = await makeBackend(fake).execute(request(), sessionContext()) + .catch((error: unknown) => error); + const secondToken = (fake.callsFor('runMicrovm')[1].args as { clientToken?: string }).clientToken; + expect(secondToken).not.toBe(firstToken); + expect(second).toEqual(EXECUTE_RESPONSE); + }); + + test('stops replaying a PENDING token that AWS rejected as reused', async () => { + const fake = fakeClient(); + /* An ambiguous failure persists the PENDING intent and its token. */ + fake.failNext('runMicrovm', new LambdaMicrovmApiError('other', 'RunMicrovm', 'connection lost')); + await expect(makeBackend(fake).execute(request(), sessionContext())).rejects.toMatchObject({ + code: 'MICROVM_LAUNCH_FAILED', + }); + const poisoned = (await readRuntimeSessionRecord('rt_session_1'))?.launch_client_token; + expect(poisoned).toBeDefined(); + + fake.failNext('runMicrovm', new LambdaMicrovmApiError( + 'validation', + 'RunMicrovm', + 'The provided clientToken was used with different request parameters.', + )); + await expect(makeBackend(fake).execute(request(), sessionContext())).rejects.toMatchObject({ + code: 'MICROVM_LAUNCH_FAILED', + }); + + await expect(makeBackend(fake).execute(request(), sessionContext())) + .resolves.toEqual(EXECUTE_RESPONSE); + const tokens = fake.callsFor('runMicrovm') + .map(call => (call.args as { clientToken?: string }).clientToken); + expect(tokens).toHaveLength(3); + expect(tokens.slice(0, 2)).toEqual([poisoned, poisoned]); + expect(tokens[2]).not.toBe(poisoned); + }); + test('does not replay a persisted token when the exact connector request changed', async () => { const fake = fakeClient(); const firstRun = fake.runMicrovm.bind(fake); From 0769fa8277a803e5b5a0477bd68ad343aae4cf26 Mon Sep 17 00:00:00 2001 From: TomasPalsson Date: Thu, 8 Oct 2026 10:38:02 +0000 Subject: [PATCH 2/3] fix(service): never reuse a MicroVM launch token after a counter reset Stateful runtime sessions launch with clientToken sess--. The rtsx:gen counter expires after 8h10m idle, and the next allocation returned the fixed config-derived seed again, so a user returning the next morning resent the token of their first launch. AWS still remembers it and rejects the reuse with "The provided clientToken was used with different request parameters", even for an identical request body. That validation failure is non-transient, so the PENDING intent kept the token and every later request replayed it until AWS forgot the token (~24h in eu-west-1). - Salt the session and hosted-app generation seeds with random bytes, so a reset counter always starts at a new generation. Token shape is unchanged. - Retire a PENDING intent when AWS rejects its token as reused; AWS launched nothing, so the next request allocates a fresh generation. Tests that used the seed as a fixed fixture value now compute it once; the hookless-launch test checks the token shape instead of the old fixed seed. --- service/src/hosted-app/control-plane.test.ts | 7 ++-- service/src/hosted-app/microvm-runtime.ts | 7 ++-- .../sandbox-backend/lambda-microvm.test.ts | 5 +-- service/src/sandbox-backend/lambda-microvm.ts | 32 +++++++++++++------ 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/service/src/hosted-app/control-plane.test.ts b/service/src/hosted-app/control-plane.test.ts index 51d7e708..3c30ca58 100644 --- a/service/src/hosted-app/control-plane.test.ts +++ b/service/src/hosted-app/control-plane.test.ts @@ -52,9 +52,12 @@ const runtimeConfig: HostedAppMicrovmConfig = { tokenTps: 8, }; +/* The seed is salted per call, so fixtures share one allocation. */ +const seededGeneration = hostedAppLaunchGenerationSeed(runtimeConfig); + class MemoryRegistry implements HostedAppRegistry { record: RuntimeSessionRecord | null = null; - generation = hostedAppLaunchGenerationSeed(runtimeConfig); + generation = seededGeneration; writes: RuntimeSessionRecord[] = []; allocations = 0; @@ -260,7 +263,7 @@ test('does not report an expired pending intent as starting', () => { }); function pendingRecord(): RuntimeSessionRecord { - const generation = hostedAppLaunchGenerationSeed(runtimeConfig); + const generation = seededGeneration; return { runtime_session_id: input.hostedAppRuntimeId, tenant_id: input.tenantId, diff --git a/service/src/hosted-app/microvm-runtime.ts b/service/src/hosted-app/microvm-runtime.ts index 8b594f37..bc0ac61b 100644 --- a/service/src/hosted-app/microvm-runtime.ts +++ b/service/src/hosted-app/microvm-runtime.ts @@ -12,7 +12,7 @@ import { type ThrottledOp, } from '../runtime-session/throttle'; import type { ResidentHostedAppSpec } from './spec'; -import { createHash } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; export interface HostedAppMicrovmConfig { imageArn: string; @@ -73,11 +73,14 @@ export function hostedAppLaunchRequestFingerprint(config: HostedAppMicrovmConfig }); } -/** Keep reset Redis counters in an image/config-specific safe-integer range. */ +/** Keep reset Redis counters in a safe-integer range. The random salt makes + * every reset counter start somewhere new, because AWS rejects a reused token + * it still remembers even when the request body is identical. */ export function hostedAppLaunchGenerationSeed(config: HostedAppMicrovmConfig): number { const offset = Number.parseInt( createHash('sha256') .update(hostedAppLaunchRequestFingerprint(config), 'utf8') + .update(randomBytes(16)) .digest('hex') .slice(0, 13), 16, diff --git a/service/src/sandbox-backend/lambda-microvm.test.ts b/service/src/sandbox-backend/lambda-microvm.test.ts index a964ad7c..4dec8fcc 100644 --- a/service/src/sandbox-backend/lambda-microvm.test.ts +++ b/service/src/sandbox-backend/lambda-microvm.test.ts @@ -690,10 +690,7 @@ describe('LambdaMicrovmSandboxBackend session execution', () => { * (image builds stay hookless), so RunMicrovm carries no runHookPayload. */ expect(runArgs.runHookPayload).toBeUndefined(); expect(runArgs.idlePolicy?.autoResume).toBe(true); - expect(runArgs.clientToken).toBe(runtimeSessionLaunchClientToken( - 'rt_session_1', - runtimeSessionLaunchGenerationSeed(config()), - )); + expect(runArgs.clientToken).toMatch(/^sess-rt_session_1-\d+$/); expect(runArgs.maximumDurationSeconds).toBe(28_800); const executeReq = captured.find((c) => c.path === '/api/v2/execute'); diff --git a/service/src/sandbox-backend/lambda-microvm.ts b/service/src/sandbox-backend/lambda-microvm.ts index d947c76e..b4478e97 100644 --- a/service/src/sandbox-backend/lambda-microvm.ts +++ b/service/src/sandbox-backend/lambda-microvm.ts @@ -1,7 +1,7 @@ import axios from 'axios'; import { nanoid } from 'nanoid'; import * as fs from 'fs'; -import { createHash } from 'crypto'; +import { createHash, randomBytes } from 'crypto'; import type { LambdaMicrovmClient, MicrovmAuthToken, MicrovmDescription, MicrovmIdlePolicy } from '../runtime-session/lambda-client'; import type { SandboxBackend, SandboxExecuteContext, SandboxRawResponse, SandboxTransportRequest } from './types'; import type { RuntimeSessionRecord } from '../runtime-session/registry'; @@ -96,8 +96,8 @@ export function runtimeSessionLaunchFingerprint(config: LambdaMicrovmBackendConf } /** Generations below this boundary were allocated by the original INCR-only - * scheme. New launches start in a fingerprint-seeded namespace so losing the - * Redis counter cannot reuse an AWS clientToken after launch inputs change. + * scheme. New launches start in a randomly seeded namespace so losing the + * Redis counter cannot reuse an AWS clientToken. * * Keep the provider token itself in the legacy `sess--` shape: * an older worker taking over a PENDING record during a rolling deployment @@ -124,13 +124,17 @@ function runtimeSessionLaunchRequestFingerprint(config: LambdaMicrovmBackendConf }); } -/** A 52-bit digest leaves ample safe-integer headroom for later INCRs while - * making a reset counter's first generation depend on the exact launch - * request. Token collisions are scoped to one runtime session. */ +/** A 52-bit digest leaves ample safe-integer headroom for later INCRs. The + * random salt makes every reset counter start somewhere new: the rtsx:gen key + * expires after an idle night while AWS still remembers the old tokens, and AWS + * rejects a reused token ("The provided clientToken was used with different + * request parameters") even when the request body is identical. Token + * collisions are scoped to one runtime session. */ export function runtimeSessionLaunchGenerationSeed(config: LambdaMicrovmBackendConfig): number { const offset = Number.parseInt( createHash('sha256') .update(runtimeSessionLaunchRequestFingerprint(config), 'utf8') + .update(randomBytes(16)) .digest('hex') .slice(0, 13), 16, @@ -731,9 +735,17 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { } /** Both the base token and its single retry reached a terminal state and - * were successfully terminated. Keeping that PENDING intent would replay - * two known-dead tokens forever, so let the next request allocate a new - * generation. Ambiguous provider failures remain persisted for recovery. */ + * were successfully terminated, or AWS refused the token as already used and + * launched nothing. Keeping that PENDING intent would replay a known-dead + * token forever, so let the next request allocate a new generation. + * Ambiguous provider failures remain persisted for recovery. */ + /** AWS reports a reused idempotency token only as a generic + * ValidationException, so the message text is the only discriminator. */ + private isClientTokenReuseRejection(error: unknown): boolean { + return this.isValidationLaunchFailure(error) + && /client ?token\b.*\bdifferent (request )?parameters/i.test((error as Error).message); + } + private async retireExhaustedLaunchIntent( launchIntent: RuntimeSessionRecord, lockToken: string, @@ -742,7 +754,7 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { const cleanlyExhausted = error instanceof SandboxBackendError && error.code === 'MICROVM_LAUNCH_FAILED' - && error.transient; + && (error.transient || this.isClientTokenReuseRejection(error)); if (!cleanlyExhausted) return; try { const retired = await writeRuntimeSessionRecord({ From 9bb52cab2af1921acf4156bf558982564f726116 Mon Sep 17 00:00:00 2001 From: TomasPalsson Date: Thu, 8 Oct 2026 10:46:16 +0000 Subject: [PATCH 3/3] test(service): assert the salted seed directly and pin the token to its generation Review follow-up: two seed assertions became true by chance once seeds were salted; they now check the salt. Moves the retire JSDoc back above its method. --- service/src/hosted-app/microvm-runtime.test.ts | 4 ++-- service/src/sandbox-backend/lambda-microvm.test.ts | 5 +++-- service/src/sandbox-backend/lambda-microvm.ts | 10 +++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/service/src/hosted-app/microvm-runtime.test.ts b/service/src/hosted-app/microvm-runtime.test.ts index eae7ad38..64221aa3 100644 --- a/service/src/hosted-app/microvm-runtime.test.ts +++ b/service/src/hosted-app/microvm-runtime.test.ts @@ -162,14 +162,14 @@ describe('HostedAppMicrovmRuntime', () => { expect(error).toBeInstanceOf(HostedAppMicrovmError); expect(error.transient).toBe(true); }); - test('seeds idempotency from exact wire inputs while keeping semantic matching order-independent', () => { + test('fingerprints exact wire inputs, keeps semantic matching order-independent, and salts seeds', () => { const first = { ...config(), ingressConnectorArns: ['arn:b', 'arn:a'] }; const reordered = { ...config(), ingressConnectorArns: ['arn:a', 'arn:b'] }; expect(hostedAppLaunchFingerprint(first)).toBe(hostedAppLaunchFingerprint(reordered)); expect(hostedAppLaunchRequestFingerprint(first)).not.toBe( hostedAppLaunchRequestFingerprint(reordered), ); - expect(hostedAppLaunchGenerationSeed(first)).not.toBe(hostedAppLaunchGenerationSeed(reordered)); + expect(hostedAppLaunchGenerationSeed(first)).not.toBe(hostedAppLaunchGenerationSeed(first)); }); test('launches the dedicated image with bounded idle policy and no egress connector', async () => { diff --git a/service/src/sandbox-backend/lambda-microvm.test.ts b/service/src/sandbox-backend/lambda-microvm.test.ts index 4dec8fcc..015b3a92 100644 --- a/service/src/sandbox-backend/lambda-microvm.test.ts +++ b/service/src/sandbox-backend/lambda-microvm.test.ts @@ -289,11 +289,11 @@ describe('normalizeMicrovmEndpoint', () => { }); describe('runtime session launch tokens', () => { - test('uses a deterministic launch namespace and stays within the AWS limit', () => { + test('salts every launch namespace seed and stays within the AWS limit', () => { const cfg = config(); const seed = runtimeSessionLaunchGenerationSeed(cfg); expect(seed).toBeGreaterThanOrEqual(RUNTIME_SESSION_NAMESPACED_GENERATION_MIN); - expect(runtimeSessionLaunchGenerationSeed({ ...cfg, imageVersion: '4' })).not.toBe(seed); + expect(runtimeSessionLaunchGenerationSeed(cfg)).not.toBe(seed); const runtimeSessionId = `rt_${'a'.repeat(40)}`; const token = runtimeSessionLaunchClientToken(runtimeSessionId, Number.MAX_SAFE_INTEGER); @@ -701,6 +701,7 @@ describe('LambdaMicrovmSandboxBackend session execution', () => { expect(record?.microvm_id).toBe([...fake.vms.keys()][0]); expect(record?.generation).toBeGreaterThanOrEqual(RUNTIME_SESSION_NAMESPACED_GENERATION_MIN); expect(record?.launch_client_token).toBe(runArgs.clientToken); + expect(runArgs.clientToken).toBe(runtimeSessionLaunchClientToken('rt_session_1', record?.generation as number)); }); test('replays a legacy recorded launch intent after the RunMicrovm response is lost', async () => { diff --git a/service/src/sandbox-backend/lambda-microvm.ts b/service/src/sandbox-backend/lambda-microvm.ts index b4478e97..00e73db2 100644 --- a/service/src/sandbox-backend/lambda-microvm.ts +++ b/service/src/sandbox-backend/lambda-microvm.ts @@ -734,11 +734,6 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { && error.cause.kind === 'validation'; } - /** Both the base token and its single retry reached a terminal state and - * were successfully terminated, or AWS refused the token as already used and - * launched nothing. Keeping that PENDING intent would replay a known-dead - * token forever, so let the next request allocate a new generation. - * Ambiguous provider failures remain persisted for recovery. */ /** AWS reports a reused idempotency token only as a generic * ValidationException, so the message text is the only discriminator. */ private isClientTokenReuseRejection(error: unknown): boolean { @@ -746,6 +741,11 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { && /client ?token\b.*\bdifferent (request )?parameters/i.test((error as Error).message); } + /** Both the base token and its single retry reached a terminal state and + * were successfully terminated, or AWS refused the token as already used and + * launched nothing. Keeping that PENDING intent would replay a known-dead + * token forever, so let the next request allocate a new generation. + * Ambiguous provider failures remain persisted for recovery. */ private async retireExhaustedLaunchIntent( launchIntent: RuntimeSessionRecord, lockToken: string,