From cfd373d1f9820667e2fe146741ed283d5810cb90 Mon Sep 17 00:00:00 2001 From: TomasPalsson Date: Thu, 8 Oct 2026 10:57:50 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=9F=EF=B8=8F=20fix:=20Never=20Reuse=20?= =?UTF-8?q?MicroVM=20Launch=20Tokens=20After=20a=20Counter=20Reset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stateful runtime sessions launch with clientToken sess--. The rtsx:gen counter expires with the session record (max duration + 10 minutes), and the next allocation returned the fixed config-derived seed again. A user returning after an idle night therefore resent the token of their first launch. AWS still remembers it and rejects the reuse with "The provided clientToken was used with different request parameters", even for a byte-identical request. That validation failure is not transient, so the PENDING intent kept the token and every later request replayed it until AWS forgot the token (about 24h in eu-west-1). - Salt the session and hosted-app generation seeds with random bytes so a reset counter always starts at a new generation. Token shape, range and length are unchanged; the allocate script still never lowers a live counter. - Retire a PENDING intent when AWS rejects its token as reused. AWS launched nothing, so the next request allocates a fresh generation. - Tests: registry loss on an unchanged config must not resend the token, and a reuse rejection must not be replayed. Fixtures that used the seed as a constant compute it once; seed tests assert the salt. --- service/src/hosted-app/control-plane.test.ts | 7 ++- .../src/hosted-app/microvm-runtime.test.ts | 4 +- service/src/hosted-app/microvm-runtime.ts | 7 ++- .../sandbox-backend/lambda-microvm.test.ts | 57 +++++++++++++++++-- service/src/sandbox-backend/lambda-microvm.ts | 32 +++++++---- 5 files changed, 85 insertions(+), 22 deletions(-) diff --git a/service/src/hosted-app/control-plane.test.ts b/service/src/hosted-app/control-plane.test.ts index 51d7e708..3c30ca58 100644 --- a/service/src/hosted-app/control-plane.test.ts +++ b/service/src/hosted-app/control-plane.test.ts @@ -52,9 +52,12 @@ const runtimeConfig: HostedAppMicrovmConfig = { tokenTps: 8, }; +/* The seed is salted per call, so fixtures share one allocation. */ +const seededGeneration = hostedAppLaunchGenerationSeed(runtimeConfig); + class MemoryRegistry implements HostedAppRegistry { record: RuntimeSessionRecord | null = null; - generation = hostedAppLaunchGenerationSeed(runtimeConfig); + generation = seededGeneration; writes: RuntimeSessionRecord[] = []; allocations = 0; @@ -260,7 +263,7 @@ test('does not report an expired pending intent as starting', () => { }); function pendingRecord(): RuntimeSessionRecord { - const generation = hostedAppLaunchGenerationSeed(runtimeConfig); + const generation = seededGeneration; return { runtime_session_id: input.hostedAppRuntimeId, tenant_id: input.tenantId, diff --git a/service/src/hosted-app/microvm-runtime.test.ts b/service/src/hosted-app/microvm-runtime.test.ts index eae7ad38..64221aa3 100644 --- a/service/src/hosted-app/microvm-runtime.test.ts +++ b/service/src/hosted-app/microvm-runtime.test.ts @@ -162,14 +162,14 @@ describe('HostedAppMicrovmRuntime', () => { expect(error).toBeInstanceOf(HostedAppMicrovmError); expect(error.transient).toBe(true); }); - test('seeds idempotency from exact wire inputs while keeping semantic matching order-independent', () => { + test('fingerprints exact wire inputs, keeps semantic matching order-independent, and salts seeds', () => { const first = { ...config(), ingressConnectorArns: ['arn:b', 'arn:a'] }; const reordered = { ...config(), ingressConnectorArns: ['arn:a', 'arn:b'] }; expect(hostedAppLaunchFingerprint(first)).toBe(hostedAppLaunchFingerprint(reordered)); expect(hostedAppLaunchRequestFingerprint(first)).not.toBe( hostedAppLaunchRequestFingerprint(reordered), ); - expect(hostedAppLaunchGenerationSeed(first)).not.toBe(hostedAppLaunchGenerationSeed(reordered)); + expect(hostedAppLaunchGenerationSeed(first)).not.toBe(hostedAppLaunchGenerationSeed(first)); }); test('launches the dedicated image with bounded idle policy and no egress connector', async () => { diff --git a/service/src/hosted-app/microvm-runtime.ts b/service/src/hosted-app/microvm-runtime.ts index f0cb69e9..3cb10941 100644 --- a/service/src/hosted-app/microvm-runtime.ts +++ b/service/src/hosted-app/microvm-runtime.ts @@ -12,7 +12,7 @@ import { type ThrottledOp, } from '../runtime-session/throttle'; import type { ResidentHostedAppSpec } from './spec'; -import { createHash } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; export interface HostedAppMicrovmConfig { imageArn: string; @@ -73,11 +73,14 @@ export function hostedAppLaunchRequestFingerprint(config: HostedAppMicrovmConfig }); } -/** Keep reset Redis counters in an image/config-specific safe-integer range. */ +/** Keep reset Redis counters in a safe-integer range. The random salt makes + * every reset counter start somewhere new, because AWS rejects a reused token + * it still remembers even when the request body is identical. */ export function hostedAppLaunchGenerationSeed(config: HostedAppMicrovmConfig): number { const offset = Number.parseInt( createHash('sha256') .update(hostedAppLaunchRequestFingerprint(config), 'utf8') + .update(randomBytes(16)) .digest('hex') .slice(0, 13), 16, diff --git a/service/src/sandbox-backend/lambda-microvm.test.ts b/service/src/sandbox-backend/lambda-microvm.test.ts index 02110edd..aec4307a 100644 --- a/service/src/sandbox-backend/lambda-microvm.test.ts +++ b/service/src/sandbox-backend/lambda-microvm.test.ts @@ -289,11 +289,11 @@ describe('normalizeMicrovmEndpoint', () => { }); describe('runtime session launch tokens', () => { - test('uses a deterministic launch namespace and stays within the AWS limit', () => { + test('salts every launch namespace seed and stays within the AWS limit', () => { const cfg = config(); const seed = runtimeSessionLaunchGenerationSeed(cfg); expect(seed).toBeGreaterThanOrEqual(RUNTIME_SESSION_NAMESPACED_GENERATION_MIN); - expect(runtimeSessionLaunchGenerationSeed({ ...cfg, imageVersion: '4' })).not.toBe(seed); + expect(runtimeSessionLaunchGenerationSeed(cfg)).not.toBe(seed); const runtimeSessionId = `rt_${'a'.repeat(40)}`; const token = runtimeSessionLaunchClientToken(runtimeSessionId, Number.MAX_SAFE_INTEGER); @@ -690,10 +690,7 @@ describe('LambdaMicrovmSandboxBackend session execution', () => { * (image builds stay hookless), so RunMicrovm carries no runHookPayload. */ expect(runArgs.runHookPayload).toBeUndefined(); expect(runArgs.idlePolicy?.autoResume).toBe(true); - expect(runArgs.clientToken).toBe(runtimeSessionLaunchClientToken( - 'rt_session_1', - runtimeSessionLaunchGenerationSeed(config()), - )); + expect(runArgs.clientToken).toMatch(/^sess-rt_session_1-\d+$/); expect(runArgs.maximumDurationSeconds).toBe(28_800); const executeReq = captured.find((c) => c.path === '/api/v2/execute'); @@ -704,6 +701,7 @@ describe('LambdaMicrovmSandboxBackend session execution', () => { expect(record?.microvm_id).toBe([...fake.vms.keys()][0]); expect(record?.generation).toBeGreaterThanOrEqual(RUNTIME_SESSION_NAMESPACED_GENERATION_MIN); expect(record?.launch_client_token).toBe(runArgs.clientToken); + expect(runArgs.clientToken).toBe(runtimeSessionLaunchClientToken('rt_session_1', record?.generation as number)); }); test('replays a legacy recorded launch intent after the RunMicrovm response is lost', async () => { @@ -875,6 +873,53 @@ describe('LambdaMicrovmSandboxBackend session execution', () => { expect((await readRuntimeSessionRecord('rt_session_1'))?.image_version).toBe('4'); }); + test('never reissues a launch token after registry loss on an unchanged config', async () => { + const fake = fakeClient(); + await makeBackend(fake).execute(request(), sessionContext()); + const firstToken = (fake.callsFor('runMicrovm')[0].args as { clientToken?: string }).clientToken; + + /* The rtsx:gen counter expires after an idle night while AWS still + * remembers the token and rejects its reuse ("The provided clientToken was + * used with different request parameters"), even for an identical body. + * A relaunch on the same config must therefore never resend the token. */ + await fake.terminateMicrovm([...fake.vms.keys()][0]); + await mock.del('rtsx:sess:rt_session_1', 'rtsx:gen:rt_session_1'); + + const second = await makeBackend(fake).execute(request(), sessionContext()) + .catch((error: unknown) => error); + const secondToken = (fake.callsFor('runMicrovm')[1].args as { clientToken?: string }).clientToken; + expect(secondToken).not.toBe(firstToken); + expect(second).toEqual(EXECUTE_RESPONSE); + }); + + test('stops replaying a PENDING token that AWS rejected as reused', async () => { + const fake = fakeClient(); + /* An ambiguous failure persists the PENDING intent and its token. */ + fake.failNext('runMicrovm', new LambdaMicrovmApiError('other', 'RunMicrovm', 'connection lost')); + await expect(makeBackend(fake).execute(request(), sessionContext())).rejects.toMatchObject({ + code: 'MICROVM_LAUNCH_FAILED', + }); + const poisoned = (await readRuntimeSessionRecord('rt_session_1'))?.launch_client_token; + expect(poisoned).toBeDefined(); + + fake.failNext('runMicrovm', new LambdaMicrovmApiError( + 'validation', + 'RunMicrovm', + 'The provided clientToken was used with different request parameters.', + )); + await expect(makeBackend(fake).execute(request(), sessionContext())).rejects.toMatchObject({ + code: 'MICROVM_LAUNCH_FAILED', + }); + + await expect(makeBackend(fake).execute(request(), sessionContext())) + .resolves.toEqual(EXECUTE_RESPONSE); + const tokens = fake.callsFor('runMicrovm') + .map(call => (call.args as { clientToken?: string }).clientToken); + expect(tokens).toHaveLength(3); + expect(tokens.slice(0, 2)).toEqual([poisoned, poisoned]); + expect(tokens[2]).not.toBe(poisoned); + }); + test('does not replay a persisted token when the exact connector request changed', async () => { const fake = fakeClient(); const firstRun = fake.runMicrovm.bind(fake); diff --git a/service/src/sandbox-backend/lambda-microvm.ts b/service/src/sandbox-backend/lambda-microvm.ts index e4ccf9a5..6e59d773 100644 --- a/service/src/sandbox-backend/lambda-microvm.ts +++ b/service/src/sandbox-backend/lambda-microvm.ts @@ -1,7 +1,7 @@ import axios from 'axios'; import { nanoid } from 'nanoid'; import * as fs from 'fs'; -import { createHash } from 'crypto'; +import { createHash, randomBytes } from 'crypto'; import type { LambdaMicrovmClient, MicrovmAuthToken, MicrovmDescription, MicrovmIdlePolicy } from '../runtime-session/lambda-client'; import type { SandboxBackend, SandboxExecuteContext, SandboxRawResponse, SandboxTransportRequest } from './types'; import type { RuntimeSessionRecord } from '../runtime-session/registry'; @@ -96,8 +96,8 @@ export function runtimeSessionLaunchFingerprint(config: LambdaMicrovmBackendConf } /** Generations below this boundary were allocated by the original INCR-only - * scheme. New launches start in a fingerprint-seeded namespace so losing the - * Redis counter cannot reuse an AWS clientToken after launch inputs change. + * scheme. New launches start in a randomly seeded namespace so losing the + * Redis counter cannot reuse an AWS clientToken. * * Keep the provider token itself in the legacy `sess--` shape: * an older worker taking over a PENDING record during a rolling deployment @@ -124,13 +124,17 @@ function runtimeSessionLaunchRequestFingerprint(config: LambdaMicrovmBackendConf }); } -/** A 52-bit digest leaves ample safe-integer headroom for later INCRs while - * making a reset counter's first generation depend on the exact launch - * request. Token collisions are scoped to one runtime session. */ +/** A 52-bit digest leaves ample safe-integer headroom for later INCRs. The + * random salt makes every reset counter start somewhere new: the rtsx:gen key + * expires after an idle night while AWS still remembers the old tokens, and AWS + * rejects a reused token ("The provided clientToken was used with different + * request parameters") even when the request body is identical. Token + * collisions are scoped to one runtime session. */ export function runtimeSessionLaunchGenerationSeed(config: LambdaMicrovmBackendConfig): number { const offset = Number.parseInt( createHash('sha256') .update(runtimeSessionLaunchRequestFingerprint(config), 'utf8') + .update(randomBytes(16)) .digest('hex') .slice(0, 13), 16, @@ -730,10 +734,18 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { && error.cause.kind === 'validation'; } + /** AWS reports a reused idempotency token only as a generic + * ValidationException, so the message text is the only discriminator. */ + private isClientTokenReuseRejection(error: unknown): boolean { + return this.isValidationLaunchFailure(error) + && /client ?token\b.*\bdifferent (request )?parameters/i.test((error as Error).message); + } + /** Both the base token and its single retry reached a terminal state and - * were successfully terminated. Keeping that PENDING intent would replay - * two known-dead tokens forever, so let the next request allocate a new - * generation. Ambiguous provider failures remain persisted for recovery. */ + * were successfully terminated, or AWS refused the token as already used and + * launched nothing. Keeping that PENDING intent would replay a known-dead + * token forever, so let the next request allocate a new generation. + * Ambiguous provider failures remain persisted for recovery. */ private async retireExhaustedLaunchIntent( launchIntent: RuntimeSessionRecord, lockToken: string, @@ -742,7 +754,7 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { const cleanlyExhausted = error instanceof SandboxBackendError && error.code === 'MICROVM_LAUNCH_FAILED' - && error.transient; + && (error.transient || this.isClientTokenReuseRejection(error)); if (!cleanlyExhausted) return; try { const retired = await writeRuntimeSessionRecord({