diff --git a/service/src/hosted-app/control-plane.test.ts b/service/src/hosted-app/control-plane.test.ts index 51d7e708..3c30ca58 100644 --- a/service/src/hosted-app/control-plane.test.ts +++ b/service/src/hosted-app/control-plane.test.ts @@ -52,9 +52,12 @@ const runtimeConfig: HostedAppMicrovmConfig = { tokenTps: 8, }; +/* The seed is salted per call, so fixtures share one allocation. */ +const seededGeneration = hostedAppLaunchGenerationSeed(runtimeConfig); + class MemoryRegistry implements HostedAppRegistry { record: RuntimeSessionRecord | null = null; - generation = hostedAppLaunchGenerationSeed(runtimeConfig); + generation = seededGeneration; writes: RuntimeSessionRecord[] = []; allocations = 0; @@ -260,7 +263,7 @@ test('does not report an expired pending intent as starting', () => { }); function pendingRecord(): RuntimeSessionRecord { - const generation = hostedAppLaunchGenerationSeed(runtimeConfig); + const generation = seededGeneration; return { runtime_session_id: input.hostedAppRuntimeId, tenant_id: input.tenantId, diff --git a/service/src/hosted-app/microvm-runtime.test.ts b/service/src/hosted-app/microvm-runtime.test.ts index eae7ad38..64221aa3 100644 --- a/service/src/hosted-app/microvm-runtime.test.ts +++ b/service/src/hosted-app/microvm-runtime.test.ts @@ -162,14 +162,14 @@ describe('HostedAppMicrovmRuntime', () => { expect(error).toBeInstanceOf(HostedAppMicrovmError); expect(error.transient).toBe(true); }); - test('seeds idempotency from exact wire inputs while keeping semantic matching order-independent', () => { + test('fingerprints exact wire inputs, keeps semantic matching order-independent, and salts seeds', () => { const first = { ...config(), ingressConnectorArns: ['arn:b', 'arn:a'] }; const reordered = { ...config(), ingressConnectorArns: ['arn:a', 'arn:b'] }; expect(hostedAppLaunchFingerprint(first)).toBe(hostedAppLaunchFingerprint(reordered)); expect(hostedAppLaunchRequestFingerprint(first)).not.toBe( hostedAppLaunchRequestFingerprint(reordered), ); - expect(hostedAppLaunchGenerationSeed(first)).not.toBe(hostedAppLaunchGenerationSeed(reordered)); + expect(hostedAppLaunchGenerationSeed(first)).not.toBe(hostedAppLaunchGenerationSeed(first)); }); test('launches the dedicated image with bounded idle policy and no egress connector', async () => { diff --git a/service/src/hosted-app/microvm-runtime.ts b/service/src/hosted-app/microvm-runtime.ts index f0cb69e9..3cb10941 100644 --- a/service/src/hosted-app/microvm-runtime.ts +++ b/service/src/hosted-app/microvm-runtime.ts @@ -12,7 +12,7 @@ import { type ThrottledOp, } from '../runtime-session/throttle'; import type { ResidentHostedAppSpec } from './spec'; -import { createHash } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; export interface HostedAppMicrovmConfig { imageArn: string; @@ -73,11 +73,14 @@ export function hostedAppLaunchRequestFingerprint(config: HostedAppMicrovmConfig }); } -/** Keep reset Redis counters in an image/config-specific safe-integer range. */ +/** Keep reset Redis counters in a safe-integer range. The random salt makes + * every reset counter start somewhere new, because AWS rejects a reused token + * it still remembers even when the request body is identical. */ export function hostedAppLaunchGenerationSeed(config: HostedAppMicrovmConfig): number { const offset = Number.parseInt( createHash('sha256') .update(hostedAppLaunchRequestFingerprint(config), 'utf8') + .update(randomBytes(16)) .digest('hex') .slice(0, 13), 16, diff --git a/service/src/sandbox-backend/lambda-microvm.test.ts b/service/src/sandbox-backend/lambda-microvm.test.ts index 02110edd..aec4307a 100644 --- a/service/src/sandbox-backend/lambda-microvm.test.ts +++ b/service/src/sandbox-backend/lambda-microvm.test.ts @@ -289,11 +289,11 @@ describe('normalizeMicrovmEndpoint', () => { }); describe('runtime session launch tokens', () => { - test('uses a deterministic launch namespace and stays within the AWS limit', () => { + test('salts every launch namespace seed and stays within the AWS limit', () => { const cfg = config(); const seed = runtimeSessionLaunchGenerationSeed(cfg); expect(seed).toBeGreaterThanOrEqual(RUNTIME_SESSION_NAMESPACED_GENERATION_MIN); - expect(runtimeSessionLaunchGenerationSeed({ ...cfg, imageVersion: '4' })).not.toBe(seed); + expect(runtimeSessionLaunchGenerationSeed(cfg)).not.toBe(seed); const runtimeSessionId = `rt_${'a'.repeat(40)}`; const token = runtimeSessionLaunchClientToken(runtimeSessionId, Number.MAX_SAFE_INTEGER); @@ -690,10 +690,7 @@ describe('LambdaMicrovmSandboxBackend session execution', () => { * (image builds stay hookless), so RunMicrovm carries no runHookPayload. */ expect(runArgs.runHookPayload).toBeUndefined(); expect(runArgs.idlePolicy?.autoResume).toBe(true); - expect(runArgs.clientToken).toBe(runtimeSessionLaunchClientToken( - 'rt_session_1', - runtimeSessionLaunchGenerationSeed(config()), - )); + expect(runArgs.clientToken).toMatch(/^sess-rt_session_1-\d+$/); expect(runArgs.maximumDurationSeconds).toBe(28_800); const executeReq = captured.find((c) => c.path === '/api/v2/execute'); @@ -704,6 +701,7 @@ describe('LambdaMicrovmSandboxBackend session execution', () => { expect(record?.microvm_id).toBe([...fake.vms.keys()][0]); expect(record?.generation).toBeGreaterThanOrEqual(RUNTIME_SESSION_NAMESPACED_GENERATION_MIN); expect(record?.launch_client_token).toBe(runArgs.clientToken); + expect(runArgs.clientToken).toBe(runtimeSessionLaunchClientToken('rt_session_1', record?.generation as number)); }); test('replays a legacy recorded launch intent after the RunMicrovm response is lost', async () => { @@ -875,6 +873,53 @@ describe('LambdaMicrovmSandboxBackend session execution', () => { expect((await readRuntimeSessionRecord('rt_session_1'))?.image_version).toBe('4'); }); + test('never reissues a launch token after registry loss on an unchanged config', async () => { + const fake = fakeClient(); + await makeBackend(fake).execute(request(), sessionContext()); + const firstToken = (fake.callsFor('runMicrovm')[0].args as { clientToken?: string }).clientToken; + + /* The rtsx:gen counter expires after an idle night while AWS still + * remembers the token and rejects its reuse ("The provided clientToken was + * used with different request parameters"), even for an identical body. + * A relaunch on the same config must therefore never resend the token. */ + await fake.terminateMicrovm([...fake.vms.keys()][0]); + await mock.del('rtsx:sess:rt_session_1', 'rtsx:gen:rt_session_1'); + + const second = await makeBackend(fake).execute(request(), sessionContext()) + .catch((error: unknown) => error); + const secondToken = (fake.callsFor('runMicrovm')[1].args as { clientToken?: string }).clientToken; + expect(secondToken).not.toBe(firstToken); + expect(second).toEqual(EXECUTE_RESPONSE); + }); + + test('stops replaying a PENDING token that AWS rejected as reused', async () => { + const fake = fakeClient(); + /* An ambiguous failure persists the PENDING intent and its token. */ + fake.failNext('runMicrovm', new LambdaMicrovmApiError('other', 'RunMicrovm', 'connection lost')); + await expect(makeBackend(fake).execute(request(), sessionContext())).rejects.toMatchObject({ + code: 'MICROVM_LAUNCH_FAILED', + }); + const poisoned = (await readRuntimeSessionRecord('rt_session_1'))?.launch_client_token; + expect(poisoned).toBeDefined(); + + fake.failNext('runMicrovm', new LambdaMicrovmApiError( + 'validation', + 'RunMicrovm', + 'The provided clientToken was used with different request parameters.', + )); + await expect(makeBackend(fake).execute(request(), sessionContext())).rejects.toMatchObject({ + code: 'MICROVM_LAUNCH_FAILED', + }); + + await expect(makeBackend(fake).execute(request(), sessionContext())) + .resolves.toEqual(EXECUTE_RESPONSE); + const tokens = fake.callsFor('runMicrovm') + .map(call => (call.args as { clientToken?: string }).clientToken); + expect(tokens).toHaveLength(3); + expect(tokens.slice(0, 2)).toEqual([poisoned, poisoned]); + expect(tokens[2]).not.toBe(poisoned); + }); + test('does not replay a persisted token when the exact connector request changed', async () => { const fake = fakeClient(); const firstRun = fake.runMicrovm.bind(fake); diff --git a/service/src/sandbox-backend/lambda-microvm.ts b/service/src/sandbox-backend/lambda-microvm.ts index e4ccf9a5..6e59d773 100644 --- a/service/src/sandbox-backend/lambda-microvm.ts +++ b/service/src/sandbox-backend/lambda-microvm.ts @@ -1,7 +1,7 @@ import axios from 'axios'; import { nanoid } from 'nanoid'; import * as fs from 'fs'; -import { createHash } from 'crypto'; +import { createHash, randomBytes } from 'crypto'; import type { LambdaMicrovmClient, MicrovmAuthToken, MicrovmDescription, MicrovmIdlePolicy } from '../runtime-session/lambda-client'; import type { SandboxBackend, SandboxExecuteContext, SandboxRawResponse, SandboxTransportRequest } from './types'; import type { RuntimeSessionRecord } from '../runtime-session/registry'; @@ -96,8 +96,8 @@ export function runtimeSessionLaunchFingerprint(config: LambdaMicrovmBackendConf } /** Generations below this boundary were allocated by the original INCR-only - * scheme. New launches start in a fingerprint-seeded namespace so losing the - * Redis counter cannot reuse an AWS clientToken after launch inputs change. + * scheme. New launches start in a randomly seeded namespace so losing the + * Redis counter cannot reuse an AWS clientToken. * * Keep the provider token itself in the legacy `sess--` shape: * an older worker taking over a PENDING record during a rolling deployment @@ -124,13 +124,17 @@ function runtimeSessionLaunchRequestFingerprint(config: LambdaMicrovmBackendConf }); } -/** A 52-bit digest leaves ample safe-integer headroom for later INCRs while - * making a reset counter's first generation depend on the exact launch - * request. Token collisions are scoped to one runtime session. */ +/** A 52-bit digest leaves ample safe-integer headroom for later INCRs. The + * random salt makes every reset counter start somewhere new: the rtsx:gen key + * expires after an idle night while AWS still remembers the old tokens, and AWS + * rejects a reused token ("The provided clientToken was used with different + * request parameters") even when the request body is identical. Token + * collisions are scoped to one runtime session. */ export function runtimeSessionLaunchGenerationSeed(config: LambdaMicrovmBackendConfig): number { const offset = Number.parseInt( createHash('sha256') .update(runtimeSessionLaunchRequestFingerprint(config), 'utf8') + .update(randomBytes(16)) .digest('hex') .slice(0, 13), 16, @@ -730,10 +734,18 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { && error.cause.kind === 'validation'; } + /** AWS reports a reused idempotency token only as a generic + * ValidationException, so the message text is the only discriminator. */ + private isClientTokenReuseRejection(error: unknown): boolean { + return this.isValidationLaunchFailure(error) + && /client ?token\b.*\bdifferent (request )?parameters/i.test((error as Error).message); + } + /** Both the base token and its single retry reached a terminal state and - * were successfully terminated. Keeping that PENDING intent would replay - * two known-dead tokens forever, so let the next request allocate a new - * generation. Ambiguous provider failures remain persisted for recovery. */ + * were successfully terminated, or AWS refused the token as already used and + * launched nothing. Keeping that PENDING intent would replay a known-dead + * token forever, so let the next request allocate a new generation. + * Ambiguous provider failures remain persisted for recovery. */ private async retireExhaustedLaunchIntent( launchIntent: RuntimeSessionRecord, lockToken: string, @@ -742,7 +754,7 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { const cleanlyExhausted = error instanceof SandboxBackendError && error.code === 'MICROVM_LAUNCH_FAILED' - && error.transient; + && (error.transient || this.isClientTokenReuseRejection(error)); if (!cleanlyExhausted) return; try { const retired = await writeRuntimeSessionRecord({