diff --git a/lambdas/libs/compute-providers/aws/ec2/scale-set.ts b/lambdas/libs/compute-providers/aws/ec2/scale-set.ts new file mode 100644 index 0000000000..671ddc69f7 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/scale-set.ts @@ -0,0 +1,32 @@ +import type { ScaleSetComputeProviderModule, ScaleSetComputeProviderPlugin } from '../../scale-set'; + +import { createEc2ScaleSetProvider, type Ec2ScaleSetProviderDependencies } from './src/scale-set/provider'; + +export type { Ec2ScaleSetProviderConfig, Ec2ScaleSetProviderDependencies } from './src/scale-set/provider'; +export { createEc2ScaleSetProvider, parseEc2ScaleSetProviderConfig } from './src/scale-set/provider'; + +export function createEc2ScaleSetPlugin( + dependencies: Ec2ScaleSetProviderDependencies = {}, +): ScaleSetComputeProviderPlugin<'ec2'> { + return { + type: 'ec2', + capabilities: { + environmentVariables: {}, + create: ({ runnerConfigName, scaleSetId, githubScope, configuration }) => + createEc2ScaleSetProvider( + { + runnerConfigName, + scaleSetId, + githubScope, + configuration: configuration as Parameters[0]['configuration'], + }, + dependencies, + ), + }, + }; +} + +export const provider = { + type: 'ec2', + createPlugin: createEc2ScaleSetPlugin, +} satisfies ScaleSetComputeProviderModule<'ec2'>; diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts new file mode 100644 index 0000000000..d6483f3204 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest'; + +import { parseEc2ScaleSetProviderConfig } from './configuration'; +import { config } from './test/fixtures'; + +describe('EC2 scale-set provider configuration', () => { + it('strictly parses the supported provider-owned configuration', () => { + expect(parseEc2ScaleSetProviderConfig(config)).toMatchObject(config); + expect(parseEc2ScaleSetProviderConfig({ ...config, runnerNamePrefix: '' })).toMatchObject({ + runnerNamePrefix: '', + }); + expect(parseEc2ScaleSetProviderConfig({ ...config, runnerNamePrefix: 'r'.repeat(45) })).toMatchObject({ + runnerNamePrefix: 'r'.repeat(45), + }); + }); + + it.each([ + [{ ...config, region: 'eu-west-one' }], + [{ ...config, subnets: ['subnet-12345678', 'subnet-12345678'] }], + [{ ...config, ec2instanceCriteria: { ...config.ec2instanceCriteria, instanceAllocationStrategy: 'diversified' } }], + [{ ...config, ec2OverrideConfig: { UserData: 'untrusted' } }], + [{ ...config, scaleErrors: ['ThrottlingException'] }], + [{ ...config, ssmParameterTags: [{ Key: 'aws:owner', Value: 'untrusted' }] }], + [{ ...config, runnerNamePrefix: 'r'.repeat(46) }], + [{ ...config, bootTimeoutMinutes: 10 }], + ])('rejects invalid or unsupported values instead of forwarding them to AWS', (invalid) => { + expect(() => parseEc2ScaleSetProviderConfig(invalid)).toThrow(); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts new file mode 100644 index 0000000000..1d03728cb8 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts @@ -0,0 +1,346 @@ +import type { Tag as SsmTag } from '@aws-sdk/client-ssm'; + +import type { Ec2OverrideConfig, RunnerInputParameters } from '../runners.d'; +import { isRecord, Ec2ScaleSetValidationError } from './reconcile'; + +const SPOT_ALLOCATION_STRATEGIES = new Set([ + 'lowest-price', + 'diversified', + 'capacity-optimized', + 'capacity-optimized-prioritized', + 'price-capacity-optimized', +]); +const ON_DEMAND_ALLOCATION_STRATEGIES = new Set(['lowest-price', 'prioritized']); + +export interface Ec2ScaleSetProviderConfig { + region: string; + environment: string; + runnerNamePrefix: string; + jitConfigParameterPath: string; + subnets: string[]; + launchTemplateName: string; + ec2instanceCriteria: RunnerInputParameters['ec2instanceCriteria']; + ec2OverrideConfig?: Ec2OverrideConfig; + amiIdSsmParameterName?: string; + tracingEnabled?: boolean; + onDemandFailoverOnError?: string[]; + useDedicatedHost?: boolean; + ssmKmsKeyId?: string; + ssmParameterTags?: SsmTag[]; +} + +export interface CreateEc2ScaleSetProviderInput { + runnerConfigName: string; + scaleSetId: number; + githubScope: string; + configuration: Ec2ScaleSetProviderConfig; +} + +function rejectUnknownKeys(value: Record, allowedKeys: ReadonlySet, name: string): void { + const unknownKey = Object.keys(value).find((key) => !allowedKeys.has(key)); + if (unknownKey !== undefined) { + throw new Ec2ScaleSetValidationError(`Unsupported EC2 scale-set configuration field '${name}.${unknownKey}'`); + } +} + +function requireString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { + if (typeof value !== 'string' || value.length === 0 || value.length > maximumLength || !pattern.test(value)) { + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function requirePossiblyEmptyString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { + if (typeof value !== 'string' || value.length > maximumLength || !pattern.test(value)) { + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function optionalString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string | undefined { + if (value === undefined) return undefined; + return requireString(value, name, pattern, maximumLength); +} + +function optionalBoolean(value: unknown, name: string): boolean | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'boolean') { + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function requireStringArray( + value: unknown, + name: string, + pattern: RegExp, + maximumItemLength: number, + allowEmpty = false, +): string[] { + if (!Array.isArray(value) || (!allowEmpty && value.length === 0) || value.length > 100) { + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); + } + const parsed = value.map((item, index) => requireString(item, `${name}[${index}]`, pattern, maximumItemLength)); + if (new Set(parsed).size !== parsed.length) { + throw new Ec2ScaleSetValidationError(`EC2 scale-set configuration field '${name}' contains duplicate values`); + } + return parsed; +} + +function parseInstanceTypePriorities(value: unknown): Record | undefined { + if (value === undefined) return undefined; + if (!isRecord(value)) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'instanceTypePriorities'"); + } + + const result = Object.create(null) as Record; + for (const [instanceType, priority] of Object.entries(value)) { + requireString(instanceType, 'instanceTypePriorities key', /^[a-z0-9][a-z0-9.-]*$/, 64); + if (typeof priority !== 'number' || !Number.isSafeInteger(priority) || priority < 0 || priority > 1000) { + throw new Ec2ScaleSetValidationError( + `Invalid EC2 scale-set configuration priority for instance type '${instanceType}'`, + ); + } + result[instanceType] = priority; + } + return result; +} + +function requireSsmTagValue(value: unknown): string { + if (typeof value !== 'string' || value.length > 256) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); + } + for (const character of value) { + const codePoint = character.codePointAt(0)!; + if (codePoint < 32 || codePoint === 127) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); + } + } + return value; +} + +function parseEc2OverrideConfig(value: unknown): Ec2OverrideConfig | undefined { + if (value === undefined) return undefined; + if (!isRecord(value)) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ec2OverrideConfig'"); + } + + const supportedKeys = new Set([ + 'InstanceType', + 'MaxPrice', + 'SubnetId', + 'AvailabilityZone', + 'AvailabilityZoneId', + 'WeightedCapacity', + 'Priority', + 'ImageId', + ]); + if (Object.keys(value).some((key) => !supportedKeys.has(key))) { + throw new Ec2ScaleSetValidationError('EC2 scale-set configuration contains an unsupported launch override'); + } + + const weightedCapacity = value.WeightedCapacity; + const priority = value.Priority; + for (const [name, number] of [ + ['WeightedCapacity', weightedCapacity], + ['Priority', priority], + ] as const) { + if (number !== undefined && (typeof number !== 'number' || !Number.isFinite(number) || number < 0)) { + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); + } + } + + return { + InstanceType: optionalString( + value.InstanceType, + 'ec2OverrideConfig.InstanceType', + /^[a-z0-9][a-z0-9.-]*$/, + 64, + ) as Ec2OverrideConfig['InstanceType'], + MaxPrice: optionalString(value.MaxPrice, 'ec2OverrideConfig.MaxPrice', /^(?:0|[1-9]\d*)(?:\.\d{1,6})?$/, 32), + SubnetId: optionalString(value.SubnetId, 'ec2OverrideConfig.SubnetId', /^subnet-[0-9a-f]+$/, 32), + AvailabilityZone: optionalString( + value.AvailabilityZone, + 'ec2OverrideConfig.AvailabilityZone', + /^[a-z]{2}(?:-[a-z0-9]+)+-\d[a-z]$/, + 64, + ), + AvailabilityZoneId: optionalString( + value.AvailabilityZoneId, + 'ec2OverrideConfig.AvailabilityZoneId', + /^[a-z0-9-]+$/, + 64, + ), + WeightedCapacity: weightedCapacity as number | undefined, + Priority: priority as number | undefined, + ImageId: optionalString(value.ImageId, 'ec2OverrideConfig.ImageId', /^ami-[0-9a-f]+$/, 32), + }; +} + +function parseSsmTags(value: unknown): SsmTag[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value) || value.length > 45) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); + } + + const tags: SsmTag[] = []; + const keys = new Set(); + for (const item of value) { + if (!isRecord(item)) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); + } + const key = requireString(item.Key, 'ssmParameterTags.Key', /^[A-Za-z0-9_.:/=+@-]+$/, 128); + const tagValue = requireSsmTagValue(item.Value); + if (key.toLowerCase().startsWith('aws:') || keys.has(key)) { + throw new Ec2ScaleSetValidationError(`Invalid or duplicate SSM tag key '${key}'`); + } + keys.add(key); + tags.push({ Key: key, Value: tagValue }); + } + return tags; +} + +export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProviderConfig { + if (!isRecord(value)) { + throw new Ec2ScaleSetValidationError('EC2 scale-set provider configuration must be an object'); + } + rejectUnknownKeys( + value, + new Set([ + 'region', + 'environment', + 'runnerNamePrefix', + 'jitConfigParameterPath', + 'subnets', + 'launchTemplateName', + 'ec2instanceCriteria', + 'ec2OverrideConfig', + 'amiIdSsmParameterName', + 'tracingEnabled', + 'onDemandFailoverOnError', + 'useDedicatedHost', + 'ssmKmsKeyId', + 'ssmParameterTags', + ]), + 'configuration', + ); + if (!isRecord(value.ec2instanceCriteria)) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ec2instanceCriteria'"); + } + rejectUnknownKeys( + value.ec2instanceCriteria, + new Set([ + 'instanceTypes', + 'instanceTypePriorities', + 'targetCapacityType', + 'maxSpotPrice', + 'instanceAllocationStrategy', + ]), + 'ec2instanceCriteria', + ); + + const targetCapacityType = value.ec2instanceCriteria.targetCapacityType; + if (targetCapacityType !== 'on-demand' && targetCapacityType !== 'spot') { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'targetCapacityType'"); + } + const instanceAllocationStrategy = requireString( + value.ec2instanceCriteria.instanceAllocationStrategy, + 'instanceAllocationStrategy', + /^[a-z-]+$/, + 64, + ) as RunnerInputParameters['ec2instanceCriteria']['instanceAllocationStrategy']; + const allowedAllocationStrategies = + targetCapacityType === 'spot' ? SPOT_ALLOCATION_STRATEGIES : ON_DEMAND_ALLOCATION_STRATEGIES; + if (!allowedAllocationStrategies.has(instanceAllocationStrategy)) { + throw new Ec2ScaleSetValidationError( + `Invalid allocation strategy '${instanceAllocationStrategy}' for '${targetCapacityType}' capacity`, + ); + } + + const jitConfigParameterPath = requireString( + value.jitConfigParameterPath, + 'jitConfigParameterPath', + /^\/(?!.*\/\/)[A-Za-z0-9_.\-/]+$/, + 900, + ).replace(/\/$/, ''); + + return { + region: requireString(value.region, 'region', /^[a-z]{2}(?:-[a-z0-9]+)+-\d$/, 32), + environment: requireString(value.environment, 'environment', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128), + runnerNamePrefix: requirePossiblyEmptyString(value.runnerNamePrefix, 'runnerNamePrefix', /^[A-Za-z0-9._-]*$/, 45), + jitConfigParameterPath, + subnets: requireStringArray(value.subnets, 'subnets', /^subnet-[0-9a-f]+$/, 32), + launchTemplateName: requireString(value.launchTemplateName, 'launchTemplateName', /^[A-Za-z0-9()./_-]+$/, 128), + ec2instanceCriteria: { + instanceTypes: requireStringArray( + value.ec2instanceCriteria.instanceTypes, + 'instanceTypes', + /^[a-z0-9][a-z0-9.-]*$/, + 64, + ), + instanceTypePriorities: parseInstanceTypePriorities(value.ec2instanceCriteria.instanceTypePriorities), + targetCapacityType, + maxSpotPrice: optionalString( + value.ec2instanceCriteria.maxSpotPrice, + 'maxSpotPrice', + /^(?:0|[1-9]\d*)(?:\.\d{1,6})?$/, + 32, + ), + instanceAllocationStrategy, + }, + ec2OverrideConfig: parseEc2OverrideConfig(value.ec2OverrideConfig), + amiIdSsmParameterName: optionalString( + value.amiIdSsmParameterName, + 'amiIdSsmParameterName', + /^\/(?!.*\/\/)[A-Za-z0-9_.\-/]+$/, + 900, + ), + tracingEnabled: optionalBoolean(value.tracingEnabled, 'tracingEnabled'), + onDemandFailoverOnError: requireStringArray( + value.onDemandFailoverOnError ?? [], + 'onDemandFailoverOnError', + /^[A-Za-z0-9._-]+$/, + 128, + true, + ), + useDedicatedHost: optionalBoolean(value.useDedicatedHost, 'useDedicatedHost'), + ssmKmsKeyId: optionalString(value.ssmKmsKeyId, 'ssmKmsKeyId', /^[A-Za-z0-9_:/+=,.@-]+$/, 2048), + ssmParameterTags: parseSsmTags(value.ssmParameterTags), + }; +} + +export function validateFactoryInput(input: CreateEc2ScaleSetProviderInput): void { + requireString(input.runnerConfigName, 'runnerConfigName', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128); + if (!Number.isSafeInteger(input.scaleSetId) || input.scaleSetId <= 0) { + throw new Ec2ScaleSetValidationError('scaleSetId must be a positive safe integer'); + } + validateCanonicalGitHubScope(input.githubScope); +} + +function validateCanonicalGitHubScope(value: unknown): string { + if (typeof value !== 'string' || value.length === 0 || value.length > 2048) { + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + let url: URL; + try { + url = new URL(value); + } catch { + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) { + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + const parts = url.pathname + .replace(/^\/+|\/+$/g, '') + .split('/') + .filter(Boolean); + if (parts.length < 1 || parts.length > 2 || (parts[0].toLowerCase() === 'enterprises' && parts.length !== 2)) { + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + url.pathname = `/${parts.join('/')}`; + const canonical = url.toString().replace(/\/$/, ''); + if (canonical !== value) { + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + return value; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts new file mode 100644 index 0000000000..c4b0347376 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts @@ -0,0 +1,123 @@ +import { createHash } from 'node:crypto'; + +import { CreateFleetCommand, DescribeInstancesCommand } from '@aws-sdk/client-ec2'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { EC2_GITHUB_SCOPE_HASH_TAG, EC2_RUNNER_CONFIG_TAG, EC2_SCALE_SET_ID_TAG } from './inventory'; +import { createRequest, githubScopeHash, githubState, ownedInstance } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set inventory', () => { + it('lists only the exact runner-config and scale-set ownership boundary', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ + ownedInstance('i-owned', { runnerId: 101, runnerName: 'runner-i-owned' }), + ownedInstance('i-other', undefined, { runnerConfigName: 'other' }), + ownedInstance( + 'i-other-scope', + { runnerId: 102, runnerName: 'runner-i-other-scope' }, + { + githubScopeHash: createHash('sha256').update('https://github.com/another', 'utf8').digest('hex'), + }, + ), + ], + }, + ], + }); + + const result = await createTestProvider().reconcile(createRequest()); + + expect(result).toMatchObject({ status: 'converged', desiredRunners: 1, currentRunners: 1 }); + expect(ec2Mock).toHaveReceivedCommandWith(DescribeInstancesCommand, { + Filters: expect.arrayContaining([ + { Name: 'tag:ghr:Application', Values: ['github-action-runner'] }, + { Name: 'tag:ghr:created_by', Values: ['scale-set-service'] }, + { Name: 'tag:ghr:environment', Values: ['unit-test'] }, + { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: ['linux'] }, + { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: ['42'] }, + { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash] }, + ]), + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('counts a young handed-off instance as serving during its bounded boot window', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ownedInstance('i-booting', { runnerId: 101, runnerName: 'runner-i-booting' })], + }, + ], + }); + + const result = await createTestProvider({ now: () => new Date('2026-08-24T10:09:59Z').getTime() }).reconcile( + createRequest(), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('uses the orchestration request boot window instead of provider configuration', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ownedInstance('i-at-timeout', { runnerId: 101, runnerName: 'runner-i-at-timeout' })], + }, + ], + }); + + const result = await createTestProvider({ now: () => new Date('2026-08-24T10:05:00Z').getTime() }).reconcile( + createRequest({ bootTimeoutMinutes: 5 }), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('counts tagged capacity after the boot window without public runner inventory', async () => { + const instance = ownedInstance('i-old', { runnerId: 101, runnerName: 'runner-i-old' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const result = await createTestProvider({ now: () => new Date('2026-08-24T10:10:00Z').getTime() }).reconcile( + createRequest({ busyRunners: 1 }), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('counts an exact JobStarted identity as serving without waiting for public inventory', async () => { + const instance = ownedInstance('i-started', { runnerId: 101, runnerName: 'runner-i-started' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + + const result = await createTestProvider({ now: () => new Date('2026-08-24T12:00:00Z').getTime() }).reconcile( + createRequest({ + runnerStates: [ + githubState(101, 'runner-i-started', { status: 'unknown', busy: undefined, lifecycle: 'started' }), + ], + }), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts new file mode 100644 index 0000000000..91941b0139 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts @@ -0,0 +1,249 @@ +import { createHash } from 'node:crypto'; + +import { DescribeInstancesCommand, type EC2Client, type Instance, type Tag } from '@aws-sdk/client-ec2'; + +import type { ScaleSetReconcileRequest, ScaleSetRunnerState } from '../../../../scale-set'; +import type { CreateEc2ScaleSetProviderInput } from './configuration'; +import { retainUnknown, type MutableReconcileState } from './reconcile'; + +export const EC2_RUNNER_CONFIG_TAG = 'ghr:runner_config'; +export const EC2_SCALE_SET_ID_TAG = 'ghr:scale_set_id'; +export const EC2_GITHUB_SCOPE_HASH_TAG = 'ghr:github_scope_hash'; +export const EC2_SCALE_SET_STATE_TAG = 'ghr:scale_set_state'; +export const EC2_RUNNER_NAME_TAG = 'ghr:runner_name'; +export const EC2_GITHUB_RUNNER_ID_TAG = 'ghr:github_runner_id'; + +const APPLICATION_TAG = 'ghr:Application'; +const APPLICATION_VALUE = 'github-action-runner'; +const CREATED_BY_TAG = 'ghr:created_by'; +export const SCALE_SET_RUNNER_SOURCE = 'scale-set-service'; +const ENVIRONMENT_TAG = 'ghr:environment'; +export const GITHUB_RUNNER_NAME_MAX_LENGTH = 64; + +type Ec2ScaleSetState = 'provisioning' | 'publishing' | 'config-published' | 'retiring'; + +export interface OwnedEc2Runner { + instanceId: string; + launchTime?: Date; + githubRunnerId?: number; + runnerName?: string; + scaleSetState?: Ec2ScaleSetState; +} + +export function githubScopeHash(githubScope: string): string { + return createHash('sha256').update(githubScope, 'utf8').digest('hex'); +} + +export function runnerIdentityFromGitHubScope(githubScope: string): { + runnerOwner: string; + runnerType: 'Org' | 'Repo'; +} { + const pathParts = new URL(githubScope).pathname.replace(/^\/+|\/+$/g, '').split('/'); + if (pathParts.length === 2 && pathParts[0].toLowerCase() !== 'enterprises') { + return { runnerOwner: pathParts.join('/'), runnerType: 'Repo' }; + } + + // The legacy EC2 tags do not have an enterprise discriminator. They remain + // informational here; exact ownership is fenced by runner config, scale-set + // ID, and the canonical GitHub-scope hash. + return { + runnerOwner: pathParts[0].toLowerCase() === 'enterprises' ? pathParts[1] : pathParts[0], + runnerType: 'Org', + }; +} + +export function ownershipTags(input: CreateEc2ScaleSetProviderInput): Tag[] { + return [ + { Key: ENVIRONMENT_TAG, Value: input.configuration.environment }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, + ]; +} + +export async function listOwnedRunners( + input: CreateEc2ScaleSetProviderInput, + ec2Client: EC2Client, + signal: AbortSignal, +): Promise { + const runners: OwnedEc2Runner[] = []; + let nextToken: string | undefined; + do { + const response = await ec2Client.send( + new DescribeInstancesCommand({ + Filters: [ + { Name: 'instance-state-name', Values: ['pending', 'running'] }, + { Name: `tag:${APPLICATION_TAG}`, Values: [APPLICATION_VALUE] }, + { Name: `tag:${CREATED_BY_TAG}`, Values: [SCALE_SET_RUNNER_SOURCE] }, + { Name: `tag:${ENVIRONMENT_TAG}`, Values: [input.configuration.environment] }, + { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: [input.runnerConfigName] }, + { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: [String(input.scaleSetId)] }, + { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash(input.githubScope)] }, + ], + NextToken: nextToken, + }), + { abortSignal: signal }, + ); + nextToken = response.NextToken; + + for (const instance of response.Reservations?.flatMap((reservation) => reservation.Instances ?? []) ?? []) { + const runner = parseOwnedRunner(instance, input); + if (runner) runners.push(runner); + } + } while (nextToken); + + return runners; +} + +function parseOwnedRunner(instance: Instance, input: CreateEc2ScaleSetProviderInput): OwnedEc2Runner | undefined { + if (!instance.InstanceId) return undefined; + const tags = new Map((instance.Tags ?? []).flatMap((tag) => (tag.Key ? [[tag.Key, tag.Value]] : []))); + + if ( + tags.get(APPLICATION_TAG) !== APPLICATION_VALUE || + tags.get(CREATED_BY_TAG) !== SCALE_SET_RUNNER_SOURCE || + tags.get(ENVIRONMENT_TAG) !== input.configuration.environment || + tags.get(EC2_RUNNER_CONFIG_TAG) !== input.runnerConfigName || + tags.get(EC2_SCALE_SET_ID_TAG) !== String(input.scaleSetId) || + tags.get(EC2_GITHUB_SCOPE_HASH_TAG) !== githubScopeHash(input.githubScope) + ) { + return undefined; + } + + const taggedRunnerId = tags.get(EC2_GITHUB_RUNNER_ID_TAG); + const githubRunnerId = taggedRunnerId === undefined ? undefined : Number(taggedRunnerId); + const rawScaleSetState = tags.get(EC2_SCALE_SET_STATE_TAG); + const scaleSetState = ['provisioning', 'publishing', 'config-published', 'retiring'].includes(rawScaleSetState ?? '') + ? (rawScaleSetState as Ec2ScaleSetState) + : undefined; + + return { + instanceId: instance.InstanceId, + launchTime: instance.LaunchTime, + githubRunnerId: Number.isSafeInteger(githubRunnerId) && githubRunnerId! > 0 ? githubRunnerId : undefined, + runnerName: tags.get(EC2_RUNNER_NAME_TAG), + scaleSetState, + }; +} + +function validRunnerState(value: ScaleSetRunnerState): boolean { + return ( + Number.isSafeInteger(value.runnerId) && + value.runnerId > 0 && + Number.isSafeInteger(value.scaleSetId) && + value.scaleSetId > 0 && + typeof value.runnerName === 'string' && + value.runnerName.length > 0 && + value.runnerName.length <= GITHUB_RUNNER_NAME_MAX_LENGTH && + ['online', 'offline', 'unknown'].includes(value.status) && + (typeof value.busy === 'boolean' || value.busy === undefined) && + ['started', 'completed', 'unknown'].includes(value.lifecycle) + ); +} + +export function indexRunnerStates( + runnerStates: readonly ScaleSetRunnerState[], + scaleSetId: number, +): { byName: Map; ambiguousNames: Set; ambiguousIds: Set } { + const byName = new Map(); + const byId = new Map(); + const ambiguousNames = new Set(); + const ambiguousIds = new Set(); + + for (const state of runnerStates) { + if (!validRunnerState(state) || state.scaleSetId !== scaleSetId) continue; + if (byName.has(state.runnerName)) ambiguousNames.add(state.runnerName); + const existingName = byId.get(state.runnerId); + if (existingName !== undefined && existingName !== state.runnerName) { + ambiguousIds.add(state.runnerId); + ambiguousNames.add(existingName); + ambiguousNames.add(state.runnerName); + } + byName.set(state.runnerName, state); + byId.set(state.runnerId, state.runnerName); + } + return { byName, ambiguousNames, ambiguousIds }; +} + +export function matchingRunnerState( + runner: OwnedEc2Runner, + index: ReturnType, + scaleSetId: number, +): ScaleSetRunnerState | undefined { + if (!runner.runnerName || !runner.githubRunnerId) return undefined; + if (index.ambiguousNames.has(runner.runnerName) || index.ambiguousIds.has(runner.githubRunnerId)) return undefined; + const state = index.byName.get(runner.runnerName); + if ( + !state || + state.runnerId !== runner.githubRunnerId || + state.runnerName !== runner.runnerName || + state.scaleSetId !== scaleSetId + ) { + return undefined; + } + return state; +} + +function isWithinBootTimeout(runner: OwnedEc2Runner, bootTimeoutMinutes: number, now: number): boolean { + const launchTime = runner.launchTime?.getTime(); + if (launchTime === undefined || !Number.isFinite(launchTime)) return false; + const ageMilliseconds = now - launchTime; + return ageMilliseconds >= 0 && ageMilliseconds < bootTimeoutMinutes * 60_000; +} + +function isConfirmedServingState(state: ScaleSetRunnerState): boolean { + return state.lifecycle === 'started' || state.status === 'online'; +} + +export function servingCapacity( + input: CreateEc2ScaleSetProviderInput, + runners: readonly OwnedEc2Runner[], + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + now: number, +): OwnedEc2Runner[] { + const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); + const serving: OwnedEc2Runner[] = []; + + for (const runner of runners) { + if (runner.scaleSetState !== 'config-published') { + // An interrupted publication may already have been consumed. Preserve it, + // but do not let it suppress replacement capacity indefinitely. + retainUnknown(state, runner.instanceId); + continue; + } + + const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); + if (githubState !== undefined && isConfirmedServingState(githubState)) { + serving.push(runner); + continue; + } + if (isWithinBootTimeout(runner, request.bootTimeoutMinutes, now)) { + serving.push(runner); + continue; + } + + // A config-published EC2 instance is provider-owned capacity. The exact + // Actions-service identity is used only when removing it; no public + // GitHub runner inventory is needed to count capacity. + if (runner.githubRunnerId !== undefined && runner.runnerName !== undefined) { + serving.push(runner); + continue; + } + + retainUnknown(state, runner.instanceId); + // Unknown lifecycle state is not allowed to suppress replacement capacity. + // Aggregate busy state still protects scale-down, while tagged EC2 + // inventory remains the source of current provider-owned capacity. + } + + return serving; +} + +export function isSafeScaleDownState(state: ScaleSetRunnerState): boolean { + return state.busy === false || (state.lifecycle === 'completed' && state.busy !== true); +} + +export function isBusyState(state: ScaleSetRunnerState): boolean { + return state.lifecycle === 'started' || state.busy === true; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts new file mode 100644 index 0000000000..704283ac64 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts @@ -0,0 +1,132 @@ +import { CreateFleetCommand, DescribeInstancesCommand, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { PutParameterCommand } from '@aws-sdk/client-ssm'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { config, createRequest, githubState, ownedInstance } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks, ssmMock } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set provider orchestration', () => { + it('rejects non-canonical GitHub ownership scopes before creating clients', () => { + expect(() => createTestProvider({ githubScope: 'https://GITHUB.com/example/' })).toThrow( + 'githubScope must be a canonical HTTPS GitHub configuration URL', + ); + }); + + it('counts an old tagged handoff as provider capacity without public inventory', async () => { + const old = ownedInstance('i-old-offline', { runnerId: 100, runnerName: 'runner-i-old-offline' }); + const replacementId = 'i-1234567890abcdef0'; + const replacement = ownedInstance( + replacementId, + { runnerId: 101, runnerName: `runner-${replacementId}` }, + { + launchTime: new Date('2026-08-24T10:10:30Z'), + }, + ); + ec2Mock + .on(DescribeInstancesCommand) + .resolvesOnce({ Reservations: [{ Instances: [old] }] }) + .resolves({ Reservations: [{ Instances: [old, replacement] }] }); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [replacementId] }] }); + const computeProvider = createTestProvider({ now: () => new Date('2026-08-24T10:11:00Z').getTime() }); + const completeInventory = createRequest({ + runnerStates: [ + githubState(100, 'runner-i-old-offline', { + status: 'offline', + busy: false, + lifecycle: 'completed', + }), + ], + }); + + const result = await computeProvider.reconcile(completeInventory); + const nextResult = await computeProvider.reconcile(completeInventory); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(nextResult).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it.each(['provisioning', 'publishing'])( + 'retains interrupted %s capacity but provisions a replacement', + async (scaleSetState) => { + const stuck = ownedInstance( + 'i-stuck', + scaleSetState === 'publishing' ? { runnerId: 100, runnerName: 'runner-i-stuck' } : undefined, + { scaleSetState }, + ); + const replacement = 'i-1234567890abcdef0'; + ec2Mock + .on(DescribeInstancesCommand) + .resolvesOnce({ Reservations: [{ Instances: [stuck] }] }) + .resolves({ + Reservations: [ + { + Instances: [stuck, ownedInstance(replacement, { runnerId: 101, runnerName: `runner-${replacement}` })], + }, + ], + }); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [replacement] }] }); + const computeProvider = createTestProvider(); + + const result = await computeProvider.reconcile(createRequest()); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 2, + actions: { launched: 1, terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-stuck'] }); + expect(ssmMock).toHaveReceivedCommandWith(PutParameterCommand, { + Name: `${config.jitConfigParameterPath}/${replacement}`, + }); + + const nextResult = await computeProvider.reconcile(createRequest()); + expect(nextResult).toMatchObject({ + status: 'retained', + currentRunners: 2, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(ec2Mock).toHaveReceivedCommandTimes(CreateFleetCommand, 1); + }, + ); + + it('caps retained-capacity replacement surge when every replacement remains ambiguous', async () => { + const ambiguous = [ + ownedInstance('i-stuck-1', undefined, { scaleSetState: 'provisioning' }), + ownedInstance('i-stuck-2', { runnerId: 102, runnerName: 'runner-i-stuck-2' }, { scaleSetState: 'publishing' }), + ]; + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: ambiguous }] }); + + const result = await createTestProvider().reconcile(createRequest()); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 2, + actions: { launched: 0, terminated: 0, retainedUnknown: 2 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('propagates cancellation instead of converting shutdown into a retry result', async () => { + const abort = new AbortController(); + abort.abort(new Error('service stopping')); + + await expect(createTestProvider().reconcile(createRequest({ signal: abort.signal }))).rejects.toThrow( + 'service stopping', + ); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts new file mode 100644 index 0000000000..3c9df100ed --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts @@ -0,0 +1,121 @@ +import { EC2Client } from '@aws-sdk/client-ec2'; +import { SSMClient } from '@aws-sdk/client-ssm'; + +import type { ScaleSetComputeProvider, ScaleSetReconcileResult } from '../../../../scale-set'; +import { createEc2RunnerClient } from '../runners'; +import { + parseEc2ScaleSetProviderConfig, + validateFactoryInput, + type CreateEc2ScaleSetProviderInput, + type Ec2ScaleSetProviderConfig, +} from './configuration'; +import { listOwnedRunners, servingCapacity, type OwnedEc2Runner } from './inventory'; +import { + emptyState, + finish, + safeError, + throwIfAborted, + validateBootTimeout, + validateBusyRunners, + validateDesiredRunners, +} from './reconcile'; +import { scaleDown } from './scale-down'; +import { scaleUp } from './scale-up'; + +export type { CreateEc2ScaleSetProviderInput, Ec2ScaleSetProviderConfig } from './configuration'; +export { parseEc2ScaleSetProviderConfig } from './configuration'; +export { + EC2_GITHUB_RUNNER_ID_TAG, + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_RUNNER_NAME_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, +} from './inventory'; + +const RETAINED_CAPACITY_REPLACEMENT_SURGE = 1; + +export interface Ec2ScaleSetProviderDependencies { + ec2Client?: EC2Client; + ssmClient?: SSMClient; + now?: () => number; +} + +function createClients(config: Ec2ScaleSetProviderConfig, dependencies: Ec2ScaleSetProviderDependencies) { + return { + ec2Client: dependencies.ec2Client ?? new EC2Client({ region: config.region }), + ssmClient: + dependencies.ssmClient ?? + new SSMClient({ + region: config.region, + maxAttempts: 10, + retryMode: 'adaptive', + }), + }; +} + +export function createEc2ScaleSetProvider( + input: CreateEc2ScaleSetProviderInput, + dependencies: Ec2ScaleSetProviderDependencies = {}, +): ScaleSetComputeProvider { + const normalizedInput = { + ...input, + configuration: parseEc2ScaleSetProviderConfig(input.configuration), + }; + validateFactoryInput(normalizedInput); + const clients = createClients(normalizedInput.configuration, dependencies); + const runnerClient = createEc2RunnerClient(clients.ec2Client); + const now = dependencies.now ?? Date.now; + + return { + async reconcile(request): Promise { + request.signal.throwIfAborted(); + const validationError = + validateDesiredRunners(request.desiredRunners) ?? + validateBusyRunners(request.busyRunners) ?? + validateBootTimeout(request.bootTimeoutMinutes); + if (validationError) { + const state = emptyState(0); + state.errors.push(validationError); + return finish(state, request.desiredRunners); + } + + const runnerOperations = runnerClient.forRequest({ signal: request.signal }); + let ownedRunners: OwnedEc2Runner[]; + try { + ownedRunners = await listOwnedRunners(normalizedInput, clients.ec2Client, request.signal); + } catch (error) { + throwIfAborted(request.signal, error); + const state = emptyState(0); + state.errors.push(safeError('list', error)); + return finish(state, request.desiredRunners); + } + + const state = emptyState(ownedRunners.length); + const servingRunners = servingCapacity(normalizedInput, ownedRunners, request, state, now()); + + if (servingRunners.length < request.desiredRunners) { + const capacityDeficit = request.desiredRunners - servingRunners.length; + const availableReplacementSlots = Math.max( + 0, + request.desiredRunners + RETAINED_CAPACITY_REPLACEMENT_SURGE - ownedRunners.length, + ); + const launchCount = Math.min(capacityDeficit, availableReplacementSlots); + if (launchCount > 0) { + await scaleUp(normalizedInput, launchCount, request, state, runnerOperations, clients.ssmClient); + } + } else if (servingRunners.length > request.desiredRunners) { + await scaleDown( + normalizedInput, + servingRunners, + servingRunners.length - request.desiredRunners, + request, + state, + runnerOperations, + ); + } + + return finish(state, request.desiredRunners); + }, + }; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts new file mode 100644 index 0000000000..a462b935b8 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts @@ -0,0 +1,32 @@ +import { DescribeInstancesCommand } from '@aws-sdk/client-ec2'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { createRequest } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set reconciliation validation', () => { + it('rejects an invalid desired count without touching AWS', async () => { + const result = await createTestProvider().reconcile(createRequest({ desiredRunners: -1 })); + + expect(result).toMatchObject({ + status: 'error', + desiredRunners: -1, + currentRunners: 0, + }); + expect(result.errors).toEqual([{ operation: 'validate', code: 'INVALID_DESIRED_RUNNER_COUNT' }]); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); + + it.each([0, 121, 1.5])('rejects invalid orchestration boot timeout %s without touching AWS', async (value) => { + const result = await createTestProvider().reconcile(createRequest({ bootTimeoutMinutes: value })); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 0, + }); + expect(result.errors).toEqual([{ operation: 'validate', code: 'INVALID_BOOT_TIMEOUT' }]); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts new file mode 100644 index 0000000000..a49e8c0293 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts @@ -0,0 +1,131 @@ +import type { + ScaleSetReconcileActions, + ScaleSetReconcileError, + ScaleSetReconcileOperation, + ScaleSetReconcileResult, +} from '../../../../scale-set'; + +const MAX_BOOT_TIMEOUT_MINUTES = 120; + +export interface MutableReconcileState { + currentRunners: number; + retainedUnknownResourceIds: Set; + actions: ScaleSetReconcileActions; + errors: ScaleSetReconcileError[]; +} + +export class Ec2ScaleSetValidationError extends Error { + constructor(message: string) { + super(message); + this.name = 'Ec2ScaleSetValidationError'; + } +} + +export function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function safeError( + operation: ScaleSetReconcileOperation, + error: unknown, + details: Pick = {}, +): ScaleSetReconcileError { + return { + operation, + code: safeErrorCode(error), + ...details, + }; +} + +function safeErrorCode(error: unknown): string { + if (error instanceof Ec2ScaleSetValidationError) return 'INVALID_CONFIGURATION'; + if (!isRecord(error)) return 'UNEXPECTED_ERROR'; + for (const candidate of [error.name, error.code]) { + if (typeof candidate === 'string' && /^[A-Za-z][A-Za-z0-9._-]{0,127}$/.test(candidate)) { + return candidate; + } + } + return 'UNEXPECTED_ERROR'; +} + +export function throwIfAborted(signal: AbortSignal, error?: unknown): void { + if (signal.aborted || (isRecord(error) && error.name === 'AbortError')) { + signal.throwIfAborted(); + throw error; + } +} + +function resultStatus(errors: readonly ScaleSetReconcileError[], current: number, desired: number) { + if (errors.length > 0 || current < desired) return 'error' as const; + if (current > desired) return 'retained' as const; + return 'converged' as const; +} + +export function finish(state: MutableReconcileState, desiredRunners: number): ScaleSetReconcileResult { + if (desiredRunners >= 0 && state.currentRunners < desiredRunners && state.errors.length === 0) { + state.errors.push({ + operation: 'reconcile', + code: 'CAPACITY_NOT_PROVISIONED', + }); + } + return { + status: resultStatus(state.errors, state.currentRunners, desiredRunners), + desiredRunners, + currentRunners: state.currentRunners, + actions: state.actions, + errors: state.errors, + }; +} + +export function emptyState(currentRunners: number): MutableReconcileState { + return { + currentRunners, + retainedUnknownResourceIds: new Set(), + actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + }; +} + +export function retainUnknown(state: MutableReconcileState, resourceId?: string): void { + if (resourceId === undefined) { + state.actions.retainedUnknown++; + return; + } + if (state.retainedUnknownResourceIds.has(resourceId)) return; + state.retainedUnknownResourceIds.add(resourceId); + state.actions.retainedUnknown++; +} + +export function validateDesiredRunners(desiredRunners: number): ScaleSetReconcileError | undefined { + if (!Number.isSafeInteger(desiredRunners) || desiredRunners < 0 || desiredRunners > 10000) { + return { + operation: 'validate', + code: 'INVALID_DESIRED_RUNNER_COUNT', + }; + } + return undefined; +} + +export function validateBootTimeout(bootTimeoutMinutes: number): ScaleSetReconcileError | undefined { + if ( + !Number.isSafeInteger(bootTimeoutMinutes) || + bootTimeoutMinutes < 1 || + bootTimeoutMinutes > MAX_BOOT_TIMEOUT_MINUTES + ) { + return { + operation: 'validate', + code: 'INVALID_BOOT_TIMEOUT', + }; + } + return undefined; +} + +export function validateBusyRunners(busyRunners: number): ScaleSetReconcileError | undefined { + if (!Number.isSafeInteger(busyRunners) || busyRunners < 0 || busyRunners > 2_147_483_647) { + return { + operation: 'validate', + code: 'INVALID_BUSY_RUNNER_COUNT', + }; + } + return undefined; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts new file mode 100644 index 0000000000..40b64e1c2b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts @@ -0,0 +1,195 @@ +import { DescribeInstancesCommand, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createRequest, githubState, ownedInstance, signal } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set scale down', () => { + it('terminates only exact known-idle or completed runners and retains busy or unknown runners', async () => { + const completed = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); + const busy = ownedInstance('i-busy', { runnerId: 102, runnerName: 'runner-busy' }); + const unknown = ownedInstance('i-unknown', { runnerId: 103, runnerName: 'runner-unknown' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [completed, busy, unknown] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 2, + runnerStates: [ + githubState(101, 'runner-completed', { status: 'offline', busy: undefined, lifecycle: 'completed' }), + githubState(102, 'runner-busy', { busy: true, lifecycle: 'started' }), + ], + removeRunner, + }), + ); + + expect(result).toEqual({ + status: 'converged', + desiredRunners: 2, + currentRunners: 2, + actions: { launched: 0, terminated: 1, retainedBusy: 1, retainedUnknown: 0 }, + errors: [], + }); + expect(removeRunner).toHaveBeenCalledTimes(1); + expect(removeRunner).toHaveBeenCalledWith({ + runnerId: 101, + runnerName: 'runner-completed', + scaleSetId: 42, + signal, + }); + expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-completed'] }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-busy'] }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-unknown'] }); + }); + + it('uses aggregate idle state to remove a tagged runner after a restart', async () => { + const instance = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + const result = await createTestProvider().reconcile( + createRequest({ desiredRunners: 0, busyRunners: 0, runnerStates: [], removeRunner }), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 0, + actions: { terminated: 1 }, + }); + expect(removeRunner).toHaveBeenCalledTimes(1); + }); + + it('never lets a completed lifecycle marker override a current busy signal', async () => { + const instance = ownedInstance('i-completed-busy', { runnerId: 101, runnerName: 'runner-completed-busy' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + busyRunners: 1, + runnerStates: [ + githubState(101, 'runner-completed-busy', { + lifecycle: 'completed', + status: 'online', + busy: true, + }), + ], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + actions: { terminated: 0, retainedBusy: 1 }, + errors: [], + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('retains a runner without an error when the exact removal check observes that it became busy', async () => { + const instance = ownedInstance('i-raced-busy', { runnerId: 101, runnerName: 'runner-raced-busy' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'retained_busy' }); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + busyRunners: 0, + runnerStates: [githubState(101, 'runner-raced-busy')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + actions: { terminated: 0, retainedBusy: 1, retainedUnknown: 0 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('retains a runner and requests inventory when exact removal observes identity drift', async () => { + const instance = ownedInstance('i-raced-unknown', { runnerId: 101, runnerName: 'runner-raced-unknown' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'retained_unknown' }); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + busyRunners: 0, + runnerStates: [githubState(101, 'runner-raced-unknown')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + actions: { terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not trust a mutable EC2 GitHub-runner-id tag when controller identity disagrees', async () => { + const instance = ownedInstance('i-mismatch', { runnerId: 999, runnerName: 'runner-exact' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + busyRunners: 0, + runnerStates: [githubState(101, 'runner-exact')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + actions: { terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not terminate compute when exact GitHub removal fails', async () => { + const instance = ownedInstance('i-idle', { runnerId: 101, runnerName: 'runner-idle' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi + .fn() + .mockRejectedValue(Object.assign(new Error('must not leak'), { name: 'ServiceUnavailable' })); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + busyRunners: 0, + runnerStates: [githubState(101, 'runner-idle')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 1, + actions: { terminated: 0, retainedUnknown: 1 }, + }); + expect(result.errors).toEqual([ + { + operation: 'remove_runner', + code: 'ServiceUnavailable', + runnerName: 'runner-idle', + resourceId: 'i-idle', + }, + ]); + expect(JSON.stringify(result)).not.toContain('must not leak'); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts new file mode 100644 index 0000000000..78fddce9cd --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts @@ -0,0 +1,118 @@ +import type { ScaleSetReconcileRequest, ScaleSetRunnerState } from '../../../../scale-set'; +import type { Ec2RunnerResourceOperations } from '../runners'; +import type { CreateEc2ScaleSetProviderInput } from './configuration'; +import { + indexRunnerStates, + isBusyState, + isSafeScaleDownState, + matchingRunnerState, + type OwnedEc2Runner, +} from './inventory'; +import { retainUnknown, safeError, throwIfAborted, type MutableReconcileState } from './reconcile'; + +async function terminateKnownIdleRunner( + runner: OwnedEc2Runner, + githubState: ScaleSetRunnerState, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runnerOperations: Ec2RunnerResourceOperations, +): Promise { + let removalResult; + try { + removalResult = await request.removeRunner({ + runnerId: githubState.runnerId, + runnerName: githubState.runnerName, + scaleSetId: githubState.scaleSetId, + signal: request.signal, + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push( + safeError('remove_runner', error, { runnerName: githubState.runnerName, resourceId: runner.instanceId }), + ); + retainUnknown(state, runner.instanceId); + return false; + } + + if (removalResult.status === 'retained_busy') { + state.actions.retainedBusy++; + return false; + } + if (removalResult.status !== 'removed') { + retainUnknown(state, runner.instanceId); + return false; + } + + try { + await runnerOperations.terminate(runner.instanceId); + state.currentRunners--; + state.actions.terminated++; + return true; + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push( + safeError('terminate', error, { runnerName: githubState.runnerName, resourceId: runner.instanceId }), + ); + retainUnknown(state, runner.instanceId); + return false; + } +} + +export async function scaleDown( + input: CreateEc2ScaleSetProviderInput, + runners: readonly OwnedEc2Runner[], + count: number, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runnerOperations: Ec2RunnerResourceOperations, +): Promise { + const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); + const candidates: { runner: OwnedEc2Runner; githubState: ScaleSetRunnerState }[] = []; + + for (const runner of runners) { + const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); + const hasContradictoryState = runner.runnerName !== undefined && runnerStateIndex.byName.has(runner.runnerName); + if (!githubState) { + if ( + !hasContradictoryState && + request.busyRunners === 0 && + runner.githubRunnerId !== undefined && + runner.runnerName !== undefined + ) { + candidates.push({ + runner, + githubState: { + runnerId: runner.githubRunnerId, + runnerName: runner.runnerName, + scaleSetId: input.scaleSetId, + status: 'unknown', + busy: false, + lifecycle: 'unknown', + }, + }); + } else { + retainUnknown(state, runner.instanceId); + } + } else if (isBusyState(githubState)) { + state.actions.retainedBusy++; + } else if (isSafeScaleDownState(githubState)) { + candidates.push({ runner, githubState }); + } else { + retainUnknown(state, runner.instanceId); + } + } + + candidates.sort((left, right) => { + const launchOrder = (right.runner.launchTime?.getTime() ?? 0) - (left.runner.launchTime?.getTime() ?? 0); + return launchOrder || left.runner.instanceId.localeCompare(right.runner.instanceId); + }); + + let remaining = count; + for (const candidate of candidates) { + if (remaining === 0) break; + request.signal.throwIfAborted(); + if (await terminateKnownIdleRunner(candidate.runner, candidate.githubState, request, state, runnerOperations)) { + remaining--; + } + } +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts new file mode 100644 index 0000000000..d99d19ebbc --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts @@ -0,0 +1,200 @@ +import { + CreateFleetCommand, + CreateTagsCommand, + DescribeInstancesCommand, + TerminateInstancesCommand, +} from '@aws-sdk/client-ec2'; +import { DeleteParameterCommand, GetParameterCommand, PutParameterCommand } from '@aws-sdk/client-ssm'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, +} from './inventory'; +import { config, createRequest, githubScopeHash, jitResult, signal } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks, ssmMock } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set scale up', () => { + it('launches owned compute, verifies JIT identity, and publishes only a SecureString', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + const generateJitConfiguration = vi.fn().mockResolvedValue(jitResult(instanceId)); + + const result = await createTestProvider().reconcile(createRequest({ generateJitConfiguration })); + + expect(result).toEqual({ + status: 'converged', + desiredRunners: 1, + currentRunners: 1, + actions: { launched: 1, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + }); + expect(generateJitConfiguration).toHaveBeenCalledWith({ runnerName: `runner-${instanceId}`, signal }); + expect(ec2Mock).toHaveReceivedCommandWith(CreateFleetCommand, { + TagSpecifications: expect.arrayContaining([ + expect.objectContaining({ + ResourceType: 'instance', + Tags: expect.arrayContaining([ + { Key: 'ghr:created_by', Value: 'scale-set-service' }, + { Key: 'ghr:Owner', Value: 'example' }, + { Key: 'ghr:Type', Value: 'Org' }, + ]), + }), + ]), + }); + expect(ec2Mock).toHaveReceivedCommandWith(CreateTagsCommand, { + Resources: [instanceId], + Tags: expect.arrayContaining([ + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'publishing' }, + ]), + }); + expect(ssmMock).toHaveReceivedCommandWith(PutParameterCommand, { + Name: `${config.jitConfigParameterPath}/${instanceId}`, + Value: 'sensitive-encoded-jit-configuration', + Type: 'SecureString', + Overwrite: false, + Tags: expect.arrayContaining([ + { Key: 'InstanceId', Value: instanceId }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, + ]), + }); + }); + + it('resolves an AMI parameter through the provider-owned SSM client', async () => { + const instanceId = 'i-1234567890abcdef0'; + const amiIdSsmParameterName = '/github-action-runners/unit-test/ami'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(GetParameterCommand).resolves({ Parameter: { Value: 'ami-0123456789abcdef0' } }); + + const result = await createTestProvider({ + configuration: { ...config, amiIdSsmParameterName }, + }).reconcile(createRequest()); + + expect(result.actions.launched).toBe(1); + expect(ssmMock).toHaveReceivedCommandWith(GetParameterCommand, { + Name: amiIdSsmParameterName, + WithDecryption: true, + }); + }); + + it('does not remove an unrelated GitHub runner when JIT identity validation fails', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile( + createRequest({ + generateJitConfiguration: vi.fn().mockResolvedValue({ + ...jitResult(instanceId), + runnerName: 'runner-owned-by-another-config', + }), + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 0, + actions: { launched: 0, terminated: 1 }, + }); + expect(result.errors).toEqual([ + { + operation: 'generate_jit_configuration', + code: 'INVALID_CONFIGURATION', + runnerName: `runner-${instanceId}`, + resourceId: instanceId, + }, + ]); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ssmMock).not.toHaveReceivedCommand(PutParameterCommand); + expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: [instanceId] }); + }); + + it('retains compute when failed JIT publication cannot be safely cancelled', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(PutParameterCommand).rejects(Object.assign(new Error('redacted secret'), { name: 'TimeoutError' })); + ssmMock.on(DeleteParameterCommand).rejects(Object.assign(new Error('missing'), { name: 'ParameterNotFound' })); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile(createRequest({ removeRunner })); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 1, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + }); + expect(result.errors).toEqual([ + { + operation: 'publish_jit_configuration', + code: 'TimeoutError', + runnerName: `runner-${instanceId}`, + resourceId: instanceId, + }, + ]); + expect(JSON.stringify(result)).not.toContain('redacted secret'); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not treat a successful DeleteParameter as proof that bootstrap did not read JIT first', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(PutParameterCommand).rejects(Object.assign(new Error('throttled'), { name: 'ThrottlingException' })); + ssmMock.on(DeleteParameterCommand).resolves({}); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + + const result = await createTestProvider().reconcile(createRequest({ removeRunner })); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 1, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + }); + expect(result.errors).toEqual([ + { + operation: 'publish_jit_configuration', + code: 'ThrottlingException', + runnerName: `runner-${instanceId}`, + resourceId: instanceId, + }, + ]); + expect(ssmMock).toHaveReceivedCommandWith(DeleteParameterCommand, { + Name: `${config.jitConfigParameterPath}/${instanceId}`, + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it.each(['ThrottlingException', 'InvalidParameterValue'])( + 'collapses EC2 launch failure %s into one scale-set error', + async (errorCode) => { + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Errors: [{ ErrorCode: errorCode }] }); + + const result = await createTestProvider().reconcile(createRequest()); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 0, + actions: { launched: 0, terminated: 0 }, + }); + expect(result.errors).toEqual([{ operation: 'launch', code: 'EC2_LAUNCH_FAILED' }]); + }, + ); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts new file mode 100644 index 0000000000..a0fc3b984b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts @@ -0,0 +1,250 @@ +import { DeleteParameterCommand, PutParameterCommand, type SSMClient, type Tag as SsmTag } from '@aws-sdk/client-ssm'; + +import type { GenerateScaleSetJitConfigurationResult, ScaleSetReconcileRequest } from '../../../../scale-set'; +import type { Ec2RunnerResourceOperations } from '../runners'; +import type { CreateEc2ScaleSetProviderInput, Ec2ScaleSetProviderConfig } from './configuration'; +import { + EC2_GITHUB_RUNNER_ID_TAG, + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_RUNNER_NAME_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, + GITHUB_RUNNER_NAME_MAX_LENGTH, + githubScopeHash, + ownershipTags, + runnerIdentityFromGitHubScope, + SCALE_SET_RUNNER_SOURCE, +} from './inventory'; +import { + Ec2ScaleSetValidationError, + retainUnknown, + safeError, + throwIfAborted, + type MutableReconcileState, +} from './reconcile'; + +const SSM_STANDARD_TIER_THRESHOLD = 4000; +const SSM_ADVANCED_TIER_MAX_BYTES = 8192; + +function jitParameterName(config: Ec2ScaleSetProviderConfig, instanceId: string): string { + return `${config.jitConfigParameterPath}/${instanceId}`; +} + +function jitParameterTags(input: CreateEc2ScaleSetProviderInput, instanceId: string): SsmTag[] { + const reserved = new Set(['InstanceId', EC2_RUNNER_CONFIG_TAG, EC2_SCALE_SET_ID_TAG, EC2_GITHUB_SCOPE_HASH_TAG]); + return [ + ...(input.configuration.ssmParameterTags ?? []).filter((tag) => tag.Key && !reserved.has(tag.Key)), + { Key: 'InstanceId', Value: instanceId }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, + ]; +} + +async function publishJitConfiguration( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + encodedJitConfiguration: string, + ssmClient: SSMClient, + signal: AbortSignal, +): Promise { + const valueSize = Buffer.byteLength(encodedJitConfiguration, 'utf8'); + if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { + throw new Ec2ScaleSetValidationError('JIT configuration must be between 1 and 8192 bytes'); + } + + await ssmClient.send( + new PutParameterCommand({ + Name: jitParameterName(input.configuration, instanceId), + Value: encodedJitConfiguration, + Type: 'SecureString', + KeyId: input.configuration.ssmKmsKeyId, + Overwrite: false, + Tier: valueSize >= SSM_STANDARD_TIER_THRESHOLD ? 'Advanced' : 'Standard', + Tags: jitParameterTags(input, instanceId), + }), + { abortSignal: signal }, + ); +} + +async function bestEffortCancelJitPublication( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + ssmClient: SSMClient, + signal: AbortSignal, +): Promise { + try { + await ssmClient.send(new DeleteParameterCommand({ Name: jitParameterName(input.configuration, instanceId) }), { + abortSignal: signal, + }); + } catch (error) { + throwIfAborted(signal, error); + } +} + +function validateJitResult( + result: GenerateScaleSetJitConfigurationResult, + expectedRunnerName: string, + scaleSetId: number, +): void { + if ( + !Number.isSafeInteger(result.runnerId) || + result.runnerId <= 0 || + result.runnerName !== expectedRunnerName || + result.scaleSetId !== scaleSetId + ) { + throw new Ec2ScaleSetValidationError('JIT configuration returned an unexpected runner identity'); + } + const valueSize = Buffer.byteLength(result.encodedJitConfiguration, 'utf8'); + if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { + throw new Ec2ScaleSetValidationError('JIT configuration has an invalid size'); + } +} + +async function terminateUnpublishedRunner( + instanceId: string, + state: MutableReconcileState, + runners: Ec2RunnerResourceOperations, + signal: AbortSignal, +): Promise { + try { + await runners.terminate(instanceId); + state.currentRunners--; + state.actions.terminated++; + } catch (error) { + throwIfAborted(signal, error); + retainUnknown(state, instanceId); + state.errors.push(safeError('terminate', error, { resourceId: instanceId })); + } +} + +async function cleanGitHubRunner( + jit: GenerateScaleSetJitConfigurationResult, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, +): Promise { + try { + await request.removeRunner({ + runnerId: jit.runnerId, + runnerName: jit.runnerName, + scaleSetId: jit.scaleSetId, + signal: request.signal, + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('remove_runner', error, { runnerName: jit.runnerName })); + } +} + +async function configureLaunchedRunner( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runners: Ec2RunnerResourceOperations, + ssmClient: SSMClient, +): Promise { + const runnerName = `${input.configuration.runnerNamePrefix}${instanceId}`; + if (runnerName.length > GITHUB_RUNNER_NAME_MAX_LENGTH) { + state.errors.push({ + operation: 'generate_jit_configuration', + code: 'RUNNER_NAME_TOO_LONG', + resourceId: instanceId, + }); + await terminateUnpublishedRunner(instanceId, state, runners, request.signal); + return; + } + + let jit: GenerateScaleSetJitConfigurationResult; + try { + jit = await request.generateJitConfiguration({ runnerName, signal: request.signal }); + validateJitResult(jit, runnerName, input.scaleSetId); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('generate_jit_configuration', error, { runnerName, resourceId: instanceId })); + await terminateUnpublishedRunner(instanceId, state, runners, request.signal); + return; + } + + try { + await runners.tag(instanceId, [ + ...ownershipTags(input), + { Key: EC2_RUNNER_NAME_TAG, Value: jit.runnerName }, + { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(jit.runnerId) }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'publishing' }, + ]); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('launch', error, { runnerName, resourceId: instanceId })); + await cleanGitHubRunner(jit, request, state); + await terminateUnpublishedRunner(instanceId, state, runners, request.signal); + return; + } + + try { + await publishJitConfiguration(input, instanceId, jit.encodedJitConfiguration, ssmClient, request.signal); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('publish_jit_configuration', error, { runnerName, resourceId: instanceId })); + await bestEffortCancelJitPublication(input, instanceId, ssmClient, request.signal); + // Main's bootstrap reads before deleting. Even a successful controller-side + // DeleteParameter can race after that read and cannot prove non-consumption. + // Preserve both GitHub and compute state until an exact lifecycle signal is observed. + retainUnknown(state, instanceId); + return; + } + + state.actions.launched++; + try { + await runners.tag(instanceId, [{ Key: EC2_SCALE_SET_STATE_TAG, Value: 'config-published' }]); + } catch (error) { + throwIfAborted(request.signal, error); + // Publication may already have been consumed. Preserve the instance and exact GitHub identity. + retainUnknown(state, instanceId); + state.errors.push(safeError('launch', error, { runnerName, resourceId: instanceId })); + } +} + +export async function scaleUp( + input: CreateEc2ScaleSetProviderInput, + count: number, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runners: Ec2RunnerResourceOperations, + ssmClient: SSMClient, +): Promise { + const runnerIdentity = runnerIdentityFromGitHubScope(input.githubScope); + let createResult; + try { + createResult = await runners.create({ + environment: input.configuration.environment, + runnerOwner: runnerIdentity.runnerOwner, + runnerType: runnerIdentity.runnerType, + subnets: input.configuration.subnets, + launchTemplateName: input.configuration.launchTemplateName, + ec2instanceCriteria: input.configuration.ec2instanceCriteria, + ec2OverrideConfig: input.configuration.ec2OverrideConfig, + numberOfRunners: count, + source: SCALE_SET_RUNNER_SOURCE, + amiIdSsmParameterName: input.configuration.amiIdSsmParameterName, + tracingEnabled: input.configuration.tracingEnabled, + onDemandFailoverOnError: input.configuration.onDemandFailoverOnError, + useDedicatedHost: input.configuration.useDedicatedHost, + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('launch', error)); + return; + } + + state.currentRunners += createResult.instances.length; + if (createResult.failedInstanceCount > 0) { + state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_FAILED' }); + } + + for (const instanceId of createResult.instances) { + request.signal.throwIfAborted(); + await configureLaunchedRunner(input, instanceId, request, state, runners, ssmClient); + } +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts new file mode 100644 index 0000000000..abbd63a16c --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts @@ -0,0 +1,111 @@ +import { createHash } from 'node:crypto'; + +import type { Instance } from '@aws-sdk/client-ec2'; +import { vi } from 'vitest'; + +import type { + GenerateScaleSetJitConfigurationResult, + ScaleSetReconcileRequest, + ScaleSetRunnerState, +} from '../../../../../scale-set'; +import type { Ec2ScaleSetProviderConfig } from '../configuration'; +import { + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_GITHUB_RUNNER_ID_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_RUNNER_NAME_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, +} from '../inventory'; + +export const signal = new AbortController().signal; +export const githubScope = 'https://github.com/example'; +export const githubScopeHash = createHash('sha256').update(githubScope, 'utf8').digest('hex'); + +export const config: Ec2ScaleSetProviderConfig = { + region: 'eu-west-1', + environment: 'unit-test', + runnerNamePrefix: 'runner-', + jitConfigParameterPath: '/github-action-runners/unit-test/runners/tokens', + subnets: ['subnet-12345678'], + launchTemplateName: 'unit-test-runners', + ec2instanceCriteria: { + instanceTypes: ['m7i.large'], + targetCapacityType: 'on-demand', + instanceAllocationStrategy: 'lowest-price', + }, + ssmParameterTags: [{ Key: 'Project', Value: 'runner-tests' }], +}; + +export function ownedInstance( + instanceId: string, + identity?: { runnerId: number; runnerName: string }, + overrides: { + runnerConfigName?: string; + scaleSetId?: number; + scaleSetState?: string; + githubScopeHash?: string; + launchTime?: Date; + } = {}, +): Instance { + return { + InstanceId: instanceId, + LaunchTime: overrides.launchTime ?? new Date('2026-08-24T10:00:00Z'), + Tags: [ + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:created_by', Value: 'scale-set-service' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: overrides.runnerConfigName ?? 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(overrides.scaleSetId ?? 42) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: overrides.githubScopeHash ?? githubScopeHash }, + { + Key: EC2_SCALE_SET_STATE_TAG, + Value: overrides.scaleSetState ?? (identity ? 'config-published' : 'provisioning'), + }, + ...(identity + ? [ + { Key: EC2_RUNNER_NAME_TAG, Value: identity.runnerName }, + { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(identity.runnerId) }, + ] + : []), + ], + }; +} + +export function githubState( + runnerId: number, + runnerName: string, + overrides: Partial = {}, +): ScaleSetRunnerState { + return { + runnerId, + runnerName, + scaleSetId: 42, + status: 'online', + busy: false, + lifecycle: 'unknown', + ...overrides, + }; +} + +export function jitResult(instanceId = 'i-1234567890abcdef0'): GenerateScaleSetJitConfigurationResult { + return { + encodedJitConfiguration: 'sensitive-encoded-jit-configuration', + runnerId: 101, + runnerName: `runner-${instanceId}`, + scaleSetId: 42, + }; +} + +export function createRequest(overrides: Partial = {}): ScaleSetReconcileRequest { + return { + desiredRunners: 1, + busyRunners: 0, + bootTimeoutMinutes: 10, + runnerStates: [], + signal, + generateJitConfiguration: vi.fn().mockResolvedValue(jitResult()), + removeRunner: vi.fn().mockResolvedValue({ status: 'removed' }), + ...overrides, + }; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/provider-harness.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/provider-harness.ts new file mode 100644 index 0000000000..510009001a --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/provider-harness.ts @@ -0,0 +1,40 @@ +import { CreateTagsCommand, EC2Client, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { DeleteParameterCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; + +import type { Ec2ScaleSetProviderConfig } from '../configuration'; +import { createEc2ScaleSetProvider } from '../provider'; +import { config, githubScope } from './fixtures'; + +export const ec2Mock = mockClient(EC2Client); +export const ssmMock = mockClient(SSMClient); +const ec2Client = new EC2Client({ region: 'eu-west-1' }); +const ssmClient = new SSMClient({ region: 'eu-west-1' }); + +export function createTestProvider( + options: { githubScope?: string; now?: () => number; configuration?: Ec2ScaleSetProviderConfig } = {}, +) { + return createEc2ScaleSetProvider( + { + runnerConfigName: 'linux', + scaleSetId: 42, + githubScope: options.githubScope ?? githubScope, + configuration: options.configuration ?? config, + }, + { + ec2Client, + ssmClient, + now: options.now ?? (() => new Date('2026-08-24T10:05:00Z').getTime()), + }, + ); +} + +export function resetAwsMocks(): void { + ec2Mock.reset(); + ssmMock.reset(); + ec2Mock.on(CreateTagsCommand).resolves({}); + ec2Mock.on(TerminateInstancesCommand).resolves({}); + ssmMock.on(PutParameterCommand).resolves({}); + ssmMock.on(DeleteParameterCommand).resolves({}); +} diff --git a/lambdas/libs/compute-providers/core/index.ts b/lambdas/libs/compute-providers/core/index.ts index c5560942fa..ece1bc603e 100644 --- a/lambdas/libs/compute-providers/core/index.ts +++ b/lambdas/libs/compute-providers/core/index.ts @@ -6,7 +6,7 @@ export interface ComputeProvider { type: ComputeProviderType; } -export type RunnerSource = 'scale-up-lambda' | 'pool-lambda'; +export type RunnerSource = 'scale-up-lambda' | 'pool-lambda' | 'scale-set-service'; export type RunnerType = 'Org' | 'Repo'; export interface CreateGitHubRunnerConfig { diff --git a/lambdas/libs/compute-providers/package.json b/lambdas/libs/compute-providers/package.json index c1806818cc..0ab5bb25de 100644 --- a/lambdas/libs/compute-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -8,8 +8,10 @@ "./provider-types": "./provider-types.ts", "./webhook": "./webhook.ts", "./control-plane": "./control-plane.ts", + "./scale-set": "./scale-set.ts", "./aws/ec2/webhook": "./aws/ec2/webhook.ts", "./aws/ec2/control-plane": "./aws/ec2/control-plane.ts", + "./aws/ec2/scale-set": "./aws/ec2/scale-set.ts", "./aws/ec2/runners": "./aws/ec2/src/runners.ts", "./aws/ec2/control-plane/runner-creation": "./aws/ec2/src/control-plane/runner-creation.ts" }, @@ -27,6 +29,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-sdk/client-ec2": "^3.1009.0", + "@aws-sdk/client-ssm": "^3.1009.0", "@octokit/rest": "22.0.1", "moment": "2.29.4", "yn": "3.1.1" diff --git a/lambdas/libs/compute-providers/providers.config.scale-set.ts b/lambdas/libs/compute-providers/providers.config.scale-set.ts new file mode 100644 index 0000000000..7f8f0ecc64 --- /dev/null +++ b/lambdas/libs/compute-providers/providers.config.scale-set.ts @@ -0,0 +1,5 @@ +import { provider as ec2 } from './aws/ec2/scale-set'; +import type { ScaleSetComputeProviderModule } from './scale-set'; + +/** Provider plugins included in the scale-set service bundle. */ +export const enabledScaleSetProviders = [ec2] as const satisfies readonly ScaleSetComputeProviderModule[]; diff --git a/lambdas/libs/compute-providers/scale-set.test.ts b/lambdas/libs/compute-providers/scale-set.test.ts new file mode 100644 index 0000000000..f56e044d86 --- /dev/null +++ b/lambdas/libs/compute-providers/scale-set.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { createEc2ScaleSetPlugin } from './aws/ec2/scale-set'; +import { + createScaleSetComputeProviderRegistry, + type ScaleSetComputeProviderPlugin, + validateScaleSetProviderEnvironmentVariables, +} from './scale-set'; + +const configuration = { + region: 'eu-west-1', + environment: 'unit-test', + runnerNamePrefix: 'runner-', + jitConfigParameterPath: '/github-action-runners/unit-test/runners/tokens', + subnets: ['subnet-12345678'], + launchTemplateName: 'unit-test-runners', + ec2instanceCriteria: { + instanceTypes: ['m7i.large'], + targetCapacityType: 'on-demand', + instanceAllocationStrategy: 'lowest-price', + }, +}; + +describe('scale-set compute-provider registry', () => { + it('creates a separate provider instance for every runner config', () => { + const registry = createScaleSetComputeProviderRegistry([createEc2ScaleSetPlugin()]); + const first = registry.create('ec2', { + runnerConfigName: 'shared', + scaleSetId: 1, + githubScope: 'https://github.com/first', + configuration, + }); + const second = registry.create('ec2', { + runnerConfigName: 'shared', + scaleSetId: 1, + githubScope: 'https://github.com/second', + configuration, + }); + + expect(first).not.toBe(second); + expect(first.reconcile).toEqual(expect.any(Function)); + expect(second.reconcile).toEqual(expect.any(Function)); + expect(registry.environmentVariables('ec2')).toEqual({}); + expect(Object.isFrozen(registry.environmentVariables('ec2'))).toBe(true); + }); + + it('rejects duplicate and missing plugins explicitly', () => { + const plugin: ScaleSetComputeProviderPlugin = { + type: 'test', + capabilities: { + environmentVariables: {}, + create: vi.fn(() => ({ reconcile: vi.fn() })), + }, + }; + + expect(() => createScaleSetComputeProviderRegistry([plugin, plugin])).toThrow( + "Duplicate scale-set compute provider plugin 'test'", + ); + expect(() => + createScaleSetComputeProviderRegistry([]).create('missing', { + runnerConfigName: 'runner', + scaleSetId: 1, + githubScope: 'https://github.com/example', + configuration: {}, + }), + ).toThrow("No scale-set compute provider plugin registered for 'missing'"); + expect(() => createScaleSetComputeProviderRegistry([]).environmentVariables('missing')).toThrow( + "No scale-set compute provider plugin registered for 'missing'", + ); + }); + + it('returns a validated immutable provider environment', () => { + const source = { EC2_ENDPOINT_MODE: 'regional' }; + const registry = createScaleSetComputeProviderRegistry([ + { + type: 'test', + capabilities: { + environmentVariables: source, + create: vi.fn(() => ({ reconcile: vi.fn() })), + }, + }, + ]); + + const environment = registry.environmentVariables('test'); + source.EC2_ENDPOINT_MODE = 'changed-after-registration'; + expect(environment).toEqual({ EC2_ENDPOINT_MODE: 'regional' }); + expect(Object.isFrozen(environment)).toBe(true); + }); + + it.each>>([ + { AWS_REGION: 'eu-west-1' }, + { SCALE_SET_OVERRIDE: 'unsafe' }, + { NODE_OPTIONS: '--import=untrusted' }, + { PATH: '/untrusted' }, + { lower_case: 'value' }, + { VALID_NAME: 'line\nbreak' }, + { VALID_NAME: 'x'.repeat(4097) }, + ])('rejects reserved or unsafe provider environment variables: %o', (environmentVariables) => { + expect(() => + createScaleSetComputeProviderRegistry([ + { + type: 'test', + capabilities: { + environmentVariables, + create: vi.fn(() => ({ reconcile: vi.fn() })), + }, + }, + ]), + ).toThrow(/reserved or invalid|invalid value/); + }); + + it('rejects malformed or oversized provider environments', () => { + expect(() => validateScaleSetProviderEnvironmentVariables(null as never)).toThrow('must be an object'); + expect(() => validateScaleSetProviderEnvironmentVariables([] as never)).toThrow('must be an object'); + expect(() => validateScaleSetProviderEnvironmentVariables({ VALID_NAME: 42 } as never)).toThrow( + 'has an invalid value', + ); + expect(() => + validateScaleSetProviderEnvironmentVariables( + Object.fromEntries(Array.from({ length: 65 }, (_, index) => [`PROVIDER_${index}`, 'value'])), + ), + ).toThrow('must contain at most 64 entries'); + }); +}); diff --git a/lambdas/libs/compute-providers/scale-set.ts b/lambdas/libs/compute-providers/scale-set.ts new file mode 100644 index 0000000000..860138bbe9 --- /dev/null +++ b/lambdas/libs/compute-providers/scale-set.ts @@ -0,0 +1,205 @@ +import { enabledScaleSetProviders } from './providers.config.scale-set'; + +export type ScaleSetRunnerStatus = 'online' | 'offline' | 'unknown'; +export type ScaleSetRunnerLifecycle = 'started' | 'completed' | 'unknown'; + +/** + * Controller-observed GitHub state for one runner. + * + * The compute provider treats missing, duplicate, or unrecognized state as + * unknown. Callers must not infer `busy: false` when GitHub did not provide a + * busy state. + */ +export interface ScaleSetRunnerState { + runnerId: number; + runnerName: string; + scaleSetId: number; + status: ScaleSetRunnerStatus; + busy: boolean | undefined; + lifecycle: ScaleSetRunnerLifecycle; +} + +export interface GenerateScaleSetJitConfigurationInput { + runnerName: string; + signal?: AbortSignal; +} + +export interface GenerateScaleSetJitConfigurationResult { + encodedJitConfiguration: string; + runnerId: number; + runnerName: string; + scaleSetId: number; +} + +export type GenerateScaleSetJitConfiguration = ( + input: GenerateScaleSetJitConfigurationInput, +) => Promise; + +export interface RemoveScaleSetRunnerInput { + runnerId: number; + runnerName: string; + scaleSetId: number; + signal?: AbortSignal; +} + +export type ScaleSetRemoveRunnerStatus = 'removed' | 'retained_busy' | 'retained_unknown'; + +export interface ScaleSetRemoveRunnerResult { + status: ScaleSetRemoveRunnerStatus; +} + +export type RemoveScaleSetRunner = (input: RemoveScaleSetRunnerInput) => Promise; + +export interface ScaleSetReconcileRequest { + desiredRunners: number; + /** Aggregate busy-runner count reported by the GitHub Actions scale-set session. */ + busyRunners: number; + bootTimeoutMinutes: number; + runnerStates: readonly ScaleSetRunnerState[]; + signal: AbortSignal; + generateJitConfiguration: GenerateScaleSetJitConfiguration; + removeRunner: RemoveScaleSetRunner; +} + +export type ScaleSetReconcileStatus = 'converged' | 'retained' | 'error'; + +export type ScaleSetReconcileOperation = + | 'validate' + | 'reconcile' + | 'list' + | 'launch' + | 'generate_jit_configuration' + | 'publish_jit_configuration' + | 'remove_runner' + | 'terminate'; + +/** Error metadata is deliberately bounded and never contains a JIT configuration or raw upstream error message. */ +export interface ScaleSetReconcileError { + operation: ScaleSetReconcileOperation; + code: string; + runnerName?: string; + resourceId?: string; +} + +export interface ScaleSetReconcileActions { + launched: number; + terminated: number; + retainedBusy: number; + retainedUnknown: number; +} + +export interface ScaleSetReconcileResult { + status: ScaleSetReconcileStatus; + desiredRunners: number; + /** Best-known owned capacity after actions completed; the next reconciliation re-observes AWS. */ + currentRunners: number; + actions: ScaleSetReconcileActions; + errors: readonly ScaleSetReconcileError[]; +} + +export interface ScaleSetComputeProvider { + reconcile(request: ScaleSetReconcileRequest): Promise; +} + +export interface ScaleSetComputeProviderFactoryInput { + runnerConfigName: string; + scaleSetId: number; + /** Canonical GitHub configuration URL used as an immutable provider ownership scope. */ + githubScope: string; + /** Provider-owned configuration. The selected provider validates it before use. */ + configuration: unknown; +} + +export interface ScaleSetComputeProviderCapabilities { + /** Provider-owned, non-secret task environment. Values are validated and immutable after registration. */ + environmentVariables: Readonly>; + create(input: ScaleSetComputeProviderFactoryInput): ScaleSetComputeProvider; +} + +export interface ScaleSetComputeProviderPlugin { + type: TType; + capabilities: ScaleSetComputeProviderCapabilities; +} + +export interface ScaleSetComputeProviderModule { + type: TType; + createPlugin(): ScaleSetComputeProviderPlugin; +} + +const SCALE_SET_ENVIRONMENT_KEY = /^[A-Z][A-Z0-9_]{0,127}$/; +const RESERVED_SCALE_SET_ENVIRONMENT_PREFIXES = ['AWS_', 'ECS_', 'GITHUB_', 'SCALE_SET_', 'NODE_']; +const RESERVED_SCALE_SET_ENVIRONMENT_KEYS = new Set(['PATH', 'HOME', 'HOSTNAME', 'PWD', 'SHLVL']); +const MAX_SCALE_SET_ENVIRONMENT_VARIABLES = 64; +const MAX_SCALE_SET_ENVIRONMENT_VALUE_BYTES = 4096; + +export function validateScaleSetProviderEnvironmentVariables( + value: Readonly>, +): Readonly> { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Scale-set provider environmentVariables must be an object'); + } + const entries = Object.entries(value); + if (entries.length > MAX_SCALE_SET_ENVIRONMENT_VARIABLES) { + throw new Error( + `Scale-set provider environmentVariables must contain at most ${MAX_SCALE_SET_ENVIRONMENT_VARIABLES} entries`, + ); + } + + const normalized = Object.create(null) as Record; + for (const [key, environmentValue] of entries) { + if ( + !SCALE_SET_ENVIRONMENT_KEY.test(key) || + RESERVED_SCALE_SET_ENVIRONMENT_KEYS.has(key) || + RESERVED_SCALE_SET_ENVIRONMENT_PREFIXES.some((prefix) => key.startsWith(prefix)) + ) { + throw new Error(`Scale-set provider environment variable '${key}' is reserved or invalid`); + } + if ( + typeof environmentValue !== 'string' || + Buffer.byteLength(environmentValue, 'utf8') > MAX_SCALE_SET_ENVIRONMENT_VALUE_BYTES || + [...environmentValue].some((character) => { + const code = character.charCodeAt(0); + return code <= 31 || code === 127; + }) + ) { + throw new Error(`Scale-set provider environment variable '${key}' has an invalid value`); + } + normalized[key] = environmentValue; + } + return Object.freeze(normalized); +} + +export function createScaleSetComputeProviderRegistry( + plugins: readonly ScaleSetComputeProviderPlugin[] = enabledScaleSetProviders.map((provider) => + provider.createPlugin(), + ), +) { + const pluginsByType = new Map(); + const environmentVariablesByType = new Map>>(); + for (const plugin of plugins) { + if (pluginsByType.has(plugin.type)) { + throw new Error(`Duplicate scale-set compute provider plugin '${plugin.type}'`); + } + pluginsByType.set(plugin.type, plugin); + environmentVariablesByType.set( + plugin.type, + validateScaleSetProviderEnvironmentVariables(plugin.capabilities.environmentVariables), + ); + } + + function get(type: string): ScaleSetComputeProviderPlugin { + const plugin = pluginsByType.get(type); + if (!plugin) throw new Error(`No scale-set compute provider plugin registered for '${type}'`); + return plugin; + } + + return { + create(type: string, input: ScaleSetComputeProviderFactoryInput): ScaleSetComputeProvider { + return get(type).capabilities.create(input); + }, + environmentVariables(type: string): Readonly> { + get(type); + return environmentVariablesByType.get(type)!; + }, + }; +} diff --git a/lambdas/libs/compute-providers/vitest.config.ts b/lambdas/libs/compute-providers/vitest.config.ts index fd62b358c0..24c4554995 100644 --- a/lambdas/libs/compute-providers/vitest.config.ts +++ b/lambdas/libs/compute-providers/vitest.config.ts @@ -16,7 +16,7 @@ export default mergeConfig(defaultConfig, { 'core/**/*.ts', 'aws/**/*.ts', ], - exclude: ['**/*.test.ts', '**/*.d.ts', 'templates/**/*'], + exclude: ['**/*.test.ts', '**/test/**/*.ts', '**/*.d.ts', 'templates/**/*'], thresholds: { statements: 96.16, branches: 95.32, diff --git a/lambdas/libs/github-actions-scale-set/LICENSE.actions-scaleset b/lambdas/libs/github-actions-scale-set/LICENSE.actions-scaleset new file mode 100644 index 0000000000..28a50fa226 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/LICENSE.actions-scaleset @@ -0,0 +1,21 @@ +MIT License + +Copyright GitHub, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lambdas/libs/github-actions-scale-set/README.md b/lambdas/libs/github-actions-scale-set/README.md new file mode 100644 index 0000000000..6696627a11 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/README.md @@ -0,0 +1,69 @@ +# GitHub Actions runner scale-set client for TypeScript + +This workspace package implements the GitHub Actions runner scale-set protocol with native `fetch`. It is intended for the `terraform-aws-github-runner` control plane and can also be reused by other Node.js scale-set listeners. + +The protocol is currently a GitHub Public Preview. This implementation follows the public [`actions/scaleset`](https://github.com/actions/scaleset) Go client at commit [`cb0405b`](https://github.com/actions/scaleset/tree/cb0405b2d874500e75ae34eff8d582ab75956b45). + +The upstream copyright and MIT permission notice are retained in [`LICENSE.actions-scaleset`](./LICENSE.actions-scaleset). + +## What it provides + +- HTTPS organization, repository, enterprise, GitHub.com, and GHES registration URLs; +- PAT authentication or an asynchronous access-token provider for existing GitHub App authentication; +- runner scale-set CRUD and runner-group lookup; +- just-in-time runner configuration generation; +- runner lookup and removal; +- message-session creation, refresh, long polling, acknowledgement, and job acquisition; +- bounded retries for idempotent requests that encounter network failures, HTTP 429, or HTTP 5xx responses. + +There is no scale-up or scale-down REST operation. A listener polls the message queue and reports its maximum capacity. GitHub returns `statistics.totalAssignedJobs`; the caller reconciles its compute capacity to that value and terminates ephemeral compute after `JobCompleted` messages. + +## Basic usage + +```ts +import { GitHubActionsScaleSetClient } from '@aws-github-runner/github-actions-scale-set'; + +const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + accessTokenProvider: async () => installationAccessToken, + systemInfo: { + system: 'terraform-aws-github-runner', + subsystem: 'scale-set-listener', + }, + retry: { + maxRetries: 4, + initialBackoffMs: 1_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 5 * 60_000, + }, +}); + +const scaleSet = await client.createRunnerScaleSet({ + name: 'linux-x64', + runnerGroupId: 1, + runnerSetting: { disableUpdate: true }, +}); + +const session = await client.createMessageSessionClient(scaleSet.id!, 'listener-01'); + +try { + const message = await session.getMessage(0, 20); + if (message) { + await session.deleteMessage(message.messageId); + + const availableIds = message.jobAvailableMessages.map((job) => job.runnerRequestId); + await session.acquireJobs(availableIds); + + // Reconcile compute from message.statistics.totalAssignedJobs and use + // client.generateJitRunnerConfig(...) for every runner being created. + } +} finally { + await session.close(); +} +``` + +Treat encoded JIT configurations and all access tokens as secrets. The upstream Go listener acknowledges a message before job acquisition and scaling callbacks; a later callback failure is returned from the listener and does not cause message redelivery. + +The retry values shown above are the defaults. Automatic transport retries apply only to idempotent methods (`GET`, `HEAD`, `OPTIONS`, `PUT`, and `DELETE`). Non-idempotent `POST` and `PATCH` operations, including JIT generation, session creation, and job acquisition, are attempted once so a lost response cannot cause the operation to be replayed. `Retry-After` is honored for eligible 429/5xx responses and capped by `maxBackoffMs`. Caller cancellation interrupts both an active request and retry backoff. + +The `/actions/runner-registration` admin bootstrap is the sole narrowly scoped exception: its `POST` retries transient transport failures and 429/5xx responses, plus 401/403 while RemoteAuth propagates. Queue 401 responses are not transport-retried; they trigger the message-session token refresh flow once. diff --git a/lambdas/libs/github-actions-scale-set/package.json b/lambdas/libs/github-actions-scale-set/package.json new file mode 100644 index 0000000000..8e0d87576a --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/package.json @@ -0,0 +1,35 @@ +{ + "name": "@aws-github-runner/github-actions-scale-set", + "version": "1.0.0", + "main": "src/index.ts", + "type": "module", + "license": "MIT", + "exports": { + ".": "./src/index.ts", + "./config": "./src/config.ts", + "./errors": "./src/errors.ts", + "./types": "./src/types.ts" + }, + "scripts": { + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint src", + "format": "prettier --write \"**/*.ts\"", + "format-check": "prettier --check \"**/*.ts\"", + "typecheck": "tsc --noEmit", + "all": "yarn format && yarn lint && yarn typecheck && yarn test" + }, + "devDependencies": { + "@types/node": "^22.19.3", + "typescript": "^5.9.3" + }, + "nx": { + "includedScripts": [ + "format", + "format-check", + "lint", + "typecheck", + "all" + ] + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/client.test.ts b/lambdas/libs/github-actions-scale-set/src/client.test.ts new file mode 100644 index 0000000000..246ee8cbb8 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/client.test.ts @@ -0,0 +1,434 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { actionsServiceUrl, GitHubActionsScaleSetClient } from './client'; +import { SCALE_SET_ERROR_CODES, ScaleSetHttpError, ScaleSetProtocolError } from './errors'; +import { ScaleSetFetch } from './types'; + +type RequestInput = Parameters[0]; +type ServiceHandler = (url: URL, init: RequestInit) => Response | Promise; + +function requestUrl(input: RequestInput): URL { + if (input instanceof Request) { + return new URL(input.url); + } + return new URL(input.toString()); +} + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function actionsAdminToken(expiresAt = Math.floor(Date.now() / 1000) + 60 * 60): string { + const header = Buffer.from(JSON.stringify({ alg: 'RS256', typ: 'JWT' })).toString('base64url'); + const payload = Buffer.from(JSON.stringify({ exp: expiresAt })).toString('base64url'); + return `${header}.${payload}.signature`; +} + +function clientFixture(serviceHandler: ServiceHandler, options: { adminToken?: () => string; now?: () => Date } = {}) { + const accessTokenProvider = vi.fn(async () => 'github-access-token'); + const registrationRequests: Array<{ url: URL; init: RequestInit }> = []; + const serviceRequests: Array<{ url: URL; init: RequestInit }> = []; + const fetchImplementation = vi.fn(async (input, init = {}) => { + const url = requestUrl(input); + + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + registrationRequests.push({ url, init }); + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + registrationRequests.push({ url, init }); + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: options.adminToken?.() ?? actionsAdminToken(), + }); + } + + serviceRequests.push({ url, init }); + return serviceHandler(url, init); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + accessTokenProvider, + fetch: fetchImplementation, + systemInfo: { system: 'unit-test', subsystem: 'sdk' }, + now: options.now, + }); + + return { + accessTokenProvider, + client, + fetchImplementation, + registrationRequests, + serviceRequests, + }; +} + +describe('actionsServiceUrl', () => { + it('normalizes a long trailing slash sequence before joining the request path', () => { + const base = `https://actions.example/tenant/123${'/'.repeat(10_000)}`; + + expect(actionsServiceUrl(base, '').pathname).toBe('/tenant/123'); + expect(actionsServiceUrl(base, '_apis/runtime/runnerscalesets').pathname).toBe( + '/tenant/123/_apis/runtime/runnerscalesets', + ); + const url = actionsServiceUrl(base, '/_apis/runtime/runnerscalesets'); + expect(url.pathname).toBe('/tenant/123/_apis/runtime/runnerscalesets'); + expect(url.searchParams.get('api-version')).toBe('6.0-preview'); + }); +}); + +describe('GitHubActionsScaleSetClient', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('bootstraps Actions authentication once and reuses the unexpired admin token', async () => { + const fixture = clientFixture(() => jsonResponse({ count: 0, value: [] })); + + await expect(fixture.client.getRunnerScaleSet(4, 'linux')).resolves.toBeNull(); + await expect(fixture.client.getRunnerScaleSet(4, 'linux')).resolves.toBeNull(); + + expect(fixture.accessTokenProvider).toHaveBeenCalledOnce(); + expect(fixture.registrationRequests).toHaveLength(2); + expect(fixture.serviceRequests).toHaveLength(2); + + const registrationTokenRequest = fixture.registrationRequests[0]; + expect(registrationTokenRequest.url.toString()).toBe( + 'https://api.github.com/orgs/example/actions/runners/registration-token', + ); + expect(new Headers(registrationTokenRequest.init.headers).get('Authorization')).toBe('Bearer github-access-token'); + expect(new Headers(registrationTokenRequest.init.headers).get('Content-Type')).toBe( + 'application/vnd.github.v3+json', + ); + expect(JSON.parse(new Headers(registrationTokenRequest.init.headers).get('User-Agent') as string)).toMatchObject({ + build_commit_sha: '', + kind: 'scaleset', + system: 'unit-test', + }); + + const adminConnectionRequest = fixture.registrationRequests[1]; + expect(adminConnectionRequest.url.toString()).toBe('https://api.github.com/actions/runner-registration'); + expect(new Headers(adminConnectionRequest.init.headers).get('Authorization')).toBe( + 'RemoteAuth runner-registration-token', + ); + expect(JSON.parse(adminConnectionRequest.init.body as string)).toEqual({ + url: 'https://github.com/example', + runner_event: 'register', + }); + + for (const { url, init } of fixture.serviceRequests) { + expect(url.pathname).toBe('/tenant/123/_apis/runtime/runnerscalesets'); + expect(url.searchParams.get('api-version')).toBe('6.0-preview'); + expect(url.searchParams.get('runnerGroupId')).toBe('4'); + expect(url.searchParams.get('name')).toBe('linux'); + expect(new Headers(init.headers).get('Authorization')).toMatch(/^Bearer /); + } + }); + + it.each([ + 'http://actions.example/tenant/123', + 'https://user:password@actions.example/tenant/123', + 'https://actions.example/tenant/123?signature=secret', + 'https://actions.example/tenant/123#fragment', + ])('rejects an unsafe Actions service admin URL: %s', async (unsafeUrl) => { + const fetchImplementation = vi.fn(async (input) => { + const url = requestUrl(input); + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + return jsonResponse({ url: unsafeUrl, token: actionsAdminToken() }); + } + return new Response(null, { status: 500 }); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-access-token', + fetch: fetchImplementation, + }); + + await expect(client.getRunnerScaleSetById(42)).rejects.toBeInstanceOf(ScaleSetProtocolError); + }); + + it('refreshes the Actions admin token when it enters the 60-second expiry window', async () => { + let nowMs = Date.UTC(2026, 7, 14, 12, 0, 0); + let tokenIssue = 0; + const fixture = clientFixture(() => jsonResponse({ count: 0, value: [] }), { + now: () => new Date(nowMs), + adminToken: () => { + tokenIssue += 1; + const lifetimeSeconds = tokenIssue === 1 ? 120 : 3_600; + return actionsAdminToken(Math.floor(nowMs / 1000) + lifetimeSeconds); + }, + }); + + await fixture.client.getRunnerScaleSet(4, 'linux'); + nowMs += 70_000; + await fixture.client.getRunnerScaleSet(4, 'linux'); + + expect(fixture.accessTokenProvider).toHaveBeenCalledTimes(2); + expect(fixture.registrationRequests).toHaveLength(4); + expect(tokenIssue).toBe(2); + }); + + it('retries transient and propagation failures only while bootstrapping the admin connection', async () => { + let registrationTokenRequests = 0; + let adminConnectionRequests = 0; + let actionsRequests = 0; + const fetchImplementation = vi.fn(async (input) => { + const url = requestUrl(input); + + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + registrationTokenRequests += 1; + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + adminConnectionRequests += 1; + if (adminConnectionRequests === 1) { + return jsonResponse({ message: 'temporarily unavailable' }, 503); + } + if (adminConnectionRequests === 2) { + return jsonResponse({ message: 'not propagated' }, 401); + } + if (adminConnectionRequests === 3) { + return jsonResponse({ message: 'not propagated' }, 403); + } + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: actionsAdminToken(), + }); + } + + actionsRequests += 1; + return jsonResponse({ count: 0, value: [] }); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-access-token', + fetch: fetchImplementation, + retry: { + maxRetries: 3, + initialBackoffMs: 0, + maxBackoffMs: 0, + requestTimeoutMs: 1_000, + }, + }); + + await expect(client.getRunnerScaleSet(4, 'linux')).resolves.toBeNull(); + + expect(registrationTokenRequests).toBe(1); + expect(adminConnectionRequests).toBe(4); + expect(actionsRequests).toBe(1); + }); + + it('does not replay JIT generation when its POST receives a transient failure', async () => { + let jitRequests = 0; + const fetchImplementation = vi.fn(async (input) => { + const url = requestUrl(input); + + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: actionsAdminToken(), + }); + } + if (url.pathname.endsWith('/generatejitconfig')) { + jitRequests += 1; + return jsonResponse({ message: 'temporarily unavailable' }, 503); + } + return new Response(null, { status: 500 }); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-access-token', + fetch: fetchImplementation, + retry: { + maxRetries: 4, + initialBackoffMs: 0, + maxBackoffMs: 0, + requestTimeoutMs: 1_000, + }, + }); + + await expect(client.generateJitRunnerConfig({ name: 'runner-71', workFolder: '_work' }, 42)).rejects.toMatchObject({ + status: 503, + }); + expect(jitRequests).toBe(1); + }); + + it('raises a typed HTTP error with Actions exception and request metadata', async () => { + const fixture = clientFixture( + () => + new Response( + JSON.stringify({ + typeName: 'Microsoft.TeamFoundation.DistributedTask.WebApi.AgentExistsException', + message: 'runner already exists', + }), + { + status: 409, + headers: { + ActivityId: 'activity-123', + 'Content-Type': 'application/json', + 'X-GitHub-Request-Id': 'github-456', + }, + }, + ), + ); + + const request = fixture.client.getRunner(71); + await expect(request).rejects.toBeInstanceOf(ScaleSetHttpError); + await expect(request).rejects.toMatchObject({ + code: SCALE_SET_ERROR_CODES.runnerExists, + status: 409, + activityId: 'activity-123', + githubRequestId: 'github-456', + exceptionName: 'Microsoft.TeamFoundation.DistributedTask.WebApi.AgentExistsException', + }); + }); + + it('uses the exact scale-set CRUD endpoints and legacy request body casing', async () => { + const fixture = clientFixture((url, init) => { + const method = init.method; + if (method === 'GET' && url.pathname.endsWith('/runnerscalesets')) { + return jsonResponse({ + count: url.searchParams.has('name') ? 1 : 2, + value: [ + { id: 11, name: 'linux', RunnerSetting: { disableUpdate: true } }, + { id: 12, name: 'windows', RunnerSetting: {} }, + ], + }); + } + if (method === 'GET' && url.pathname.endsWith('/runnerscalesets/11')) { + return jsonResponse({ id: 11, name: 'linux', RunnerSetting: { disableUpdate: true } }); + } + if (method === 'POST') { + return jsonResponse({ id: 11, name: 'linux', RunnerSetting: { disableUpdate: true } }); + } + if (method === 'PATCH') { + return jsonResponse({ id: 11, name: 'linux', RunnerSetting: { disableUpdate: false } }); + } + if (method === 'DELETE') { + return new Response(null, { status: 204 }); + } + return new Response(null, { status: 500 }); + }); + const createInput = { + name: 'linux', + runnerGroupId: 4, + runnerSetting: { disableUpdate: true }, + }; + const updateInput = { + labels: [{ name: 'arm64' }], + runnerSetting: { disableUpdate: false }, + }; + + await expect(fixture.client.getRunnerScaleSet(4, 'linux')).resolves.toMatchObject({ + id: 11, + runnerSetting: { disableUpdate: true }, + }); + await expect(fixture.client.listRunnerScaleSets(4)).resolves.toHaveLength(2); + await expect(fixture.client.getRunnerScaleSetById(11)).resolves.toMatchObject({ id: 11 }); + await expect(fixture.client.createRunnerScaleSet(createInput)).resolves.toMatchObject({ id: 11 }); + await expect(fixture.client.updateRunnerScaleSet(11, updateInput)).resolves.toMatchObject({ + id: 11, + }); + await expect(fixture.client.deleteRunnerScaleSet(11)).resolves.toBeUndefined(); + + expect(createInput).toMatchObject({ labels: [{ name: 'linux', type: 'System' }] }); + expect(updateInput).toMatchObject({ labels: [{ name: 'arm64', type: 'System' }] }); + + const createRequest = fixture.serviceRequests.find(({ init }) => init.method === 'POST'); + const updateRequest = fixture.serviceRequests.find(({ init }) => init.method === 'PATCH'); + expect(createRequest).toBeDefined(); + expect(updateRequest).toBeDefined(); + + const createBody = JSON.parse(createRequest?.init.body as string) as Record; + expect(createBody).toMatchObject({ + name: 'linux', + runnerGroupId: 4, + labels: [{ name: 'linux', type: 'System' }], + RunnerSetting: { disableUpdate: true }, + }); + expect(createBody).not.toHaveProperty('runnerSetting'); + + const updateBody = JSON.parse(updateRequest?.init.body as string) as Record; + expect(updateBody).toMatchObject({ + labels: [{ name: 'arm64', type: 'System' }], + RunnerSetting: { disableUpdate: false }, + }); + expect(updateBody).not.toHaveProperty('runnerSetting'); + + expect(fixture.serviceRequests.map(({ url, init }) => [init.method, url.pathname])).toEqual([ + ['GET', '/tenant/123/_apis/runtime/runnerscalesets'], + ['GET', '/tenant/123/_apis/runtime/runnerscalesets'], + ['GET', '/tenant/123/_apis/runtime/runnerscalesets/11'], + ['POST', '/tenant/123/_apis/runtime/runnerscalesets'], + ['PATCH', '/tenant/123/_apis/runtime/runnerscalesets/11'], + ['DELETE', '/tenant/123/_apis/runtime/runnerscalesets/11'], + ]); + for (const { url } of fixture.serviceRequests) { + expect(url.searchParams.get('api-version')).toBe('6.0-preview'); + } + }); + + it('uses the exact runner-group, JIT, and agent endpoints', async () => { + const fixture = clientFixture((url, init) => { + if (url.pathname.endsWith('/runnergroups/')) { + return jsonResponse({ + count: 1, + value: [{ id: 8, name: 'default', size: 0, isDefaultGroup: true }], + }); + } + if (url.pathname.endsWith('/generatejitconfig')) { + return jsonResponse({ + runner: { id: 71, name: 'runner-71', runnerScaleSetId: 42 }, + encodedJITConfig: 'encoded-jit', + }); + } + if (init.method === 'GET' && url.pathname.endsWith('/agents/71')) { + return jsonResponse({ id: 71, name: 'runner-71', runnerScaleSetId: 42 }); + } + if (init.method === 'GET' && url.pathname.endsWith('/agents')) { + return jsonResponse({ + count: 1, + value: [{ id: 71, name: 'runner-71', runnerScaleSetId: 42 }], + }); + } + if (init.method === 'DELETE' && url.pathname.endsWith('/agents/71')) { + return new Response(null, { status: 204 }); + } + return new Response(null, { status: 500 }); + }); + + await expect(fixture.client.getRunnerGroupByName('default')).resolves.toMatchObject({ id: 8 }); + await expect( + fixture.client.generateJitRunnerConfig({ name: 'runner-71', workFolder: '_work' }, 42), + ).resolves.toEqual({ + runner: { id: 71, name: 'runner-71', runnerScaleSetId: 42 }, + encodedJITConfig: 'encoded-jit', + }); + await expect(fixture.client.getRunner(71)).resolves.toMatchObject({ id: 71 }); + await expect(fixture.client.getRunnerByName('runner-71')).resolves.toMatchObject({ id: 71 }); + await expect(fixture.client.removeRunner(71)).resolves.toBeUndefined(); + + expect(fixture.serviceRequests.map(({ url, init }) => [init.method, url.pathname])).toEqual([ + ['GET', '/tenant/123/_apis/runtime/runnergroups/'], + ['POST', '/tenant/123/_apis/runtime/runnerscalesets/42/generatejitconfig'], + ['GET', '/tenant/123/_apis/distributedtask/pools/0/agents/71'], + ['GET', '/tenant/123/_apis/distributedtask/pools/0/agents'], + ['DELETE', '/tenant/123/_apis/distributedtask/pools/0/agents/71'], + ]); + expect(fixture.serviceRequests[0].url.searchParams.get('groupName')).toBe('default'); + expect(fixture.serviceRequests[3].url.searchParams.get('agentName')).toBe('runner-71'); + expect(JSON.parse(fixture.serviceRequests[1].init.body as string)).toEqual({ + name: 'runner-71', + workFolder: '_work', + }); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/client.ts b/lambdas/libs/github-actions-scale-set/src/client.ts new file mode 100644 index 0000000000..36e89cd586 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/client.ts @@ -0,0 +1,554 @@ +import { githubApiUrl, ParsedGitHubConfig, parseGitHubConfigUrl, runnerRegistrationTokenPath } from './config'; +import { ACTIONS_API_VERSION, RUNNER_ENDPOINT, RUNNER_GROUP_ENDPOINT, SCALE_SET_ENDPOINT } from './endpoints'; +import { ScaleSetProtocolError } from './errors'; +import { createRetryingFetch, executeRequest, HttpResult, parseJsonResponse } from './http'; +import { MessageSessionClient } from './message-session-client'; +import { + AccessTokenProvider, + GitHubActionsScaleSetClientOptions, + RunnerGroup, + RunnerReference, + RunnerScaleSet, + RunnerScaleSetJitRunnerConfig, + RunnerScaleSetJitRunnerSetting, + ScaleSetFetch, + ScaleSetRequestOptions, + SystemInfo, +} from './types'; +import { trimTrailingSlashes } from './url'; + +const ADMIN_TOKEN_REFRESH_SKEW_MS = 60_000; +const SUCCESS_STATUSES = Array.from({ length: 100 }, (_, index) => index + 200); + +interface RegistrationTokenResponse { + token?: string; + expires_at?: string; +} + +interface ActionsServiceAdminConnectionResponse { + url?: string; + token?: string; +} + +interface ActionsServiceAdminToken { + token: string; + expiresAt: Date; + url: string; +} + +interface RunnerScaleSetListResponse { + count: number; + value: RunnerScaleSet[]; +} + +interface RunnerGroupListResponse { + count: number; + value: RunnerGroup[]; +} + +interface RunnerReferenceListResponse { + count: number; + value: RunnerReference[]; +} + +interface ActionsRequestOptions extends ScaleSetRequestOptions { + query?: Record; + body?: unknown; + expectedStatuses: readonly number[]; + authorization?: string; +} + +function joinUrlPath(base: string, path: string): string { + const normalizedBase = trimTrailingSlashes(base); + if (normalizedBase === '') { + if (path === '') { + return ''; + } + return path.startsWith('/') ? path : `/${path}`; + } + if (path === '') { + return normalizedBase; + } + return `${normalizedBase}${path.startsWith('/') ? '' : '/'}${path}`; +} + +export function actionsServiceUrl( + base: string, + path: string, + query: Record = {}, +): URL { + const [pathOnly, pathQuery = ''] = path.split('?', 2); + const result = new URL(joinUrlPath(base, pathOnly)); + const mergedQuery = new URLSearchParams(pathQuery); + + for (const [name, value] of Object.entries(query)) { + if (value !== undefined) { + mergedQuery.set(name, String(value)); + } + } + if (!mergedQuery.get('api-version')) { + mergedQuery.set('api-version', ACTIONS_API_VERSION); + } + result.search = mergedQuery.toString(); + return result; +} + +function encodeSystemUserAgent(systemInfo: SystemInfo): string { + return JSON.stringify({ + system: systemInfo.system ?? '', + version: systemInfo.version ?? '', + commit_sha: systemInfo.commitSha ?? '', + scale_set_id: systemInfo.scaleSetId ?? 0, + subsystem: systemInfo.subsystem ?? '', + build_version: '1.0.0', + build_commit_sha: '', + kind: 'scaleset', + }); +} + +function parseJwtExpiration(token: string): Date { + const parts = token.split('.'); + if (parts.length < 2) { + throw new ScaleSetProtocolError('Actions service admin token is not a JWT'); + } + + let claims: unknown; + try { + claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')) as unknown; + } catch (error) { + throw new ScaleSetProtocolError('failed to decode Actions service admin token claims', { + cause: error, + }); + } + + if ( + typeof claims !== 'object' || + claims === null || + !('exp' in claims) || + typeof claims.exp !== 'number' || + !Number.isFinite(claims.exp) + ) { + throw new ScaleSetProtocolError('Actions service admin token is missing a numeric exp claim'); + } + + return new Date(claims.exp * 1000); +} + +function applyDefaultLabelTypes(scaleSet: RunnerScaleSet): void { + for (const label of scaleSet.labels ?? []) { + label.type ||= 'System'; + } +} + +function ensureLabels(scaleSet: RunnerScaleSet): void { + if ((scaleSet.labels?.length ?? 0) > 0) { + return; + } + if (!scaleSet.name) { + throw new ScaleSetProtocolError('runner scale set must have a name or at least one label'); + } + scaleSet.labels = [{ name: scaleSet.name, type: 'System' }]; +} + +function runnerScaleSetRequestBody(scaleSet: RunnerScaleSet): Record { + const wire = { ...scaleSet } as Record; + delete wire.runnerSetting; + // The capital R is intentional and matches the Actions scale set wire contract. + wire.RunnerSetting = scaleSet.runnerSetting ?? {}; + return wire; +} + +function normalizeRunnerScaleSet(scaleSet: RunnerScaleSet | null): RunnerScaleSet | null { + if (scaleSet === null) { + return null; + } + + const wire = scaleSet as RunnerScaleSet & { RunnerSetting?: RunnerScaleSet['runnerSetting'] }; + if (wire.runnerSetting === undefined && wire.RunnerSetting !== undefined) { + wire.runnerSetting = wire.RunnerSetting; + } + delete wire.RunnerSetting; + return wire; +} + +/** A native-fetch client for the GitHub Actions runner scale set APIs. */ +export class GitHubActionsScaleSetClient { + private readonly config: ParsedGitHubConfig; + private readonly fetchImplementation: ScaleSetFetch; + private readonly adminConnectionFetchImplementation: ScaleSetFetch; + private readonly accessTokenProvider: AccessTokenProvider; + private readonly now: () => Date; + private readonly customUserAgent?: string; + private currentSystemInfo: SystemInfo; + private currentUserAgent: string; + private adminToken?: ActionsServiceAdminToken; + private adminTokenRefresh?: Promise; + + constructor(options: GitHubActionsScaleSetClientOptions) { + this.config = parseGitHubConfigUrl(options.gitHubConfigUrl, options.forceGhes); + const fetchImplementation = options.fetch ?? globalThis.fetch; + if (typeof fetchImplementation !== 'function') { + throw new TypeError('a fetch implementation is required'); + } + this.fetchImplementation = createRetryingFetch(fetchImplementation, options.retry); + // Upstream retries transient RemoteAuth propagation failures only for this + // bootstrap POST. Queue 401s must remain owned by session-token refresh, + // and other non-idempotent SDK requests must never use this opt-in wrapper. + this.adminConnectionFetchImplementation = createRetryingFetch(fetchImplementation, options.retry, { + additionalRetryStatuses: [401, 403], + additionalRetryMethods: ['POST'], + }); + + const hasPersonalAccessToken = + typeof options.personalAccessToken === 'string' && options.personalAccessToken !== ''; + const hasAccessTokenProvider = typeof options.accessTokenProvider === 'function'; + if (hasPersonalAccessToken === hasAccessTokenProvider) { + throw new TypeError('provide exactly one of personalAccessToken or accessTokenProvider'); + } + + this.accessTokenProvider = hasPersonalAccessToken + ? async () => options.personalAccessToken as string + : (options.accessTokenProvider as AccessTokenProvider); + this.now = options.now ?? (() => new Date()); + this.currentSystemInfo = { ...options.systemInfo }; + this.customUserAgent = options.userAgent; + this.currentUserAgent = options.userAgent ?? encodeSystemUserAgent(this.currentSystemInfo); + } + + get gitHubConfig(): ParsedGitHubConfig { + return { + ...this.config, + configUrl: new URL(this.config.configUrl), + }; + } + + get systemInfo(): SystemInfo { + return { ...this.currentSystemInfo }; + } + + setSystemInfo(systemInfo: SystemInfo): void { + this.currentSystemInfo = { ...systemInfo }; + if (this.customUserAgent === undefined) { + this.currentUserAgent = encodeSystemUserAgent(systemInfo); + } + } + + debugInfo(): string { + return JSON.stringify({ + system_info: this.currentUserAgent, + }); + } + + async getRunnerScaleSet( + runnerGroupId: number, + runnerScaleSetName: string, + options: ScaleSetRequestOptions = {}, + ): Promise { + const { result, url } = await this.actionsRequest('GET', SCALE_SET_ENDPOINT, { + expectedStatuses: [200], + query: { runnerGroupId, name: runnerScaleSetName }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + + if (list.count === 0) { + return null; + } + if (list.count !== 1) { + throw new ScaleSetProtocolError( + `multiple runner scale sets found with name ${JSON.stringify(runnerScaleSetName)}`, + ); + } + if (list.value.length === 0) { + throw new ScaleSetProtocolError('runner scale set response count was 1 but value was empty'); + } + return normalizeRunnerScaleSet(list.value[0]); + } + + async listRunnerScaleSets(runnerGroupId: number, options: ScaleSetRequestOptions = {}): Promise { + const { result, url } = await this.actionsRequest('GET', SCALE_SET_ENDPOINT, { + expectedStatuses: [200], + query: { runnerGroupId }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + return list.value.map((scaleSet) => normalizeRunnerScaleSet(scaleSet) as RunnerScaleSet); + } + + async getRunnerScaleSetById( + runnerScaleSetId: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${runnerScaleSetId}`; + const { result, url } = await this.actionsRequest('GET', path, { + expectedStatuses: [200], + signal: options.signal, + }); + return normalizeRunnerScaleSet(parseJsonResponse(result, 'GET', url)); + } + + async getRunnerGroupByName(runnerGroupName: string, options: ScaleSetRequestOptions = {}): Promise { + const { result, url } = await this.actionsRequest('GET', `/${RUNNER_GROUP_ENDPOINT}`, { + expectedStatuses: [200], + query: { groupName: runnerGroupName }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + + if (list.count === 0) { + throw new ScaleSetProtocolError(`no runner group found with name ${JSON.stringify(runnerGroupName)}`); + } + if (list.count !== 1) { + throw new ScaleSetProtocolError(`multiple runner groups found with name ${JSON.stringify(runnerGroupName)}`); + } + if (list.value.length === 0) { + throw new ScaleSetProtocolError('runner group response count was 1 but value was empty'); + } + return list.value[0]; + } + + async createRunnerScaleSet(scaleSet: RunnerScaleSet, options: ScaleSetRequestOptions = {}): Promise { + ensureLabels(scaleSet); + applyDefaultLabelTypes(scaleSet); + const { result, url } = await this.actionsRequest('POST', SCALE_SET_ENDPOINT, { + body: runnerScaleSetRequestBody(scaleSet), + expectedStatuses: [200], + signal: options.signal, + }); + return normalizeRunnerScaleSet(parseJsonResponse(result, 'POST', url)) as RunnerScaleSet; + } + + async updateRunnerScaleSet( + runnerScaleSetId: number, + scaleSet: RunnerScaleSet, + options: ScaleSetRequestOptions = {}, + ): Promise { + applyDefaultLabelTypes(scaleSet); + const path = `${SCALE_SET_ENDPOINT}/${runnerScaleSetId}`; + const { result, url } = await this.actionsRequest('PATCH', path, { + body: runnerScaleSetRequestBody(scaleSet), + expectedStatuses: [200], + signal: options.signal, + }); + return normalizeRunnerScaleSet(parseJsonResponse(result, 'PATCH', url)) as RunnerScaleSet; + } + + async deleteRunnerScaleSet(runnerScaleSetId: number, options: ScaleSetRequestOptions = {}): Promise { + await this.actionsRequest('DELETE', `/${SCALE_SET_ENDPOINT}/${runnerScaleSetId}`, { + expectedStatuses: [204], + signal: options.signal, + }); + } + + async generateJitRunnerConfig( + setting: RunnerScaleSetJitRunnerSetting, + runnerScaleSetId: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${runnerScaleSetId}/generatejitconfig`; + const { result, url } = await this.actionsRequest('POST', path, { + body: setting, + expectedStatuses: [200], + signal: options.signal, + }); + return parseJsonResponse(result, 'POST', url); + } + + async getRunner(runnerId: number, options: ScaleSetRequestOptions = {}): Promise { + const path = `/${RUNNER_ENDPOINT}/${runnerId}`; + const { result, url } = await this.actionsRequest('GET', path, { + expectedStatuses: [200], + signal: options.signal, + }); + return parseJsonResponse(result, 'GET', url); + } + + async getRunnerByName(runnerName: string, options: ScaleSetRequestOptions = {}): Promise { + const { result, url } = await this.actionsRequest('GET', RUNNER_ENDPOINT, { + expectedStatuses: [200], + query: { agentName: runnerName }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + + if (list.count === 0) { + return null; + } + if (list.count !== 1) { + throw new ScaleSetProtocolError(`multiple runners found with name ${JSON.stringify(runnerName)}`); + } + if (list.value.length === 0) { + throw new ScaleSetProtocolError('runner response count was 1 but value was empty'); + } + return list.value[0]; + } + + async removeRunner(runnerId: number, options: ScaleSetRequestOptions = {}): Promise { + await this.actionsRequest('DELETE', `/${RUNNER_ENDPOINT}/${runnerId}`, { + expectedStatuses: [204], + signal: options.signal, + }); + } + + async createMessageSessionClient( + runnerScaleSetId: number, + owner: string, + options: ScaleSetRequestOptions = {}, + ): Promise { + return MessageSessionClient.create({ + runnerScaleSetId, + owner, + fetchImplementation: this.fetchImplementation, + userAgent: () => this.currentUserAgent, + actionsRequest: (method, path, requestOptions) => this.actionsRequest(method, path, requestOptions), + signal: options.signal, + }); + } + + /** Alias matching the upstream Go client's factory name. */ + async messageSessionClient( + runnerScaleSetId: number, + owner: string, + options: ScaleSetRequestOptions = {}, + ): Promise { + return this.createMessageSessionClient(runnerScaleSetId, owner, options); + } + + private async actionsRequest( + method: string, + path: string, + options: ActionsRequestOptions, + ): Promise<{ result: HttpResult; url: URL }> { + const adminToken = await this.getAdminToken(options.signal); + const url = actionsServiceUrl(adminToken.url, path, options.query); + const headers: Record = { + 'Content-Type': 'application/json', + Authorization: options.authorization ?? `Bearer ${adminToken.token}`, + 'User-Agent': this.currentUserAgent, + }; + const body = options.body === undefined ? undefined : JSON.stringify(options.body); + const result = await executeRequest( + this.fetchImplementation, + url, + { + method, + headers, + body, + signal: options.signal, + }, + options.expectedStatuses, + ); + return { result, url }; + } + + private async getAdminToken(signal?: AbortSignal): Promise { + if (this.adminTokenIsUsable(this.adminToken)) { + return this.adminToken; + } + + if (this.adminTokenRefresh === undefined) { + this.adminTokenRefresh = this.refreshAdminToken(signal).finally(() => { + this.adminTokenRefresh = undefined; + }); + } + return this.adminTokenRefresh; + } + + private adminTokenIsUsable(token?: ActionsServiceAdminToken): token is ActionsServiceAdminToken { + return token !== undefined && this.now().getTime() + ADMIN_TOKEN_REFRESH_SKEW_MS < token.expiresAt.getTime(); + } + + private async refreshAdminToken(signal?: AbortSignal): Promise { + const registrationToken = await this.getRunnerRegistrationToken(signal); + const adminConnection = await this.getActionsServiceAdminConnection(registrationToken, signal); + const refreshedToken: ActionsServiceAdminToken = { + token: adminConnection.token, + expiresAt: parseJwtExpiration(adminConnection.token), + url: adminConnection.url, + }; + this.adminToken = refreshedToken; + return refreshedToken; + } + + private async getRunnerRegistrationToken(signal?: AbortSignal): Promise { + const accessToken = await this.getAccessToken(); + const url = githubApiUrl(this.config, runnerRegistrationTokenPath(this.config)); + const result = await executeRequest( + this.fetchImplementation, + url, + { + method: 'POST', + headers: { + 'Content-Type': 'application/vnd.github.v3+json', + Authorization: `Bearer ${accessToken}`, + 'User-Agent': this.currentUserAgent, + }, + body: '', + signal, + }, + [201], + ); + const response = parseJsonResponse(result, 'POST', url); + if (!response.token) { + throw new ScaleSetProtocolError('runner registration token response is missing token'); + } + return response.token; + } + + private async getAccessToken(): Promise { + const providedToken = await this.accessTokenProvider(); + const accessToken = typeof providedToken === 'string' ? providedToken : providedToken.token; + if (accessToken === '') throw new ScaleSetProtocolError('access token provider returned an empty token'); + return accessToken; + } + + private async getActionsServiceAdminConnection( + registrationToken: string, + signal?: AbortSignal, + ): Promise<{ url: string; token: string }> { + const url = githubApiUrl(this.config, '/actions/runner-registration'); + const result = await executeRequest( + this.adminConnectionFetchImplementation, + url, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `RemoteAuth ${registrationToken}`, + 'User-Agent': this.currentUserAgent, + }, + body: JSON.stringify({ + url: this.config.configUrl.toString(), + runner_event: 'register', + }), + signal, + }, + SUCCESS_STATUSES, + ); + const response = parseJsonResponse(result, 'POST', url); + if (!response.url) { + throw new ScaleSetProtocolError('Actions service admin connection is missing url'); + } + if (!response.token) { + throw new ScaleSetProtocolError('Actions service admin connection is missing token'); + } + let actionsServiceUrl: URL; + try { + actionsServiceUrl = new URL(response.url); + } catch (error) { + throw new ScaleSetProtocolError('Actions service admin connection contains an invalid url', { cause: error }); + } + if ( + actionsServiceUrl.protocol !== 'https:' || + actionsServiceUrl.username || + actionsServiceUrl.password || + actionsServiceUrl.search || + actionsServiceUrl.hash + ) { + throw new ScaleSetProtocolError( + 'Actions service admin connection url must be HTTPS and contain no credentials, query, or fragment', + ); + } + return { url: trimTrailingSlashes(actionsServiceUrl.toString()), token: response.token }; + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/config.test.ts b/lambdas/libs/github-actions-scale-set/src/config.test.ts new file mode 100644 index 0000000000..316fbab015 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/config.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest'; + +import { InvalidGitHubConfigUrlError, parseGitHubConfigUrl } from './config'; + +describe('GitHub configuration URL parsing', () => { + it('requires HTTPS for GitHub.com and GHES configuration URLs', () => { + expect(() => parseGitHubConfigUrl('http://github.com/example')).toThrow(InvalidGitHubConfigUrlError); + expect(() => parseGitHubConfigUrl('http://github.example.com/example', true)).toThrow(/should be HTTPS/); + }); + + it('continues to accept an HTTPS GitHub configuration URL', () => { + expect(parseGitHubConfigUrl('https://github.com/example')).toMatchObject({ + scope: 'organization', + organization: 'example', + isHosted: true, + }); + }); + + it('normalizes long slash sequences', () => { + const slashSequence = '/'.repeat(10_000); + + expect(parseGitHubConfigUrl(`https://github.com${slashSequence}example${slashSequence}`)).toMatchObject({ + configUrl: new URL('https://github.com/example'), + scope: 'organization', + organization: 'example', + isHosted: true, + }); + }); + + it.each(['https://github.com////', 'https://github.com/org//repository', 'https://github.com/org/repository/extra'])( + 'continues to reject an invalid path after slash normalization: %s', + (configUrl) => { + expect(() => parseGitHubConfigUrl(configUrl)).toThrow(InvalidGitHubConfigUrlError); + }, + ); + + it.each([ + ['https://github.com/org/repository////', 'repository'], + ['https://github.com/enterprises/example////', 'enterprise'], + ])('retains the scope when normalizing %s', (configUrl, scope) => { + expect(parseGitHubConfigUrl(configUrl)).toMatchObject({ scope }); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/config.ts b/lambdas/libs/github-actions-scale-set/src/config.ts new file mode 100644 index 0000000000..e9b5e1a4a3 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/config.ts @@ -0,0 +1,119 @@ +import { trimSurroundingSlashes, trimTrailingSlashes } from './url'; + +export const GITHUB_SCOPES = { + enterprise: 'enterprise', + organization: 'organization', + repository: 'repository', +} as const; + +export type GitHubScope = (typeof GITHUB_SCOPES)[keyof typeof GITHUB_SCOPES]; + +export interface ParsedGitHubConfig { + configUrl: URL; + scope: GitHubScope; + enterprise?: string; + organization?: string; + repository?: string; + isHosted: boolean; +} + +export class InvalidGitHubConfigUrlError extends Error { + constructor(configUrl: string, options?: ErrorOptions) { + super( + `${JSON.stringify(configUrl)}: invalid config URL, should be HTTPS and point to an enterprise, org, or repository`, + options, + ); + this.name = 'InvalidGitHubConfigUrlError'; + } +} + +function environmentForcesGhes(): boolean { + return ( + typeof process !== 'undefined' && Object.prototype.hasOwnProperty.call(process.env, 'GITHUB_ACTIONS_FORCE_GHES') + ); +} + +function isHostedGitHubUrl(configUrl: URL, forceGhes?: boolean): boolean { + if (forceGhes ?? environmentForcesGhes()) { + return false; + } + + const host = configUrl.host.toLowerCase(); + return host === 'github.com' || host === 'www.github.com' || host === 'github.localhost' || host.endsWith('.ghe.com'); +} + +/** Parse a repository, organization, or enterprise registration URL. */ +export function parseGitHubConfigUrl(configUrl: string, forceGhes?: boolean): ParsedGitHubConfig { + let parsedUrl: URL; + try { + parsedUrl = new URL(trimTrailingSlashes(configUrl.trim())); + } catch (error) { + throw new InvalidGitHubConfigUrlError(configUrl, { cause: error }); + } + + if (parsedUrl.protocol !== 'https:') { + throw new InvalidGitHubConfigUrlError(configUrl); + } + + const pathParts = trimSurroundingSlashes(parsedUrl.pathname).split('/'); + const isHosted = isHostedGitHubUrl(parsedUrl, forceGhes); + + if (pathParts.length === 1 && pathParts[0] !== '') { + parsedUrl.pathname = `/${pathParts[0]}`; + return { + configUrl: parsedUrl, + scope: GITHUB_SCOPES.organization, + organization: pathParts[0], + isHosted, + }; + } + + if (pathParts.length === 2 && pathParts.every((part) => part !== '')) { + parsedUrl.pathname = `/${pathParts.join('/')}`; + if (pathParts[0].toLowerCase() === 'enterprises') { + return { + configUrl: parsedUrl, + scope: GITHUB_SCOPES.enterprise, + enterprise: pathParts[1], + isHosted, + }; + } + + return { + configUrl: parsedUrl, + scope: GITHUB_SCOPES.repository, + organization: pathParts[0], + repository: pathParts[1], + isHosted, + }; + } + + throw new InvalidGitHubConfigUrlError(configUrl); +} + +/** Build a GitHub REST API URL for GitHub.com, ghe.com, or GHES. */ +export function githubApiUrl(config: ParsedGitHubConfig, path: string): URL { + const result = new URL(config.configUrl.origin); + const normalizedPath = path.startsWith('/') ? path : `/${path}`; + + if (config.isHosted) { + result.host = + config.configUrl.host.toLowerCase() === 'www.github.com' ? 'api.github.com' : `api.${config.configUrl.host}`; + result.pathname = normalizedPath; + return result; + } + + result.pathname = `/api/v3${normalizedPath}`; + return result; +} + +export function runnerRegistrationTokenPath(config: ParsedGitHubConfig): string { + switch (config.scope) { + case GITHUB_SCOPES.organization: + return `/orgs/${config.organization}/actions/runners/registration-token`; + case GITHUB_SCOPES.enterprise: + return `/enterprises/${config.enterprise}/actions/runners/registration-token`; + case GITHUB_SCOPES.repository: + return `/repos/${config.organization}/${config.repository}/actions/runners/registration-token`; + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/endpoints.ts b/lambdas/libs/github-actions-scale-set/src/endpoints.ts new file mode 100644 index 0000000000..d409f73704 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/endpoints.ts @@ -0,0 +1,4 @@ +export const RUNNER_ENDPOINT = '_apis/distributedtask/pools/0/agents'; +export const SCALE_SET_ENDPOINT = '_apis/runtime/runnerscalesets'; +export const RUNNER_GROUP_ENDPOINT = '_apis/runtime/runnergroups/'; +export const ACTIONS_API_VERSION = '6.0-preview'; diff --git a/lambdas/libs/github-actions-scale-set/src/errors.ts b/lambdas/libs/github-actions-scale-set/src/errors.ts new file mode 100644 index 0000000000..860e4aaa7d --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/errors.ts @@ -0,0 +1,171 @@ +export const SCALE_SET_ERROR_CODES = { + badRequest: 'BAD_REQUEST', + conflict: 'CONFLICT', + jobStillRunning: 'JOB_STILL_RUNNING', + messageQueueTokenExpired: 'MESSAGE_QUEUE_TOKEN_EXPIRED', + notFound: 'NOT_FOUND', + runnerExists: 'RUNNER_EXISTS', + runnerNotFound: 'RUNNER_NOT_FOUND', + unauthorized: 'UNAUTHORIZED', + unexpectedStatus: 'UNEXPECTED_STATUS', +} as const; + +export type ScaleSetErrorCode = (typeof SCALE_SET_ERROR_CODES)[keyof typeof SCALE_SET_ERROR_CODES]; + +export interface ScaleSetHttpErrorDetails { + method: string; + url: string; + status: number; + statusText: string; + headers: Headers; + responseBody: string; + code?: ScaleSetErrorCode; + cause?: unknown; +} + +interface ActionsException { + typeName?: unknown; + message?: unknown; +} + +export function redactUrlForError(value: string): string { + try { + const url = new URL(value); + url.username = ''; + url.password = ''; + url.search = ''; + url.hash = ''; + return url.toString(); + } catch { + return ''; + } +} + +function statusErrorCode(status: number): ScaleSetErrorCode { + switch (status) { + case 400: + return SCALE_SET_ERROR_CODES.badRequest; + case 401: + return SCALE_SET_ERROR_CODES.unauthorized; + case 404: + return SCALE_SET_ERROR_CODES.notFound; + case 409: + return SCALE_SET_ERROR_CODES.conflict; + default: + return SCALE_SET_ERROR_CODES.unexpectedStatus; + } +} + +function exceptionErrorCode(typeName?: string): ScaleSetErrorCode | undefined { + if (typeName?.includes('AgentExistsException')) { + return SCALE_SET_ERROR_CODES.runnerExists; + } + if (typeName?.includes('AgentNotFoundException')) { + return SCALE_SET_ERROR_CODES.runnerNotFound; + } + if (typeName?.includes('JobStillRunningException')) { + return SCALE_SET_ERROR_CODES.jobStillRunning; + } + return undefined; +} + +function parseActionsException(responseBody: string): { typeName?: string; message?: string } { + if (responseBody === '') { + return {}; + } + + try { + const parsed = JSON.parse(responseBody) as ActionsException; + return { + typeName: typeof parsed.typeName === 'string' ? parsed.typeName : undefined, + message: typeof parsed.message === 'string' ? parsed.message : undefined, + }; + } catch { + return {}; + } +} + +/** An unsuccessful HTTP response from either GitHub or the Actions service. */ +export class ScaleSetHttpError extends Error { + readonly code: ScaleSetErrorCode; + readonly status: number; + readonly statusText: string; + readonly method: string; + readonly url: string; + readonly activityId?: string; + readonly githubRequestId?: string; + readonly exceptionName?: string; + readonly responseBody: string; + + constructor(details: ScaleSetHttpErrorDetails) { + const safeUrl = redactUrlForError(details.url); + const exception = parseActionsException(details.responseBody); + const activityId = details.headers.get('ActivityId') ?? undefined; + const githubRequestId = details.headers.get('X-GitHub-Request-Id') ?? undefined; + const responseDescription = [details.status, details.statusText].filter(Boolean).join(' '); + const metadata = [ + `status=${JSON.stringify(responseDescription)}`, + activityId ? `activity_id=${JSON.stringify(activityId)}` : undefined, + githubRequestId ? `github_request_id=${JSON.stringify(githubRequestId)}` : undefined, + ] + .filter((part): part is string => part !== undefined) + .join(', '); + const responseMessage = exception.message ?? (details.responseBody || 'unknown error'); + const exceptionPrefix = exception.typeName ? `${exception.typeName}: ` : ''; + + super(`request ${details.method} ${safeUrl} failed (${metadata}): ${exceptionPrefix}${responseMessage}`, { + cause: details.cause, + }); + this.name = 'ScaleSetHttpError'; + this.code = details.code ?? exceptionErrorCode(exception.typeName) ?? statusErrorCode(details.status); + this.status = details.status; + this.statusText = details.statusText; + this.method = details.method; + this.url = safeUrl; + this.activityId = activityId; + this.githubRequestId = githubRequestId; + this.exceptionName = exception.typeName; + this.responseBody = details.responseBody; + } +} + +export class ScaleSetRequestError extends Error { + readonly method: string; + readonly url: string; + readonly attempts: number; + + constructor(method: string, url: string, cause: unknown, attempts = 1) { + const safeUrl = redactUrlForError(url); + super(`request ${method} ${safeUrl} failed before receiving a response after ${attempts} attempt(s)`, { cause }); + this.name = 'ScaleSetRequestError'; + this.method = method; + this.url = safeUrl; + this.attempts = attempts; + } +} + +export class ScaleSetRequestTimeoutError extends ScaleSetRequestError { + readonly timeoutMs: number; + + constructor(method: string, url: string, timeoutMs: number, attempts: number) { + super(method, url, new Error(`request attempt exceeded ${timeoutMs}ms`), attempts); + this.name = 'ScaleSetRequestTimeoutError'; + this.timeoutMs = timeoutMs; + } +} + +export class ScaleSetProtocolError extends Error { + readonly method?: string; + readonly url?: string; + + constructor(message: string, options: { method?: string; url?: string; cause?: unknown } = {}) { + super(message, { cause: options.cause }); + this.name = 'ScaleSetProtocolError'; + this.method = options.method; + this.url = options.url; + } +} + +export function isScaleSetHttpError(error: unknown): error is ScaleSetHttpError { + return error instanceof ScaleSetHttpError; +} diff --git a/lambdas/libs/github-actions-scale-set/src/http.test.ts b/lambdas/libs/github-actions-scale-set/src/http.test.ts new file mode 100644 index 0000000000..76f506d13f --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/http.test.ts @@ -0,0 +1,238 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { ScaleSetRequestError, ScaleSetRequestTimeoutError } from './errors'; +import { + createRetryingFetch, + DEFAULT_SCALE_SET_RETRY_OPTIONS, + executeRequest, + resolveScaleSetRetryOptions, +} from './http'; +import { ScaleSetFetch } from './types'; + +function okResponse(): Response { + return new Response('{"ok":true}', { status: 200 }); +} + +describe('retrying fetch', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it('uses bounded defaults close to the upstream client and validates overrides', () => { + expect(resolveScaleSetRetryOptions()).toEqual({ + maxRetries: 4, + initialBackoffMs: 1_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 300_000, + }); + expect(DEFAULT_SCALE_SET_RETRY_OPTIONS).toEqual(resolveScaleSetRetryOptions()); + expect(() => resolveScaleSetRetryOptions({ maxRetries: -1 })).toThrow(/retry\.maxRetries/); + expect(() => resolveScaleSetRetryOptions({ maxRetries: 1.5 })).toThrow(/integer/); + expect(() => resolveScaleSetRetryOptions({ requestTimeoutMs: 0 })).toThrow(/requestTimeoutMs/); + }); + + it('retries a network error after deterministic exponential backoff', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi + .fn() + .mockRejectedValueOnce(new TypeError('socket closed')) + .mockResolvedValueOnce(okResponse()); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 25, + maxBackoffMs: 100, + requestTimeoutMs: 1_000, + }); + + const request = fetchWithRetry('https://actions.example/test'); + await vi.advanceTimersByTimeAsync(24); + expect(underlyingFetch).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); + + await expect(request).resolves.toMatchObject({ status: 200 }); + expect(underlyingFetch).toHaveBeenCalledTimes(2); + }); + + it('honors Retry-After for 429 responses and caps the wait at maxBackoffMs', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi + .fn() + .mockResolvedValueOnce( + new Response('{"message":"slow down"}', { + status: 429, + headers: { 'Retry-After': '120' }, + }), + ) + .mockResolvedValueOnce(okResponse()); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 10, + maxBackoffMs: 30_000, + requestTimeoutMs: 1_000, + }); + + const request = fetchWithRetry('https://actions.example/test'); + await vi.advanceTimersByTimeAsync(29_999); + expect(underlyingFetch).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); + + await expect(request).resolves.toMatchObject({ status: 200 }); + expect(underlyingFetch).toHaveBeenCalledTimes(2); + }); + + it('retries 5xx responses only up to maxRetries and returns the final response', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn( + async () => + new Response('{"message":"unavailable"}', { + status: 503, + headers: { 'Retry-After': 'invalid' }, + }), + ); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 2, + initialBackoffMs: 10, + maxBackoffMs: 15, + requestTimeoutMs: 1_000, + }); + + const request = fetchWithRetry('https://actions.example/test'); + await vi.runAllTimersAsync(); + + await expect(request).resolves.toMatchObject({ status: 503 }); + expect(underlyingFetch).toHaveBeenCalledTimes(3); + }); + + it('does not retry a queue 401 so message-session refresh remains the owner', async () => { + const underlyingFetch = vi.fn(async () => new Response(null, { status: 401 })); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + await expect(fetchWithRetry('https://queue.example/messages')).resolves.toMatchObject({ status: 401 }); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('does not replay a POST after a retryable HTTP response', async () => { + const underlyingFetch = vi.fn(async () => new Response(null, { status: 503 })); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + await expect( + fetchWithRetry('https://actions.example/generatejitconfig', { method: 'POST' }), + ).resolves.toMatchObject({ status: 503 }); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('does not replay a POST carried by a Request after a network failure', async () => { + const underlyingFetch = vi.fn(async () => { + throw new TypeError('network unavailable'); + }); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + await expect( + fetchWithRetry(new Request('https://actions.example/sessions', { method: 'POST' })), + ).rejects.toMatchObject({ + name: 'ScaleSetRequestError', + method: 'POST', + attempts: 1, + } satisfies Partial); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('times out each attempt and stops after the configured retry bound', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn(() => new Promise(() => undefined)); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 10, + maxBackoffMs: 10, + requestTimeoutMs: 50, + }); + + const request = fetchWithRetry('https://actions.example/hangs'); + const rejection = expect(request).rejects.toMatchObject({ + name: 'ScaleSetRequestTimeoutError', + attempts: 2, + timeoutMs: 50, + } satisfies Partial); + await vi.advanceTimersByTimeAsync(50); + expect(underlyingFetch).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(10); + expect(underlyingFetch).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(50); + + await rejection; + }); + + it('interrupts retry backoff immediately when the caller aborts', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn(async () => new Response(null, { status: 503 })); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 30_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 60_000, + }); + const controller = new AbortController(); + const request = fetchWithRetry('https://actions.example/test', { signal: controller.signal }); + await vi.advanceTimersByTimeAsync(0); + + controller.abort(new Error('caller cancelled')); + + await expect(request).rejects.toThrow('caller cancelled'); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('wraps an exhausted idempotent network failure with the final attempt count', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn(async () => { + throw new TypeError('network unavailable'); + }); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + const request = fetchWithRetry('https://actions.example/test', { method: 'GET' }); + const rejection = expect(request).rejects.toMatchObject({ + name: 'ScaleSetRequestError', + method: 'GET', + attempts: 2, + } satisfies Partial); + await vi.runAllTimersAsync(); + + await rejection; + }); + + it('redacts signed query strings from request errors', async () => { + const fetchImplementation = vi.fn(async () => { + throw new TypeError('network unavailable'); + }); + const request = executeRequest( + fetchImplementation, + 'https://queue.example/messages?signature=do-not-log&token=also-secret', + { method: 'GET' }, + [200], + ); + + await expect(request).rejects.toMatchObject({ + url: 'https://queue.example/messages', + message: expect.not.stringContaining('do-not-log'), + } satisfies Partial); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/http.ts b/lambdas/libs/github-actions-scale-set/src/http.ts new file mode 100644 index 0000000000..ef5f47bab9 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/http.ts @@ -0,0 +1,327 @@ +import { + ScaleSetErrorCode, + ScaleSetHttpError, + ScaleSetProtocolError, + ScaleSetRequestError, + ScaleSetRequestTimeoutError, + redactUrlForError, +} from './errors'; +import { ScaleSetFetch, ScaleSetRetryOptions } from './types'; + +export interface ResolvedScaleSetRetryOptions { + maxRetries: number; + initialBackoffMs: number; + maxBackoffMs: number; + requestTimeoutMs: number; +} + +export const DEFAULT_SCALE_SET_RETRY_OPTIONS: Readonly = Object.freeze({ + maxRetries: 4, + initialBackoffMs: 1_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 5 * 60_000, +}); + +export interface HttpResult { + response: Response; + body: string; +} + +interface RetryingFetchPolicy { + additionalRetryStatuses?: readonly number[]; + /** + * Escape hatch for a known-safe, operation-scoped wrapper. Non-idempotent + * methods are never retried by the default transport policy. + */ + additionalRetryMethods?: readonly string[]; +} + +const MAX_RESPONSE_BODY_BYTES = 1024 * 1024; +const IDEMPOTENT_RETRY_METHODS = new Set(['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE']); + +function trimByteOrderMark(body: string): string { + return body.startsWith('\uFEFF') ? body.slice(1) : body; +} + +function boundedNumber(name: string, value: number, minimum: number, integer: boolean): number { + if (!Number.isFinite(value) || value < minimum || (integer && !Number.isInteger(value))) { + throw new TypeError(`${name} must be ${integer ? 'an integer' : 'a number'} greater than or equal to ${minimum}`); + } + return value; +} + +export function resolveScaleSetRetryOptions(options: ScaleSetRetryOptions = {}): ResolvedScaleSetRetryOptions { + return { + maxRetries: boundedNumber( + 'retry.maxRetries', + options.maxRetries ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.maxRetries, + 0, + true, + ), + initialBackoffMs: boundedNumber( + 'retry.initialBackoffMs', + options.initialBackoffMs ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.initialBackoffMs, + 0, + false, + ), + maxBackoffMs: boundedNumber( + 'retry.maxBackoffMs', + options.maxBackoffMs ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.maxBackoffMs, + 0, + false, + ), + requestTimeoutMs: boundedNumber( + 'retry.requestTimeoutMs', + options.requestTimeoutMs ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.requestTimeoutMs, + 1, + false, + ), + }; +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException('The operation was aborted', 'AbortError'); +} + +function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { + if (signal?.aborted) { + return Promise.reject(abortReason(signal)); + } + if (delayMs === 0) { + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, delayMs); + const onAbort = () => { + clearTimeout(timeout); + reject(abortReason(signal as AbortSignal)); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + +function retryableStatus(status: number, additionalRetryStatuses: ReadonlySet): boolean { + return status === 429 || (status >= 500 && status <= 599) || additionalRetryStatuses.has(status); +} + +function requestMethod(input: RequestInput, init: RequestInit): string { + return (init.method ?? (input instanceof Request ? input.method : 'GET')).toUpperCase(); +} + +function exponentialBackoffMs(retryIndex: number, options: ResolvedScaleSetRetryOptions): number { + return Math.min(options.initialBackoffMs * 2 ** retryIndex, options.maxBackoffMs); +} + +function retryAfterMs(response: Response, options: ResolvedScaleSetRetryOptions): number | undefined { + const value = response.headers.get('Retry-After')?.trim(); + if (!value) { + return undefined; + } + + let delayMs: number; + if (/^\d+$/.test(value)) { + delayMs = Number(value) * 1_000; + } else { + const retryAt = Date.parse(value); + if (!Number.isFinite(retryAt)) { + return undefined; + } + delayMs = Math.max(0, retryAt - Date.now()); + } + return Math.min(delayMs, options.maxBackoffMs); +} + +async function fetchAttempt( + fetchImplementation: ScaleSetFetch, + input: RequestInput, + init: RequestInit, + options: ResolvedScaleSetRetryOptions, + attempt: number, +): Promise { + const method = requestMethod(input, init); + const url = input instanceof Request ? input.url : input.toString(); + const callerSignal = init.signal ?? undefined; + if (callerSignal?.aborted) { + throw abortReason(callerSignal); + } + + const attemptController = new AbortController(); + const forwardAbort = () => attemptController.abort(abortReason(callerSignal as AbortSignal)); + callerSignal?.addEventListener('abort', forwardAbort, { once: true }); + const timeoutError = new ScaleSetRequestTimeoutError(method, url, options.requestTimeoutMs, attempt); + const timeout = setTimeout(() => attemptController.abort(timeoutError), options.requestTimeoutMs); + + let onAttemptAbort: (() => void) | undefined; + const aborted = new Promise((_, reject) => { + onAttemptAbort = () => reject(abortReason(attemptController.signal)); + attemptController.signal.addEventListener('abort', onAttemptAbort, { once: true }); + }); + + try { + return await Promise.race([fetchImplementation(input, { ...init, signal: attemptController.signal }), aborted]); + } finally { + clearTimeout(timeout); + callerSignal?.removeEventListener('abort', forwardAbort); + if (onAttemptAbort !== undefined) { + attemptController.signal.removeEventListener('abort', onAttemptAbort); + } + } +} + +type RequestInput = Parameters[0]; + +/** Wrap native fetch with the bounded retry and timeout policy used by every SDK request. */ +export function createRetryingFetch( + fetchImplementation: ScaleSetFetch, + retryOptions: ScaleSetRetryOptions = {}, + policy: RetryingFetchPolicy = {}, +): ScaleSetFetch { + const options = resolveScaleSetRetryOptions(retryOptions); + const additionalRetryStatusSet = new Set(policy.additionalRetryStatuses ?? []); + const additionalRetryMethodSet = new Set((policy.additionalRetryMethods ?? []).map((method) => method.toUpperCase())); + + return async (input, init = {}) => { + const method = requestMethod(input, init); + const url = input instanceof Request ? input.url : input.toString(); + const maxRetries = + IDEMPOTENT_RETRY_METHODS.has(method) || additionalRetryMethodSet.has(method) ? options.maxRetries : 0; + + for (let retryIndex = 0; retryIndex <= maxRetries; retryIndex += 1) { + const attempt = retryIndex + 1; + try { + const response = await fetchAttempt(fetchImplementation, input, init, options, attempt); + if (!retryableStatus(response.status, additionalRetryStatusSet) || retryIndex === maxRetries) { + return response; + } + + const delayMs = retryAfterMs(response, options) ?? exponentialBackoffMs(retryIndex, options); + await response.body?.cancel().catch(() => undefined); + await waitForRetry(delayMs, init.signal ?? undefined); + } catch (error) { + if (init.signal?.aborted) { + throw abortReason(init.signal); + } + if (retryIndex === maxRetries) { + if (error instanceof ScaleSetRequestTimeoutError) { + throw error; + } + throw new ScaleSetRequestError(method, url, error, attempt); + } + await waitForRetry(exponentialBackoffMs(retryIndex, options), init.signal ?? undefined); + } + } + + throw new ScaleSetRequestError(method, url, new Error('retry loop exhausted'), maxRetries + 1); + }; +} + +export async function executeRequest( + fetchImplementation: ScaleSetFetch, + url: string | URL, + init: RequestInit, + expectedStatuses: readonly number[], + errorCode?: ScaleSetErrorCode | ((response: Response) => ScaleSetErrorCode | undefined), +): Promise { + const method = init.method ?? 'GET'; + const urlString = url.toString(); + const displayUrl = redactUrlForError(urlString); + let response: Response; + + try { + response = await fetchImplementation(url, { ...init, redirect: 'error' }); + } catch (error) { + if (init.signal?.aborted || (error instanceof Error && error.name === 'AbortError')) { + throw error; + } + if (error instanceof ScaleSetRequestError) { + throw error; + } + throw new ScaleSetRequestError(method, urlString, error); + } + + let body: string; + try { + const contentLength = response.headers.get('content-length'); + if (contentLength !== null && /^\d+$/.test(contentLength) && Number(contentLength) > MAX_RESPONSE_BODY_BYTES) { + throw new ScaleSetProtocolError( + `response body from ${method} ${displayUrl} exceeds ${MAX_RESPONSE_BODY_BYTES} bytes`, + { + method, + url: displayUrl, + }, + ); + } + if (response.body === null) { + body = ''; + } else { + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > MAX_RESPONSE_BODY_BYTES) { + await reader.cancel().catch(() => undefined); + throw new ScaleSetProtocolError( + `response body from ${method} ${displayUrl} exceeds ${MAX_RESPONSE_BODY_BYTES} bytes`, + { method, url: displayUrl }, + ); + } + chunks.push(value); + } + const bytes = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + body = trimByteOrderMark(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + } + } catch (error) { + if (error instanceof ScaleSetProtocolError) throw error; + throw new ScaleSetProtocolError(`failed to read the response body from ${method} ${displayUrl}`, { + method, + url: displayUrl, + cause: error, + }); + } + + if (!expectedStatuses.includes(response.status)) { + throw new ScaleSetHttpError({ + method, + url: displayUrl, + status: response.status, + statusText: response.statusText, + headers: response.headers, + responseBody: body, + code: typeof errorCode === 'function' ? errorCode(response) : errorCode, + }); + } + + return { response, body }; +} + +export function parseJsonResponse(result: HttpResult, method: string, url: string | URL): T { + const urlString = redactUrlForError(url.toString()); + if (result.body === '') { + throw new ScaleSetProtocolError(`empty JSON response from ${method} ${urlString}`, { + method, + url: urlString, + }); + } + + try { + return JSON.parse(result.body) as T; + } catch (error) { + throw new ScaleSetProtocolError(`invalid JSON response from ${method} ${urlString}`, { + method, + url: urlString, + cause: error, + }); + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/index.ts b/lambdas/libs/github-actions-scale-set/src/index.ts new file mode 100644 index 0000000000..ab0aad76db --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/index.ts @@ -0,0 +1,29 @@ +export { + actionsServiceUrl, + GitHubActionsScaleSetClient as Client, + GitHubActionsScaleSetClient, + GitHubActionsScaleSetClient as ScaleSetClient, +} from './client'; +export { ACTIONS_API_VERSION, RUNNER_ENDPOINT, RUNNER_GROUP_ENDPOINT, SCALE_SET_ENDPOINT } from './endpoints'; +export { + GITHUB_SCOPES, + githubApiUrl, + InvalidGitHubConfigUrlError, + parseGitHubConfigUrl, + runnerRegistrationTokenPath, +} from './config'; +export type { GitHubScope, ParsedGitHubConfig } from './config'; +export { + isScaleSetHttpError, + redactUrlForError, + SCALE_SET_ERROR_CODES, + ScaleSetHttpError, + ScaleSetProtocolError, + ScaleSetRequestError, + ScaleSetRequestTimeoutError, +} from './errors'; +export type { ScaleSetErrorCode, ScaleSetHttpErrorDetails } from './errors'; +export { DEFAULT_SCALE_SET_RETRY_OPTIONS } from './http'; +export type { ResolvedScaleSetRetryOptions } from './http'; +export { HEADER_SCALE_SET_MAX_CAPACITY, MessageSessionClient } from './message-session-client'; +export * from './types'; diff --git a/lambdas/libs/github-actions-scale-set/src/message-session-client.test.ts b/lambdas/libs/github-actions-scale-set/src/message-session-client.test.ts new file mode 100644 index 0000000000..ed4e7d41ae --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/message-session-client.test.ts @@ -0,0 +1,281 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { GitHubActionsScaleSetClient } from './client'; +import { ScaleSetProtocolError } from './errors'; +import { HEADER_SCALE_SET_MAX_CAPACITY } from './message-session-client'; +import { RunnerScaleSetStatistic, ScaleSetFetch } from './types'; + +type RequestInput = Parameters[0]; +type RequestHandler = (url: URL, init: RequestInit) => Response | Promise; + +const statistics: RunnerScaleSetStatistic = { + totalAvailableJobs: 2, + totalAcquiredJobs: 1, + totalAssignedJobs: 1, + totalRunningJobs: 1, + totalRegisteredRunners: 2, + totalBusyRunners: 1, + totalIdleRunners: 1, +}; + +function requestUrl(input: RequestInput): URL { + if (input instanceof Request) { + return new URL(input.url); + } + return new URL(input.toString()); +} + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function actionsAdminToken(): string { + const payload = Buffer.from(JSON.stringify({ exp: Math.floor(Date.now() / 1000) + 60 * 60 })).toString('base64url'); + return `header.${payload}.signature`; +} + +function sessionFixture(handler: RequestHandler) { + const requests: Array<{ url: URL; init: RequestInit }> = []; + const fetchImplementation = vi.fn(async (input, init = {}) => { + const url = requestUrl(input); + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: actionsAdminToken(), + }); + } + + requests.push({ url, init }); + return handler(url, init); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-token', + fetch: fetchImplementation, + systemInfo: { system: 'unit-test', subsystem: 'listener' }, + }); + + return { client, fetchImplementation, requests }; +} + +function sessionResponse(token = 'queue-token') { + return { + sessionId: '11111111-1111-1111-1111-111111111111', + ownerName: 'listener-1', + runnerScaleSet: { id: 42, name: 'linux' }, + messageQueueUrl: 'https://queue.example/messages?existing=1', + messageQueueAccessToken: token, + statistics, + }; +} + +describe('MessageSessionClient', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('normalizes the capitalized RunnerSetting returned when creating a session', async () => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse({ + ...sessionResponse(), + runnerScaleSet: { id: 42, name: 'linux', RunnerSetting: { disableUpdate: true } }, + }); + } + return new Response(null, { status: 500 }); + }); + + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + expect(session.session.runnerScaleSet?.runnerSetting).toEqual({ disableUpdate: true }); + expect(session.session.runnerScaleSet).not.toHaveProperty('RunnerSetting'); + }); + + it('maps a 202 poll to null and sends the queue capacity contract', async () => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse()); + } + if (url.hostname === 'queue.example' && init.method === 'GET') { + return new Response(null, { status: 202 }); + } + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + await expect(session.getMessage(0, 17)).resolves.toBeNull(); + + expect(session.session.statistics).toEqual(statistics); + const queueRequest = fixture.requests.find(({ url }) => url.hostname === 'queue.example'); + expect(queueRequest).toBeDefined(); + expect(queueRequest?.url.toString()).toBe('https://queue.example/messages?existing=1'); + const headers = new Headers(queueRequest?.init.headers); + expect(headers.get('Accept')).toBe('application/json; api-version=6.0-preview'); + expect(headers.get('Authorization')).toBe('Bearer queue-token'); + expect(headers.get(HEADER_SCALE_SET_MAX_CAPACITY)).toBe('17'); + expect(headers.get('User-Agent')).toContain('"kind":"scaleset"'); + }); + + it('decodes known batched messages, ignores unknown types, acknowledges, and acquires jobs', async () => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse()); + } + if (url.hostname === 'queue.example' && init.method === 'GET') { + return jsonResponse({ + messageId: 19, + messageType: 'RunnerScaleSetJobMessages', + statistics, + body: JSON.stringify([ + { + messageType: 'JobAvailable', + runnerRequestId: 501, + acquireJobUrl: 'https://actions.example/acquire/501', + }, + { + messageType: 'FutureMessageType', + runnerRequestId: 999, + }, + { + messageType: 'JobCompleted', + runnerRequestId: 500, + runnerId: 71, + runnerName: 'runner-71', + result: 'Succeeded', + }, + ]), + }); + } + if (url.hostname === 'queue.example' && init.method === 'DELETE') { + return new Response(null, { status: 204 }); + } + if (url.pathname.endsWith('/runnerscalesets/42/acquirejobs') && init.method === 'POST') { + return jsonResponse({ count: 1, value: [501] }); + } + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + const message = await session.getMessage(18, 4); + expect(message).toMatchObject({ messageId: 19, statistics }); + expect(message?.jobAvailableMessages).toEqual([ + expect.objectContaining({ messageType: 'JobAvailable', runnerRequestId: 501 }), + ]); + expect(message?.jobCompletedMessages).toEqual([ + expect.objectContaining({ messageType: 'JobCompleted', runnerName: 'runner-71' }), + ]); + expect(message?.jobAssignedMessages).toEqual([]); + expect(message?.jobStartedMessages).toEqual([]); + + await expect(session.deleteMessage(19)).resolves.toBeUndefined(); + await expect(session.acquireJobs([501, 999])).resolves.toEqual([501]); + + const pollRequest = fixture.requests.find( + ({ url, init }) => url.hostname === 'queue.example' && init.method === 'GET', + ); + expect(pollRequest?.url.searchParams.get('lastMessageId')).toBe('18'); + + const ackRequest = fixture.requests.find( + ({ url, init }) => url.hostname === 'queue.example' && init.method === 'DELETE', + ); + expect(ackRequest?.url.pathname).toBe('/messages/19'); + expect(ackRequest?.url.searchParams.get('existing')).toBe('1'); + expect(new Headers(ackRequest?.init.headers).get('Authorization')).toBe('Bearer queue-token'); + + const acquireRequest = fixture.requests.find(({ url }) => url.pathname.endsWith('/acquirejobs')); + expect(acquireRequest?.url.pathname).toBe('/tenant/123/_apis/runtime/runnerscalesets/42/acquirejobs'); + expect(acquireRequest?.url.searchParams.get('api-version')).toBe('6.0-preview'); + expect(new Headers(acquireRequest?.init.headers).get('Authorization')).toBe('Bearer queue-token'); + expect(JSON.parse(acquireRequest?.init.body as string)).toEqual([501, 999]); + }); + + it.each([ + ['message id', { messageId: 0, messageType: 'RunnerScaleSetJobMessages', statistics, body: '[]' }], + [ + 'statistics', + { + messageId: 1, + messageType: 'RunnerScaleSetJobMessages', + statistics: { ...statistics, totalAssignedJobs: -1 }, + body: '[]', + }, + ], + [ + 'runner request id', + { + messageId: 1, + messageType: 'RunnerScaleSetJobMessages', + statistics, + body: JSON.stringify([{ messageType: 'JobAvailable', runnerRequestId: 0 }]), + }, + ], + [ + 'runner identity', + { + messageId: 1, + messageType: 'RunnerScaleSetJobMessages', + statistics, + body: JSON.stringify([ + { messageType: 'JobCompleted', runnerRequestId: 1, runnerId: 2, runnerName: 'bad\nname' }, + ]), + }, + ], + ])('rejects malformed known message %s fields', async (_name, payload) => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse()); + } + if (url.hostname === 'queue.example' && init.method === 'GET') return jsonResponse(payload); + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + await expect(session.getMessage(0, 10)).rejects.toBeInstanceOf(ScaleSetProtocolError); + }); + + it('refreshes the message session once on a queue 401 and retries with the new token', async () => { + let refreshCount = 0; + let oldTokenPolls = 0; + let newTokenPolls = 0; + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse('old-queue-token')); + } + if (url.pathname.includes('/runnerscalesets/42/sessions/') && init.method === 'PATCH') { + refreshCount += 1; + return jsonResponse({ + ...sessionResponse('new-queue-token'), + runnerScaleSet: { id: 42, name: 'linux', RunnerSetting: { disableUpdate: false } }, + }); + } + if (url.hostname === 'queue.example' && init.method === 'GET') { + const authorization = new Headers(init.headers).get('Authorization'); + if (authorization === 'Bearer old-queue-token') { + oldTokenPolls += 1; + return jsonResponse({ message: 'expired' }, 401); + } + if (authorization === 'Bearer new-queue-token') { + newTokenPolls += 1; + return new Response(null, { status: 202 }); + } + } + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + await expect(session.getMessage(0, 10)).resolves.toBeNull(); + + expect(oldTokenPolls).toBe(1); + expect(newTokenPolls).toBe(1); + expect(refreshCount).toBe(1); + expect(session.session.messageQueueAccessToken).toBe('new-queue-token'); + expect(session.session.runnerScaleSet?.runnerSetting).toEqual({ disableUpdate: false }); + expect(session.session.runnerScaleSet).not.toHaveProperty('RunnerSetting'); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/message-session-client.ts b/lambdas/libs/github-actions-scale-set/src/message-session-client.ts new file mode 100644 index 0000000000..130e68482e --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/message-session-client.ts @@ -0,0 +1,467 @@ +import { SCALE_SET_ERROR_CODES, ScaleSetHttpError, ScaleSetProtocolError } from './errors'; +import { executeRequest, HttpResult, parseJsonResponse } from './http'; +import { SCALE_SET_ENDPOINT } from './endpoints'; +import { + JobAssigned, + JobAvailable, + JobCompleted, + JobStarted, + MESSAGE_TYPES, + RunnerScaleSetMessage, + RunnerScaleSet, + RunnerScaleSetSession, + RunnerScaleSetStatistic, + ScaleSetFetch, + ScaleSetRequestOptions, +} from './types'; + +export const HEADER_SCALE_SET_MAX_CAPACITY = 'X-ScaleSetMaxCapacity'; + +interface ActionsRequestOptions extends ScaleSetRequestOptions { + body?: unknown; + expectedStatuses: readonly number[]; + authorization?: string; +} + +interface MessageSessionClientCreateOptions extends ScaleSetRequestOptions { + runnerScaleSetId: number; + owner: string; + fetchImplementation: ScaleSetFetch; + userAgent: () => string; + actionsRequest: ( + method: string, + path: string, + options: ActionsRequestOptions, + ) => Promise<{ result: HttpResult; url: URL }>; +} + +interface RunnerScaleSetMessageResponse { + messageId: number; + messageType: string; + body?: string; + statistics?: RunnerScaleSetStatistic | null; +} + +interface AcquireJobsResponse { + count: number; + value: number[]; +} + +interface JobMessageType { + messageType?: unknown; +} + +const STATISTIC_FIELDS = [ + 'totalAvailableJobs', + 'totalAcquiredJobs', + 'totalAssignedJobs', + 'totalRunningJobs', + 'totalRegisteredRunners', + 'totalBusyRunners', + 'totalIdleRunners', +] as const satisfies readonly (keyof RunnerScaleSetStatistic)[]; + +function positiveInteger(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || (value as number) <= 0) { + throw new ScaleSetProtocolError(`${field} must be a positive integer`); + } + return value as number; +} + +function validateStatistics(value: unknown): RunnerScaleSetStatistic | null { + if (value === null || value === undefined) return null; + if (typeof value !== 'object' || Array.isArray(value)) { + throw new ScaleSetProtocolError('runner scale set statistics must be an object'); + } + for (const field of STATISTIC_FIELDS) { + const statistic = (value as Record)[field]; + if (!Number.isSafeInteger(statistic) || (statistic as number) < 0) { + throw new ScaleSetProtocolError(`statistics.${field} must be a non-negative integer`); + } + } + return value as RunnerScaleSetStatistic; +} + +function validateKnownJobMessage(rawMessage: Record, messageType: string): void { + positiveInteger(rawMessage.runnerRequestId, `${messageType}.runnerRequestId`); + if (messageType === MESSAGE_TYPES.jobStarted || messageType === MESSAGE_TYPES.jobCompleted) { + positiveInteger(rawMessage.runnerId, `${messageType}.runnerId`); + if ( + typeof rawMessage.runnerName !== 'string' || + rawMessage.runnerName.length === 0 || + rawMessage.runnerName.length > 256 || + hasAsciiControlCharacter(rawMessage.runnerName) + ) { + throw new ScaleSetProtocolError(`${messageType}.runnerName is invalid`); + } + } +} + +function hasAsciiControlCharacter(value: string): boolean { + return [...value].some((character) => { + const code = character.charCodeAt(0); + return code <= 31 || code === 127; + }); +} + +function normalizeRunnerScaleSet(scaleSet: RunnerScaleSet | null | undefined): RunnerScaleSet | null | undefined { + if (scaleSet === null || scaleSet === undefined) { + return scaleSet; + } + + const wire = scaleSet as RunnerScaleSet & { RunnerSetting?: RunnerScaleSet['runnerSetting'] }; + if (wire.runnerSetting === undefined && wire.RunnerSetting !== undefined) { + wire.runnerSetting = wire.RunnerSetting; + } + delete wire.RunnerSetting; + return wire; +} + +function normalizeSession(session: RunnerScaleSetSession): RunnerScaleSetSession { + session.runnerScaleSet = normalizeRunnerScaleSet(session.runnerScaleSet); + return session; +} + +function cloneSession(session: RunnerScaleSetSession): RunnerScaleSetSession { + return { + ...session, + runnerScaleSet: + session.runnerScaleSet === undefined || session.runnerScaleSet === null + ? session.runnerScaleSet + : { + ...session.runnerScaleSet, + labels: session.runnerScaleSet.labels?.map((label) => ({ ...label })), + runnerSetting: + session.runnerScaleSet.runnerSetting === undefined + ? undefined + : { ...session.runnerScaleSet.runnerSetting }, + statistics: + session.runnerScaleSet.statistics === undefined || session.runnerScaleSet.statistics === null + ? session.runnerScaleSet.statistics + : { ...session.runnerScaleSet.statistics }, + }, + statistics: + session.statistics === undefined || session.statistics === null ? session.statistics : { ...session.statistics }, + }; +} + +function validateSession(session: RunnerScaleSetSession): RunnerScaleSetSession { + if (!session.sessionId) { + throw new ScaleSetProtocolError('message session response is missing sessionId'); + } + if (!session.messageQueueUrl) { + throw new ScaleSetProtocolError('message session response is missing messageQueueUrl'); + } + if (!session.messageQueueAccessToken) { + throw new ScaleSetProtocolError('message session response is missing messageQueueAccessToken'); + } + let queueUrl: URL; + try { + queueUrl = new URL(session.messageQueueUrl); + } catch (error) { + throw new ScaleSetProtocolError('message session response contains an invalid messageQueueUrl', { cause: error }); + } + if (queueUrl.protocol !== 'https:' || queueUrl.username || queueUrl.password || queueUrl.hash) { + throw new ScaleSetProtocolError( + 'message session messageQueueUrl must be HTTPS and contain no credentials or fragment', + ); + } + return session; +} + +function parseRunnerScaleSetMessage(result: HttpResult, url: URL): RunnerScaleSetMessage { + const response = parseJsonResponse(result, 'GET', url); + positiveInteger(response.messageId, 'messageId'); + if (response.messageType !== 'RunnerScaleSetJobMessages') { + throw new ScaleSetProtocolError(`unsupported message type: ${response.messageType}`); + } + + let batchedMessages: unknown[] = []; + if (response.body) { + try { + const parsed = JSON.parse(response.body) as unknown; + if (!Array.isArray(parsed)) { + throw new TypeError('message body is not an array'); + } + batchedMessages = parsed; + if (batchedMessages.length > 50) { + throw new TypeError('message body contains more than 50 entries'); + } + } catch (error) { + throw new ScaleSetProtocolError('failed to unmarshal batched runner scale set messages', { + cause: error, + }); + } + } + + const message: RunnerScaleSetMessage = { + messageId: response.messageId, + statistics: validateStatistics(response.statistics), + jobAvailableMessages: [], + jobAssignedMessages: [], + jobStartedMessages: [], + jobCompletedMessages: [], + }; + + for (const rawMessage of batchedMessages) { + if (typeof rawMessage !== 'object' || rawMessage === null) { + throw new ScaleSetProtocolError('runner scale set job message is not an object'); + } + const messageType = (rawMessage as JobMessageType).messageType; + if (typeof messageType !== 'string') { + throw new ScaleSetProtocolError('runner scale set job message is missing messageType'); + } + switch (messageType) { + case MESSAGE_TYPES.jobAvailable: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobAvailableMessages.push(rawMessage as JobAvailable); + break; + case MESSAGE_TYPES.jobAssigned: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobAssignedMessages.push(rawMessage as JobAssigned); + break; + case MESSAGE_TYPES.jobStarted: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobStartedMessages.push(rawMessage as JobStarted); + break; + case MESSAGE_TYPES.jobCompleted: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobCompletedMessages.push(rawMessage as JobCompleted); + break; + default: + // The upstream client ignores unknown job message types for forward compatibility. + break; + } + } + + return message; +} + +/** A message queue session scoped to one runner scale set. */ +export class MessageSessionClient { + private readonly runnerScaleSetId: number; + private readonly fetchImplementation: ScaleSetFetch; + private readonly userAgent: () => string; + private readonly actionsRequest: MessageSessionClientCreateOptions['actionsRequest']; + private currentSession: RunnerScaleSetSession; + private sessionRefresh?: Promise; + + private constructor(options: MessageSessionClientCreateOptions, session: RunnerScaleSetSession) { + this.runnerScaleSetId = options.runnerScaleSetId; + this.fetchImplementation = options.fetchImplementation; + this.userAgent = options.userAgent; + this.actionsRequest = options.actionsRequest; + this.currentSession = session; + } + + static async create(options: MessageSessionClientCreateOptions): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${options.runnerScaleSetId}/sessions`; + const { result, url } = await options.actionsRequest('POST', path, { + body: { ownerName: options.owner }, + expectedStatuses: [200], + signal: options.signal, + }); + const session = validateSession(normalizeSession(parseJsonResponse(result, 'POST', url))); + return new MessageSessionClient(options, session); + } + + /** A defensive snapshot of the current session and its latest statistics. */ + get session(): RunnerScaleSetSession { + return cloneSession(this.currentSession); + } + + getSession(): RunnerScaleSetSession { + return this.session; + } + + async close(options: ScaleSetRequestOptions = {}): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${this.runnerScaleSetId}/sessions/${this.currentSession.sessionId}`; + await this.actionsRequest('DELETE', path, { + expectedStatuses: [204], + signal: options.signal, + }); + } + + /** + * Long-poll for a batched scale set message. A 202 response means no message + * is currently available and is represented as `null`. + */ + async getMessage( + lastMessageId: number, + maxCapacity: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + return this.withMessageTokenRefresh( + (session) => this.getMessageWithSession(session, lastMessageId, maxCapacity, options), + options, + ); + } + + async pollMessage( + lastMessageId: number, + maxCapacity: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + return this.getMessage(lastMessageId, maxCapacity, options); + } + + /** Delete a queue message after processing it, which acknowledges the batch. */ + async deleteMessage(messageId: number, options: ScaleSetRequestOptions = {}): Promise { + await this.withMessageTokenRefresh( + (session) => this.deleteMessageWithSession(session, messageId, options), + options, + ); + } + + async acknowledgeMessage(messageId: number, options: ScaleSetRequestOptions = {}): Promise { + return this.deleteMessage(messageId, options); + } + + /** Return the authoritative subset of runner request IDs acquired by the service. */ + async acquireJobs(requestIds: number[], options: ScaleSetRequestOptions = {}): Promise { + return this.withMessageTokenRefresh( + (session) => this.acquireJobsWithSession(session, requestIds, options), + options, + ); + } + + private async getMessageWithSession( + session: RunnerScaleSetSession, + lastMessageId: number, + maxCapacity: number, + options: ScaleSetRequestOptions, + ): Promise { + const url = new URL(session.messageQueueUrl); + if (lastMessageId > 0) { + url.searchParams.set('lastMessageId', String(lastMessageId)); + } + const result = await executeRequest( + this.fetchImplementation, + url, + { + method: 'GET', + headers: { + Accept: 'application/json; api-version=6.0-preview', + Authorization: `Bearer ${session.messageQueueAccessToken}`, + 'User-Agent': this.userAgent(), + [HEADER_SCALE_SET_MAX_CAPACITY]: String(maxCapacity), + }, + signal: options.signal, + }, + [200, 202], + (response) => (response.status === 401 ? SCALE_SET_ERROR_CODES.messageQueueTokenExpired : undefined), + ); + + if (result.response.status === 202) { + return null; + } + return parseRunnerScaleSetMessage(result, url); + } + + private async deleteMessageWithSession( + session: RunnerScaleSetSession, + messageId: number, + options: ScaleSetRequestOptions, + ): Promise { + const url = new URL(session.messageQueueUrl); + const valueAfterOrigin = session.messageQueueUrl.slice(url.origin.length); + const originalPath = valueAfterOrigin.startsWith('/') ? url.pathname : ''; + url.pathname = `${originalPath}/${messageId}`; + await executeRequest( + this.fetchImplementation, + url, + { + method: 'DELETE', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${session.messageQueueAccessToken}`, + 'User-Agent': this.userAgent(), + }, + signal: options.signal, + }, + [204], + (response) => (response.status === 401 ? SCALE_SET_ERROR_CODES.messageQueueTokenExpired : undefined), + ); + } + + private async acquireJobsWithSession( + session: RunnerScaleSetSession, + requestIds: number[], + options: ScaleSetRequestOptions, + ): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${this.runnerScaleSetId}/acquirejobs`; + try { + const { result, url } = await this.actionsRequest('POST', path, { + body: requestIds, + expectedStatuses: [200], + authorization: `Bearer ${session.messageQueueAccessToken}`, + signal: options.signal, + }); + return parseJsonResponse(result, 'POST', url).value; + } catch (error) { + if (error instanceof ScaleSetHttpError && error.status === 401) { + throw new ScaleSetHttpError({ + method: error.method, + url: error.url, + status: error.status, + statusText: error.statusText, + headers: new Headers({ + ...(error.activityId ? { ActivityId: error.activityId } : {}), + ...(error.githubRequestId ? { 'X-GitHub-Request-Id': error.githubRequestId } : {}), + }), + responseBody: error.responseBody, + code: SCALE_SET_ERROR_CODES.messageQueueTokenExpired, + cause: error, + }); + } + throw error; + } + } + + private async withMessageTokenRefresh( + operation: (session: RunnerScaleSetSession) => Promise, + options: ScaleSetRequestOptions, + ): Promise { + const expiredSession = this.currentSession; + try { + return await operation(expiredSession); + } catch (error) { + if (!(error instanceof ScaleSetHttpError) || error.code !== SCALE_SET_ERROR_CODES.messageQueueTokenExpired) { + throw error; + } + } + + await this.refreshMessageSession(expiredSession, options); + return operation(this.currentSession); + } + + private async refreshMessageSession( + expiredSession: RunnerScaleSetSession, + options: ScaleSetRequestOptions, + ): Promise { + if ( + this.currentSession.sessionId !== expiredSession.sessionId || + this.currentSession.messageQueueAccessToken !== expiredSession.messageQueueAccessToken + ) { + return; + } + + if (this.sessionRefresh === undefined) { + this.sessionRefresh = this.doRefreshMessageSession(options).finally(() => { + this.sessionRefresh = undefined; + }); + } + await this.sessionRefresh; + } + + private async doRefreshMessageSession(options: ScaleSetRequestOptions): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${this.runnerScaleSetId}/sessions/${this.currentSession.sessionId}`; + const { result, url } = await this.actionsRequest('PATCH', path, { + expectedStatuses: [200], + signal: options.signal, + }); + this.currentSession = validateSession( + normalizeSession(parseJsonResponse(result, 'PATCH', url)), + ); + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/types.ts b/lambdas/libs/github-actions-scale-set/src/types.ts new file mode 100644 index 0000000000..11d387bf4c --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/types.ts @@ -0,0 +1,183 @@ +export const DEFAULT_RUNNER_GROUP = 'default'; + +export const MESSAGE_TYPES = { + jobAvailable: 'JobAvailable', + jobAssigned: 'JobAssigned', + jobStarted: 'JobStarted', + jobCompleted: 'JobCompleted', +} as const; + +export type MessageType = (typeof MESSAGE_TYPES)[keyof typeof MESSAGE_TYPES]; + +export interface JobMessageBase { + messageType: MessageType; + runnerRequestId: number; + repositoryName: string; + ownerName: string; + jobId: string; + jobWorkflowRef: string; + jobDisplayName: string; + workflowRunId: number; + eventName: string; + requestLabels: string[]; + queueTime: string; + scaleSetAssignTime: string; + runnerAssignTime: string; + finishTime: string; +} + +export interface JobAvailable extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobAvailable; + acquireJobUrl: string; +} + +export interface JobAssigned extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobAssigned; +} + +export interface JobStarted extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobStarted; + runnerId: number; + runnerName: string; +} + +export interface JobCompleted extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobCompleted; + result: string; + runnerId: number; + runnerName: string; +} + +export interface Label { + type?: string; + name: string; +} + +export interface RunnerGroup { + id: number; + name: string; + size: number; + isDefaultGroup: boolean; +} + +export interface RunnerSetting { + disableUpdate?: boolean; +} + +/** + * Runner scale set representation used by the Actions service. + * + * The same shape is accepted for create and update operations, so server-owned + * fields are optional. The client translates `runnerSetting` to the upstream + * wire key `RunnerSetting` when it sends a request. + */ +export interface RunnerScaleSet { + id?: number; + name?: string; + runnerGroupId?: number; + runnerGroupName?: string; + labels?: Label[]; + runnerSetting?: RunnerSetting; + createdOn?: string; + runnerJitConfigUrl?: string; + statistics?: RunnerScaleSetStatistic | null; +} + +export interface RunnerScaleSetJitRunnerSetting { + name: string; + workFolder?: string; +} + +export interface RunnerReference { + id: number; + name: string; + runnerScaleSetId: number; +} + +export interface RunnerScaleSetJitRunnerConfig { + runner: RunnerReference | null; + encodedJITConfig: string; +} + +export interface RunnerScaleSetStatistic { + totalAvailableJobs: number; + totalAcquiredJobs: number; + totalAssignedJobs: number; + totalRunningJobs: number; + totalRegisteredRunners: number; + totalBusyRunners: number; + totalIdleRunners: number; +} + +export interface RunnerScaleSetSession { + sessionId: string; + ownerName: string; + runnerScaleSet?: RunnerScaleSet | null; + messageQueueUrl: string; + messageQueueAccessToken: string; + statistics?: RunnerScaleSetStatistic | null; +} + +export interface RunnerScaleSetMessage { + messageId: number; + statistics: RunnerScaleSetStatistic | null; + jobAvailableMessages: JobAvailable[]; + jobAssignedMessages: JobAssigned[]; + jobStartedMessages: JobStarted[]; + jobCompletedMessages: JobCompleted[]; +} + +export interface SystemInfo { + system?: string; + version?: string; + commitSha?: string; + scaleSetId?: number; + subsystem?: string; +} + +export interface AccessToken { + token: string; + expiresAt?: string | Date; +} + +export type AccessTokenProvider = () => Promise; + +export type ScaleSetFetch = typeof globalThis.fetch; + +export interface ScaleSetRequestOptions { + signal?: AbortSignal; +} + +export interface ScaleSetRetryOptions { + /** Number of retries after the initial request for retry-eligible operations. */ + maxRetries?: number; + /** Initial exponential-backoff delay. */ + initialBackoffMs?: number; + /** Upper bound for exponential backoff and Retry-After delays. */ + maxBackoffMs?: number; + /** Timeout applied independently to each fetch attempt. */ + requestTimeoutMs?: number; +} + +interface ScaleSetClientBaseOptions { + gitHubConfigUrl: string; + systemInfo?: SystemInfo; + fetch?: ScaleSetFetch; + forceGhes?: boolean; + userAgent?: string; + /** Intended for deterministic tests. Defaults to `new Date()`. */ + now?: () => Date; + retry?: ScaleSetRetryOptions; +} + +export type GitHubActionsScaleSetClientOptions = ScaleSetClientBaseOptions & + ( + | { + personalAccessToken: string; + accessTokenProvider?: never; + } + | { + personalAccessToken?: never; + accessTokenProvider: AccessTokenProvider; + } + ); diff --git a/lambdas/libs/github-actions-scale-set/src/url.ts b/lambdas/libs/github-actions-scale-set/src/url.ts new file mode 100644 index 0000000000..55d9ecb468 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/url.ts @@ -0,0 +1,24 @@ +const FORWARD_SLASH = '/'.charCodeAt(0); + +export function trimTrailingSlashes(value: string): string { + let end = value.length; + while (end > 0 && value.charCodeAt(end - 1) === FORWARD_SLASH) { + end -= 1; + } + + return end === value.length ? value : value.slice(0, end); +} + +export function trimSurroundingSlashes(value: string): string { + let start = 0; + while (start < value.length && value.charCodeAt(start) === FORWARD_SLASH) { + start += 1; + } + + let end = value.length; + while (end > start && value.charCodeAt(end - 1) === FORWARD_SLASH) { + end -= 1; + } + + return start === 0 && end === value.length ? value : value.slice(start, end); +} diff --git a/lambdas/libs/github-actions-scale-set/tsconfig.json b/lambdas/libs/github-actions-scale-set/tsconfig.json new file mode 100644 index 0000000000..714aa27b6b --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/lambdas/libs/github-actions-scale-set/vitest.config.ts b/lambdas/libs/github-actions-scale-set/vitest.config.ts new file mode 100644 index 0000000000..c52b8d7522 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/vitest.config.ts @@ -0,0 +1,22 @@ +import { mergeConfig } from 'vitest/config'; + +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + root: __dirname, + test: { + coverage: { + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts', 'src/**/*.d.ts', 'src/index.ts'], + thresholds: { + // Measured by the package-scoped wire-contract suite. These floors keep + // meaningful regression protection without pretending every defensive + // parser/error branch is exercised by production-path tests. + statements: 75, + branches: 60, + functions: 80, + lines: 75, + }, + }, + }, +}); diff --git a/lambdas/package.json b/lambdas/package.json index c6fa5d72c3..0a0b0b088a 100644 --- a/lambdas/package.json +++ b/lambdas/package.json @@ -3,7 +3,8 @@ "private": true, "workspaces": [ "functions/*", - "libs/*" + "libs/*", + "services/*" ], "scripts": { "build": "nx run-many --target=build --all", diff --git a/lambdas/services/scale-set/Dockerfile b/lambdas/services/scale-set/Dockerfile new file mode 100644 index 0000000000..d1bfbe429f --- /dev/null +++ b/lambdas/services/scale-set/Dockerfile @@ -0,0 +1,25 @@ +FROM node:24-bookworm-slim@sha256:3638d9a6fe4030bd716be989438248074489337ba3275657f93595428be4fc03 AS build +WORKDIR /workspace/lambdas + +COPY lambdas/package.json lambdas/yarn.lock lambdas/.yarnrc.yml ./ +COPY lambdas/.yarn ./.yarn +COPY lambdas/tsconfig.json lambdas/vitest.base.config.ts ./ +COPY lambdas/functions ./functions +COPY lambdas/libs ./libs +COPY lambdas/services ./services + +RUN corepack enable && yarn install --immutable +RUN yarn workspace @aws-github-runner/scale-set-service build + +FROM node:24-bookworm-slim@sha256:3638d9a6fe4030bd716be989438248074489337ba3275657f93595428be4fc03 AS runtime +ENV NODE_ENV=production \ + SCALE_SET_HEALTH_PORT=8080 +WORKDIR /app + +COPY --from=build --chown=node:node /workspace/lambdas/services/scale-set/dist/ ./ +COPY --from=build --chown=node:node /workspace/lambdas/services/scale-set/healthcheck.cjs ./healthcheck.cjs + +USER node +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=90s --retries=3 CMD ["node", "/app/healthcheck.cjs"] +ENTRYPOINT ["node", "--disable-proto=delete", "/app/index.js"] diff --git a/lambdas/services/scale-set/Dockerfile.dockerignore b/lambdas/services/scale-set/Dockerfile.dockerignore new file mode 100644 index 0000000000..7c72e0e7ab --- /dev/null +++ b/lambdas/services/scale-set/Dockerfile.dockerignore @@ -0,0 +1,7 @@ +**/coverage +**/dist +**/node_modules +.git +.nx +lambdas/.yarn/install-state.gz +*.log diff --git a/lambdas/services/scale-set/README.md b/lambdas/services/scale-set/README.md new file mode 100644 index 0000000000..e42ada323d --- /dev/null +++ b/lambdas/services/scale-set/README.md @@ -0,0 +1,107 @@ +# Scale-set controller service + +This workspace builds the long-running, topology-neutral controller used by the scale-set orchestration provider. + +Each controller group maps to one ECS service, one task definition, and normally one running task. The task contains one application container and one `ScaleSetController`, which supervises one independent reconciler per runner config: + +```text +ECS service (controller group) +└── one ECS task + └── one scale-set container + └── ScaleSetController + ├── reconciler: runner config A → scale set A → session A + └── reconciler: runner config B → scale set B → session B +``` + +A group is only a packing and deployment boundary. Every reconciler retains its own GitHub message session, lifecycle state, retry loop, health, and compute-provider instance. One reconciler failure does not exit the others. + +## Production configuration + +The ECS task receives only these group selectors: + +- `SCALE_SET_CONTROLLER_GROUP_NAME` +- `SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH` +- `SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION` + +The service reads every direct child under the SSM path with paginated `GetParametersByPath`. Each child name must equal its `runnerConfigName`, and each value uses this flat, versioned schema: + +```json +{ + "schemaVersion": 1, + "runnerConfigName": "linux-x64", + "githubConfigUrl": "https://github.com/example", + "scaleSetName": "linux-x64", + "runnerGroupName": "self-hosted-linux", + "runnerGroupIdParameterName": "/runners/github-app/runner-group/self-hosted-linux", + "minRunners": 0, + "maxRunners": 20, + "bootTimeoutMinutes": 10, + "sslVerify": true, + "githubApp": { + "appIdParameterName": "/runners/github-app/id", + "privateKeyParameterName": "/runners/github-app/key" + }, + "computeProvider": { + "type": "ec2", + "configuration": { + "region": "eu-west-1", + "environment": "example-linux-x64", + "runnerNamePrefix": "", + "jitConfigParameterPath": "/runners/example-linux-x64/tokens", + "subnets": ["subnet-0123456789abcdef0"], + "launchTemplateName": "example-linux-x64-action-runner", + "ec2instanceCriteria": { + "instanceTypes": ["m7i.large"], + "targetCapacityType": "spot", + "instanceAllocationStrategy": "price-capacity-optimized" + }, + "onDemandFailoverOnError": [], + "useDedicatedHost": false, + "ssmParameterTags": [] + } + } +} +``` + +`scaleSetName` and `runnerGroupName` are the GitHub names supplied by the operator. `runnerGroupIdParameterName` is an optional SSM cache path. When present, the service reads the runner-group ID from that parameter; if it is missing, the service resolves the name through the configured GitHub Actions service endpoint and writes the ID back as a non-secret `String` parameter with overwrite enabled. The service resolves the scale-set ID from the group and scale-set names; if the named scale set does not exist, it registers it in the resolved runner group and uses the ID returned by GitHub. `scaleSetId` is only an optional legacy pin for an already-known ID. `expectedRunnerGroupId` can be omitted or null; when supplied, it is treated as an additional consistency check. Optional fields are `scaleSetId`, `runnerGroupIdParameterName`, `expectedRunnerGroupId`, `sessionOwner`, `workFolder`, `forceGhes`, `sslVerify`, and `userAgent`. `sslVerify` defaults to `true`; when false, the service uses a reconciler-scoped Undici dispatcher for both GitHub App token and scale-set requests without changing `NODE_TLS_REJECT_UNAUTHORIZED` or the global dispatcher. `userAgent` becomes the `system` identity inside the required structured scale-set protocol User-Agent rather than replacing that header. `bootTimeoutMinutes` defaults to `10`; it is orchestration-owned and is passed to the selected compute provider on every reconciliation. + +GitHub App ID and private-key values are reloaded from SSM whenever an installation token is requested, so key rotation does not require a task restart. `installationIdParameterName` is optional; when it is absent or its parameter is not present, the service creates a short-lived App JWT and discovers the installation by matching the configured organization or enterprise account through `GET /app/installations`. This works with GitHub.com, GHES, and GitHub Enterprise Cloud data-residency API hosts derived from `githubConfigUrl`. A SHA-256 credential fingerprint keeps the same Octokit auth instance—and its token cache—while the values remain unchanged, and replaces it after rotation. Private keys, installation tokens, App JWTs, message-session tokens, message bodies, and JIT configurations are never accepted as manifest values and are redacted from logs. + +`SCALE_SET_CONTROLLER_MANIFEST` is supported only as a bounded local/test convenience. It contains `{ "version": 1, "groupName": "...", "reconcilers": [...] }` and uses the same reconciler objects. + +Runtime settings: + +| Environment variable | Default | +| --------------------------------------------- | ------- | +| `SCALE_SET_HEALTH_PORT` | `8080` | +| `SCALE_SET_HEALTH_STALE_AFTER_SECONDS` | `180` | +| `SCALE_SET_SHUTDOWN_TIMEOUT_SECONDS` | `110` | +| `SCALE_SET_SESSION_CLOSE_TIMEOUT_SECONDS` | `10` | +| `SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS` | `1` | +| `SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS` | `30` | +| `LOG_LEVEL` | `info` | + +## Reconciliation and health + +Demand is calculated as `max(totalAssignedJobs, min(maxRunners, minRunners + totalAssignedJobs))`. The maximum therefore bounds requested idle capacity without ever requesting scale-down below work GitHub has already assigned. Job-started and job-completed messages maintain a bounded in-memory lifecycle cache. After a restart, lifecycle state is unknown, but provider-owned EC2 tags still identify current capacity. The aggregate scale-set busy count protects unknown runners from scale-down until it reaches zero. Runner deletion executes inside the serialized reconcile loop and re-checks the exact Actions-service runner identity by name before removal. + +The public GitHub runner inventory is not fetched by the scale-set service. The selected compute provider reconciles its own capacity inventory, while the scale-set session remains the source of demand and aggregate busy-runner statistics. This keeps the service aligned with the upstream scale-set API and supports GitHub.com, GHES, and data-residency endpoints without relying on a separate public REST runner endpoint. + +Messages follow the upstream scale-set listener order: acknowledge first, then acquire available jobs, update lifecycle state, and reconcile compute. Provider failures therefore stop that reconciler after the message has been acknowledged; provider results expose one error outcome rather than the control-plane scaling retry classification. A typed busy/unknown retention remains a successful reconciliation. Session and transport failures are handled separately by bounded client retries or session recreation. + +The EC2 provider uses the tagged, `config-published` EC2 instances as its capacity inventory. The GitHub Actions scale-set session supplies `desiredRunners` through `totalAssignedJobs` and the aggregate `totalBusyRunners` count. When capacity is above desired and the aggregate busy count is zero, tagged runners may be removed through the Actions service and their matching EC2 instances terminated; busy, contradictory, or unknown identities are retained. The provider uses the boot window (`bootTimeoutMinutes`, default `10`) for newly launched instances and keeps interrupted publication states from being counted as serving. EC2 ownership includes a SHA-256 hash of the canonical GitHub configuration scope, preventing the same runner-config name and numeric scale-set ID in another GitHub scope from colliding. No public GitHub REST runner inventory call is required. + +- `GET /healthz` reports controller liveness and is used by Docker/ECS. External GitHub outages remain live but degraded to avoid restart loops. +- `GET /readyz` reports readiness and returns 503 unless every reconciler is ready. + +## Container + +Build from the repository root: + +```shell +docker build --target runtime -f lambdas/services/scale-set/Dockerfile -t scale-set-controller . +``` + +The image supports `linux/amd64` and `linux/arm64`, uses a digest-pinned multi-stage Node image, runs as the unprivileged `node` user, includes a Node-based health check, and does not require filesystem writes. Deploy with a read-only root filesystem, all Linux capabilities dropped, no Docker socket, and only the task-role permissions required by the selected group. + +The module's official GHCR package must allow anonymous pulls so the default image works without registry credentials. Production deployments should select a released image by digest and verify its provenance/attestation. A private ECR override requires `container.ecr_repository.arn`; private non-ECR registry credentials are not currently exposed by the Terraform orchestration module. diff --git a/lambdas/services/scale-set/healthcheck.cjs b/lambdas/services/scale-set/healthcheck.cjs new file mode 100644 index 0000000000..66c833829c --- /dev/null +++ b/lambdas/services/scale-set/healthcheck.cjs @@ -0,0 +1,13 @@ +'use strict'; + +const http = require('node:http'); +const port = Number(process.env.SCALE_SET_HEALTH_PORT || '8080'); +const request = http.get( + { host: '127.0.0.1', port, path: '/healthz', timeout: 4000, headers: { Connection: 'close' } }, + (response) => { + response.resume(); + process.exit(response.statusCode === 200 ? 0 : 1); + }, +); +request.on('timeout', () => request.destroy(new Error('health check timed out'))); +request.on('error', () => process.exit(1)); diff --git a/lambdas/services/scale-set/package.json b/lambdas/services/scale-set/package.json new file mode 100644 index 0000000000..9a912211bd --- /dev/null +++ b/lambdas/services/scale-set/package.json @@ -0,0 +1,42 @@ +{ + "name": "@aws-github-runner/scale-set-service", + "version": "0.1.0", + "private": true, + "main": "dist/index.js", + "type": "module", + "license": "MIT", + "scripts": { + "build": "ncc build src/main.ts -o dist --minify", + "typecheck": "tsc --noEmit", + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint src", + "format": "prettier --write \"**/*.ts\"", + "format-check": "prettier --check \"**/*.ts\"", + "all": "yarn typecheck && yarn build && yarn format-check && yarn lint && yarn test" + }, + "devDependencies": { + "@types/node": "^22.19.3", + "@vercel/ncc": "0.38.4", + "typescript": "^5.9.3" + }, + "dependencies": { + "@aws-github-runner/aws-ssm-util": "*", + "@aws-github-runner/compute-providers": "*", + "@aws-github-runner/github-actions-scale-set": "*", + "@aws-sdk/client-ssm": "^3.1009.0", + "@octokit/auth-app": "8.2.0", + "@octokit/request": "^9.2.2", + "undici": "^6.19.2" + }, + "nx": { + "includedScripts": [ + "build", + "typecheck", + "format", + "format-check", + "lint", + "all" + ] + } +} diff --git a/lambdas/services/scale-set/src/config.test.ts b/lambdas/services/scale-set/src/config.test.ts new file mode 100644 index 0000000000..6fe4d30805 --- /dev/null +++ b/lambdas/services/scale-set/src/config.test.ts @@ -0,0 +1,205 @@ +import { + MAX_MANIFEST_BYTES, + parseScaleSetControllerManifest, + parseScaleSetReconcilerConfig, + parseScaleSetServiceConfig, +} from './config'; + +function runnerConfig(overrides: Record = {}) { + return { + schemaVersion: 1, + runnerConfigName: 'linux-x64', + githubConfigUrl: 'https://github.com/example', + scaleSetId: 123, + expectedScaleSetName: 'linux-x64', + expectedRunnerGroupId: null, + minRunners: 0, + maxRunners: 20, + githubApp: { + appIdParameterName: '/runner/app/id', + privateKeyParameterName: '/runner/app/key', + installationIdParameterName: '/runner/app/installation-id', + }, + computeProvider: { type: 'ec2', configuration: { subnetIds: ['subnet-1'] } }, + ...overrides, + }; +} + +describe('scale-set service configuration', () => { + it('parses the production SSM group source and runtime defaults', () => { + expect( + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_GROUP_NAME: 'ec2-default', + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/runner/groups/ec2-default', + SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION: '42', + }), + ).toEqual({ + groupName: 'ec2-default', + groupConfigPath: '/runner/groups/ec2-default', + groupRevision: '42', + healthPort: 8080, + healthStaleAfterMs: 180000, + shutdownTimeoutMs: 110000, + sessionCloseTimeoutMs: 10000, + reconnectInitialBackoffMs: 1000, + reconnectMaxBackoffMs: 30000, + }); + }); + + it('supports bounded inline manifests for local use', () => { + const manifest = JSON.stringify({ version: 1, groupName: 'local', reconcilers: [runnerConfig()] }); + expect(parseScaleSetServiceConfig({ SCALE_SET_CONTROLLER_MANIFEST: manifest }).manifest).toBe(manifest); + expect(() => + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_MANIFEST: manifest, + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/both', + }), + ).toThrow('provide exactly one'); + expect(() => + parseScaleSetServiceConfig({ SCALE_SET_CONTROLLER_MANIFEST: 'x'.repeat(MAX_MANIFEST_BYTES + 1) }), + ).toThrow('must not exceed'); + }); + + it('validates production selectors and numeric runtime settings', () => { + expect(() => parseScaleSetServiceConfig({})).toThrow('provide exactly one'); + expect(() => + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_GROUP_NAME: 'bad name', + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/valid', + SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION: '1', + }), + ).toThrow('group name'); + expect(() => + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_GROUP_NAME: 'valid', + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/valid', + SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION: '1', + SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS: '31', + SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS: '30', + }), + ).toThrow('must not exceed'); + }); +}); + +describe('parseScaleSetControllerManifest', () => { + it('parses the frozen flat runner-config schema and defaults', () => { + expect(parseScaleSetReconcilerConfig(runnerConfig(), 0, 'group')).toMatchObject({ + schemaVersion: 1, + runnerConfigName: 'linux-x64', + scaleSetName: 'linux-x64', + bootTimeoutMinutes: 10, + sessionOwner: 'group.linux-x64', + workFolder: '_work', + forceGhes: false, + sslVerify: true, + computeProvider: { type: 'ec2', configuration: { subnetIds: ['subnet-1'] } }, + }); + }); + + it('normalizes explicit optional settings', () => { + expect( + parseScaleSetReconcilerConfig( + runnerConfig({ + expectedRunnerGroupId: 7, + sessionOwner: 'owner/group', + workFolder: 'runner/_work', + forceGhes: true, + sslVerify: false, + userAgent: 'github-aws-runners/test', + bootTimeoutMinutes: 30, + }), + 0, + 'group', + ), + ).toMatchObject({ + expectedRunnerGroupId: 7, + sessionOwner: 'owner/group', + workFolder: 'runner/_work', + forceGhes: true, + sslVerify: false, + bootTimeoutMinutes: 30, + }); + }); + + it('accepts a runner-group name for runtime ID resolution', () => { + expect( + parseScaleSetReconcilerConfig(runnerConfig({ runnerGroupName: 'self-hosted-linux' }), 0, 'group'), + ).toMatchObject({ runnerGroupName: 'self-hosted-linux' }); + }); + + it('accepts scaleSetName without a GitHub-generated scale-set ID', () => { + const parsed = parseScaleSetReconcilerConfig( + runnerConfig({ + scaleSetName: 'linux-x64', + expectedScaleSetName: undefined, + runnerGroupName: 'self-hosted-linux', + scaleSetId: undefined, + }), + 0, + 'group', + ); + expect(parsed).toMatchObject({ scaleSetName: 'linux-x64', runnerGroupName: 'self-hosted-linux' }); + expect(parsed).not.toHaveProperty('scaleSetId'); + }); + + it('bounds the derived session owner for maximum-length names', () => { + const parsed = parseScaleSetReconcilerConfig( + runnerConfig({ runnerConfigName: 'r'.repeat(128) }), + 0, + 'g'.repeat(128), + ); + expect(parsed.sessionOwner).toHaveLength(256); + expect(parsed.sessionOwner).toMatch(/\.[a-f0-9]{16}$/); + }); + + it('rejects unsafe URLs, unknown fields, prototype keys, and schema drift', () => { + expect(() => + parseScaleSetReconcilerConfig(runnerConfig({ githubConfigUrl: 'http://github.com/example' }), 0, 'g'), + ).toThrow('must use HTTPS'); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ extra: true }), 0, 'g')).toThrow('unknown field'); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ schemaVersion: 2 }), 0, 'g')).toThrow('schemaVersion'); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ bootTimeoutMinutes: 0 }), 0, 'g')).toThrow( + 'bootTimeoutMinutes', + ); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ bootTimeoutMinutes: 121 }), 0, 'g')).toThrow( + 'bootTimeoutMinutes', + ); + const polluted = JSON.parse('{"__proto__":{"admin":true}}') as unknown; + expect(() => + parseScaleSetReconcilerConfig( + runnerConfig({ computeProvider: { type: 'ec2', configuration: polluted } }), + 0, + 'g', + ), + ).toThrow('forbidden field'); + }); + + it('rejects duplicate runner names and scale-set IDs within an equivalent GitHub scope', () => { + expect(() => + parseScaleSetControllerManifest({ version: 1, groupName: 'g', reconcilers: [runnerConfig(), runnerConfig()] }), + ).toThrow('duplicated'); + expect(() => + parseScaleSetControllerManifest({ + version: 1, + groupName: 'g', + reconcilers: [ + runnerConfig(), + runnerConfig({ runnerConfigName: 'other', githubConfigUrl: 'https://GITHUB.com/example/' }), + ], + }), + ).toThrow('duplicated'); + }); + + it('allows the same numeric scale-set ID in different GitHub scopes', () => { + expect( + parseScaleSetControllerManifest({ + version: 1, + groupName: 'g', + reconcilers: [ + runnerConfig(), + runnerConfig({ runnerConfigName: 'other', githubConfigUrl: 'https://github.com/another' }), + ], + }).reconcilers, + ).toHaveLength(2); + }); +}); diff --git a/lambdas/services/scale-set/src/config.ts b/lambdas/services/scale-set/src/config.ts new file mode 100644 index 0000000000..0f048dbe6c --- /dev/null +++ b/lambdas/services/scale-set/src/config.ts @@ -0,0 +1,515 @@ +import { createHash } from 'node:crypto'; + +export const SCALE_SET_CONTROLLER_MANIFEST_VERSION = 1; +export const MAX_MANIFEST_BYTES = 256 * 1024; + +export type JsonPrimitive = boolean | number | string | null; +export type JsonValue = JsonPrimitive | JsonValue[] | { readonly [key: string]: JsonValue }; + +export interface GitHubAppParameterReferences { + appIdParameterName: string; + installationIdParameterName?: string; + privateKeyParameterName: string; +} + +export interface ScaleSetReconcilerConfig { + schemaVersion: 1; + runnerConfigName: string; + scaleSetId?: number; + scaleSetName: string; + runnerGroupName?: string; + runnerGroupIdParameterName?: string; + expectedRunnerGroupId?: number; + githubConfigUrl: string; + githubApp: GitHubAppParameterReferences; + computeProvider: { + type: string; + configuration: Readonly>; + }; + minRunners: number; + maxRunners: number; + bootTimeoutMinutes: number; + sessionOwner: string; + workFolder: string; + forceGhes: boolean; + sslVerify: boolean; + userAgent?: string; +} + +export interface ScaleSetControllerManifest { + version: typeof SCALE_SET_CONTROLLER_MANIFEST_VERSION; + groupName: string; + revision?: string; + reconcilers: readonly ScaleSetReconcilerConfig[]; +} + +export interface ScaleSetServiceConfig { + manifest?: string; + groupConfigPath?: string; + groupName?: string; + groupRevision?: string; + healthPort: number; + healthStaleAfterMs: number; + shutdownTimeoutMs: number; + sessionCloseTimeoutMs: number; + reconnectInitialBackoffMs: number; + reconnectMaxBackoffMs: number; +} + +export type ScaleSetServiceEnvironment = Readonly>; + +const MAX_SCALE_SET_CAPACITY = 2_147_483_647; +const DEFAULT_BOOT_TIMEOUT_MINUTES = 10; +const MAX_BOOT_TIMEOUT_MINUTES = 120; +const DEFAULT_HEALTH_PORT = 8080; +const DEFAULT_HEALTH_STALE_AFTER_SECONDS = 180; +const DEFAULT_SHUTDOWN_TIMEOUT_SECONDS = 110; +const DEFAULT_SESSION_CLOSE_TIMEOUT_SECONDS = 10; +const DEFAULT_RECONNECT_INITIAL_BACKOFF_SECONDS = 1; +const DEFAULT_RECONNECT_MAX_BACKOFF_SECONDS = 30; +const MAX_RECONCILERS = 1000; +const MAX_PROVIDER_CONFIG_NODES = 10_000; +const MAX_PROVIDER_CONFIG_DEPTH = 32; +const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +const SAFE_PROVIDER_TYPE = /^[a-z][a-z0-9_-]{0,63}$/; +const SAFE_SSM_PARAMETER = /^\/[A-Za-z0-9_.\-/]{1,2047}$/; +const PROTOTYPE_KEYS = new Set(['__proto__', 'constructor', 'prototype']); + +export class ScaleSetConfigurationError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'ScaleSetConfigurationError'; + } +} + +function parseInteger( + environment: ScaleSetServiceEnvironment, + name: string, + options: { defaultValue: number; minimum: number; maximum: number }, +): number { + const raw = environment[name]?.trim(); + if (!raw) return options.defaultValue; + if (!/^\d+$/.test(raw)) throw new ScaleSetConfigurationError(`${name} must be an integer`); + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < options.minimum || value > options.maximum) { + throw new ScaleSetConfigurationError(`${name} must be between ${options.minimum} and ${options.maximum}`); + } + return value; +} + +export function parseScaleSetServiceConfig(environment: ScaleSetServiceEnvironment): ScaleSetServiceConfig { + const manifest = environment.SCALE_SET_CONTROLLER_MANIFEST?.trim(); + const groupConfigPath = environment.SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH?.trim(); + if ((manifest === undefined || manifest === '') === (groupConfigPath === undefined || groupConfigPath === '')) { + throw new ScaleSetConfigurationError( + 'provide exactly one of SCALE_SET_CONTROLLER_MANIFEST or SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH', + ); + } + if (manifest !== undefined && Buffer.byteLength(manifest, 'utf8') > MAX_MANIFEST_BYTES) { + throw new ScaleSetConfigurationError(`SCALE_SET_CONTROLLER_MANIFEST must not exceed ${MAX_MANIFEST_BYTES} bytes`); + } + let groupName: string | undefined; + let groupRevision: string | undefined; + if (groupConfigPath !== undefined) { + validateSsmParameterName(groupConfigPath, 'group config path'); + groupName = validateSafeName(environment.SCALE_SET_CONTROLLER_GROUP_NAME?.trim() ?? '', 'group name'); + groupRevision = environment.SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION?.trim(); + if (!groupRevision || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(groupRevision)) { + throw new ScaleSetConfigurationError('SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION is invalid'); + } + } + + const reconnectInitialBackoffMs = + parseInteger(environment, 'SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS', { + defaultValue: DEFAULT_RECONNECT_INITIAL_BACKOFF_SECONDS, + minimum: 1, + maximum: 300, + }) * 1000; + const reconnectMaxBackoffMs = + parseInteger(environment, 'SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS', { + defaultValue: DEFAULT_RECONNECT_MAX_BACKOFF_SECONDS, + minimum: 1, + maximum: 3600, + }) * 1000; + if (reconnectInitialBackoffMs > reconnectMaxBackoffMs) { + throw new ScaleSetConfigurationError( + 'SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS must not exceed SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS', + ); + } + + return { + ...(manifest ? { manifest } : {}), + ...(groupConfigPath ? { groupConfigPath, groupName, groupRevision } : {}), + healthPort: parseInteger(environment, 'SCALE_SET_HEALTH_PORT', { + defaultValue: DEFAULT_HEALTH_PORT, + minimum: 1, + maximum: 65535, + }), + healthStaleAfterMs: + parseInteger(environment, 'SCALE_SET_HEALTH_STALE_AFTER_SECONDS', { + defaultValue: DEFAULT_HEALTH_STALE_AFTER_SECONDS, + minimum: 30, + maximum: 3600, + }) * 1000, + shutdownTimeoutMs: + parseInteger(environment, 'SCALE_SET_SHUTDOWN_TIMEOUT_SECONDS', { + defaultValue: DEFAULT_SHUTDOWN_TIMEOUT_SECONDS, + minimum: 1, + maximum: 300, + }) * 1000, + sessionCloseTimeoutMs: + parseInteger(environment, 'SCALE_SET_SESSION_CLOSE_TIMEOUT_SECONDS', { + defaultValue: DEFAULT_SESSION_CLOSE_TIMEOUT_SECONDS, + minimum: 1, + maximum: 60, + }) * 1000, + reconnectInitialBackoffMs, + reconnectMaxBackoffMs, + }; +} + +function objectValue(value: unknown, path: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new ScaleSetConfigurationError(`${path} must be an object`); + } + return value as Record; +} + +function exactKeys(value: Record, allowed: readonly string[], path: string): void { + const allowedSet = new Set(allowed); + const unknown = Object.keys(value).filter((key) => !allowedSet.has(key)); + if (unknown.length > 0) + throw new ScaleSetConfigurationError(`${path} contains unknown field ${JSON.stringify(unknown[0])}`); +} + +function requiredString(value: Record, key: string, path: string): string { + const result = value[key]; + if (typeof result !== 'string' || result.trim() === '') { + throw new ScaleSetConfigurationError(`${path}.${key} must be a non-empty string`); + } + return result.trim(); +} + +function optionalString(value: Record, key: string, path: string): string | undefined { + const result = value[key]; + if (result === undefined) return undefined; + if (typeof result !== 'string' || result.trim() === '') { + throw new ScaleSetConfigurationError(`${path}.${key} must be a non-empty string when set`); + } + return result.trim(); +} + +function integer(value: Record, key: string, path: string, minimum: number, maximum: number): number { + const result = value[key]; + if (!Number.isSafeInteger(result) || (result as number) < minimum || (result as number) > maximum) { + throw new ScaleSetConfigurationError(`${path}.${key} must be an integer between ${minimum} and ${maximum}`); + } + return result as number; +} + +function optionalBoolean(value: Record, key: string, path: string, fallback: boolean): boolean { + const result = value[key]; + if (result === undefined) return fallback; + if (typeof result !== 'boolean') throw new ScaleSetConfigurationError(`${path}.${key} must be a boolean`); + return result; +} + +function validateSafeName(value: string, path: string): string { + if (!SAFE_NAME.test(value)) { + throw new ScaleSetConfigurationError( + `${path} must start with an ASCII letter or digit and contain only letters, digits, dots, underscores, or hyphens`, + ); + } + return value; +} + +function validateSsmParameterName(value: string, path: string): string { + if (!SAFE_SSM_PARAMETER.test(value) || value.includes('//') || value.endsWith('/')) { + throw new ScaleSetConfigurationError(`${path} must be an absolute SSM parameter name`); + } + return value; +} + +function validateGitHubConfigUrl(raw: string, path: string): string { + let url: URL; + try { + url = new URL(raw); + } catch (error) { + throw new ScaleSetConfigurationError(`${path} must be a valid URL`, { cause: error }); + } + if (url.protocol !== 'https:') throw new ScaleSetConfigurationError(`${path} must use HTTPS`); + if (url.username || url.password || url.search || url.hash) { + throw new ScaleSetConfigurationError(`${path} must not contain credentials, a query, or a fragment`); + } + const parts = url.pathname + .replace(/^\/+|\/+$/g, '') + .split('/') + .filter(Boolean); + if (parts.length < 1 || parts.length > 2 || (parts[0].toLowerCase() === 'enterprises' && parts.length !== 2)) { + throw new ScaleSetConfigurationError(`${path} must identify a GitHub organization, repository, or enterprise`); + } + url.pathname = `/${parts.join('/')}`; + return url.toString().replace(/\/$/, ''); +} + +function validateWorkFolder(value: string, path: string): string { + if ( + value.length > 128 || + value.startsWith('/') || + value.includes('\\') || + value.split('/').some((part) => part === '' || part === '.' || part === '..') || + !/^[A-Za-z0-9._/-]+$/.test(value) + ) { + throw new ScaleSetConfigurationError(`${path} must be a safe relative path`); + } + return value; +} + +function validateUserAgent(value: string | undefined, path: string): string | undefined { + if (value === undefined) return undefined; + if (value.length > 256 || !/^[\x20-\x7E]+$/.test(value)) { + throw new ScaleSetConfigurationError(`${path} must contain at most 256 visible ASCII characters`); + } + return value; +} + +function validateJsonValue(value: unknown, path: string, depth = 0, counter = { value: 0 }): JsonValue { + counter.value += 1; + if (counter.value > MAX_PROVIDER_CONFIG_NODES || depth > MAX_PROVIDER_CONFIG_DEPTH) { + throw new ScaleSetConfigurationError(`${path} exceeds the provider configuration complexity limit`); + } + if (value === null || typeof value === 'boolean' || typeof value === 'string') return value; + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new ScaleSetConfigurationError(`${path} contains a non-finite number`); + return value; + } + if (Array.isArray(value)) + return value.map((item, index) => validateJsonValue(item, `${path}[${index}]`, depth + 1, counter)); + const record = objectValue(value, path); + const result: Record = Object.create(null) as Record; + for (const [key, child] of Object.entries(record)) { + if (PROTOTYPE_KEYS.has(key)) + throw new ScaleSetConfigurationError(`${path} contains forbidden field ${JSON.stringify(key)}`); + if (key.length === 0 || key.length > 128) + throw new ScaleSetConfigurationError(`${path} contains an invalid field name`); + result[key] = validateJsonValue(child, `${path}.${key}`, depth + 1, counter); + } + return result; +} + +function parseGitHubApp(value: unknown, path: string): GitHubAppParameterReferences { + const record = objectValue(value, path); + exactKeys(record, ['appIdParameterName', 'installationIdParameterName', 'privateKeyParameterName'], path); + const installationIdParameterName = optionalString(record, 'installationIdParameterName', path); + return { + appIdParameterName: validateSsmParameterName( + requiredString(record, 'appIdParameterName', path), + `${path}.appIdParameterName`, + ), + ...(installationIdParameterName === undefined + ? {} + : { + installationIdParameterName: validateSsmParameterName( + installationIdParameterName, + `${path}.installationIdParameterName`, + ), + }), + privateKeyParameterName: validateSsmParameterName( + requiredString(record, 'privateKeyParameterName', path), + `${path}.privateKeyParameterName`, + ), + }; +} + +function parseComputeProvider(value: unknown, path: string): ScaleSetReconcilerConfig['computeProvider'] { + const record = objectValue(value, path); + exactKeys(record, ['type', 'configuration'], path); + const type = requiredString(record, 'type', path); + if (!SAFE_PROVIDER_TYPE.test(type)) throw new ScaleSetConfigurationError(`${path}.type is invalid`); + const configuration = validateJsonValue(record.configuration, `${path}.configuration`); + if (typeof configuration !== 'object' || configuration === null || Array.isArray(configuration)) { + throw new ScaleSetConfigurationError(`${path}.configuration must be an object`); + } + return { type, configuration }; +} + +export function parseScaleSetReconcilerConfig( + value: unknown, + index: number, + groupName: string, + basePath = 'manifest.reconcilers', +): ScaleSetReconcilerConfig { + const path = `${basePath}[${index}]`; + const record = objectValue(value, path); + exactKeys( + record, + [ + 'schemaVersion', + 'runnerConfigName', + 'scaleSetId', + 'scaleSetName', + 'expectedScaleSetName', + 'runnerGroupName', + 'runnerGroupIdParameterName', + 'expectedRunnerGroupId', + 'githubConfigUrl', + 'githubApp', + 'computeProvider', + 'minRunners', + 'maxRunners', + 'bootTimeoutMinutes', + 'sessionOwner', + 'workFolder', + 'forceGhes', + 'sslVerify', + 'userAgent', + ], + path, + ); + const runnerConfigName = validateSafeName( + requiredString(record, 'runnerConfigName', path), + `${path}.runnerConfigName`, + ); + const minRunners = integer(record, 'minRunners', path, 0, MAX_SCALE_SET_CAPACITY); + const maxRunners = integer(record, 'maxRunners', path, 0, MAX_SCALE_SET_CAPACITY); + const bootTimeoutMinutes = + record.bootTimeoutMinutes === undefined + ? DEFAULT_BOOT_TIMEOUT_MINUTES + : integer(record, 'bootTimeoutMinutes', path, 1, MAX_BOOT_TIMEOUT_MINUTES); + if (minRunners > maxRunners) throw new ScaleSetConfigurationError(`${path}.minRunners must not exceed maxRunners`); + const sessionOwner = optionalString(record, 'sessionOwner', path) ?? defaultSessionOwner(groupName, runnerConfigName); + if (!/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/.test(sessionOwner)) { + throw new ScaleSetConfigurationError(`${path}.sessionOwner is invalid`); + } + const userAgent = validateUserAgent(optionalString(record, 'userAgent', path), `${path}.userAgent`); + if (record.schemaVersion !== 1) throw new ScaleSetConfigurationError(`${path}.schemaVersion must be 1`); + const expectedRunnerGroupId = + record.expectedRunnerGroupId === undefined || record.expectedRunnerGroupId === null + ? undefined + : integer(record, 'expectedRunnerGroupId', path, 1, MAX_SCALE_SET_CAPACITY); + const runnerGroupName = + record.runnerGroupName === undefined + ? undefined + : validateScaleSetName(requiredString(record, 'runnerGroupName', path), `${path}.runnerGroupName`); + const runnerGroupIdParameterName = + record.runnerGroupIdParameterName === undefined + ? undefined + : validateSsmParameterName( + requiredString(record, 'runnerGroupIdParameterName', path), + `${path}.runnerGroupIdParameterName`, + ); + if (record.scaleSetName !== undefined && record.expectedScaleSetName !== undefined) { + throw new ScaleSetConfigurationError(`${path} must configure only one of scaleSetName or expectedScaleSetName`); + } + const scaleSetName = validateScaleSetName( + requiredString(record, record.scaleSetName === undefined ? 'expectedScaleSetName' : 'scaleSetName', path), + `${path}.scaleSetName`, + ); + const scaleSetId = + record.scaleSetId === undefined ? undefined : integer(record, 'scaleSetId', path, 1, MAX_SCALE_SET_CAPACITY); + if (scaleSetId === undefined && runnerGroupName === undefined) { + throw new ScaleSetConfigurationError(`${path}.runnerGroupName is required when scaleSetId is omitted`); + } + return { + schemaVersion: 1, + runnerConfigName, + ...(scaleSetId === undefined ? {} : { scaleSetId }), + scaleSetName, + ...(runnerGroupName === undefined ? {} : { runnerGroupName }), + ...(runnerGroupIdParameterName === undefined ? {} : { runnerGroupIdParameterName }), + ...(expectedRunnerGroupId === undefined ? {} : { expectedRunnerGroupId }), + githubConfigUrl: validateGitHubConfigUrl( + requiredString(record, 'githubConfigUrl', path), + `${path}.githubConfigUrl`, + ), + githubApp: parseGitHubApp(record.githubApp, `${path}.githubApp`), + computeProvider: parseComputeProvider(record.computeProvider, `${path}.computeProvider`), + minRunners, + maxRunners, + bootTimeoutMinutes, + sessionOwner, + workFolder: validateWorkFolder(optionalString(record, 'workFolder', path) ?? '_work', `${path}.workFolder`), + forceGhes: optionalBoolean(record, 'forceGhes', path, false), + sslVerify: optionalBoolean(record, 'sslVerify', path, true), + ...(userAgent === undefined ? {} : { userAgent }), + }; +} + +function defaultSessionOwner(groupName: string, runnerConfigName: string): string { + const candidate = `${groupName}.${runnerConfigName}`; + if (candidate.length <= 256) return candidate; + const suffix = createHash('sha256').update(candidate).digest('hex').slice(0, 16); + return `${candidate.slice(0, 239)}.${suffix}`; +} + +function validateScaleSetName(value: string, path: string): string { + if (value.length > 128 || !/^[\x20-\x7E]+$/.test(value)) { + throw new ScaleSetConfigurationError(`${path} must contain at most 128 visible ASCII characters`); + } + return value; +} + +export function parseScaleSetControllerManifest(input: string | unknown): ScaleSetControllerManifest { + let parsed = input; + if (typeof input === 'string') { + if (Buffer.byteLength(input, 'utf8') > MAX_MANIFEST_BYTES) { + throw new ScaleSetConfigurationError(`controller manifest must not exceed ${MAX_MANIFEST_BYTES} bytes`); + } + try { + parsed = JSON.parse(input) as unknown; + } catch (error) { + throw new ScaleSetConfigurationError('controller manifest must contain valid JSON', { cause: error }); + } + } + const manifest = objectValue(parsed, 'manifest'); + exactKeys(manifest, ['version', 'groupName', 'revision', 'reconcilers'], 'manifest'); + if (manifest.version !== SCALE_SET_CONTROLLER_MANIFEST_VERSION) { + throw new ScaleSetConfigurationError(`manifest.version must be ${SCALE_SET_CONTROLLER_MANIFEST_VERSION}`); + } + const groupName = validateSafeName(requiredString(manifest, 'groupName', 'manifest'), 'manifest.groupName'); + if ( + !Array.isArray(manifest.reconcilers) || + manifest.reconcilers.length < 1 || + manifest.reconcilers.length > MAX_RECONCILERS + ) { + throw new ScaleSetConfigurationError(`manifest.reconcilers must contain between 1 and ${MAX_RECONCILERS} entries`); + } + const revision = optionalString(manifest, 'revision', 'manifest'); + if (revision !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(revision)) { + throw new ScaleSetConfigurationError('manifest.revision is invalid'); + } + const reconcilers = manifest.reconcilers.map((value, index) => + parseScaleSetReconcilerConfig(value, index, groupName), + ); + validateUniqueReconcilers(reconcilers); + return { + version: SCALE_SET_CONTROLLER_MANIFEST_VERSION, + groupName, + ...(revision === undefined ? {} : { revision }), + reconcilers, + }; +} + +export function validateUniqueReconcilers(reconcilers: readonly ScaleSetReconcilerConfig[]): void { + const names = new Set(); + const scopedScaleSets = new Set(); + for (const reconciler of reconcilers) { + if (names.has(reconciler.runnerConfigName)) { + throw new ScaleSetConfigurationError( + `runner config ${JSON.stringify(reconciler.runnerConfigName)} is duplicated`, + ); + } + const scopedScaleSet = [ + reconciler.githubConfigUrl, + reconciler.runnerGroupName ?? String(reconciler.expectedRunnerGroupId ?? ''), + reconciler.scaleSetName, + ].join('\u0000'); + if (scopedScaleSets.has(scopedScaleSet)) { + throw new ScaleSetConfigurationError( + `scale set ${JSON.stringify(reconciler.scaleSetName)} is duplicated within GitHub scope ${JSON.stringify(reconciler.githubConfigUrl)}`, + ); + } + names.add(reconciler.runnerConfigName); + scopedScaleSets.add(scopedScaleSet); + } +} diff --git a/lambdas/services/scale-set/src/controller.ts b/lambdas/services/scale-set/src/controller.ts new file mode 100644 index 0000000000..5cbd2ac229 --- /dev/null +++ b/lambdas/services/scale-set/src/controller.ts @@ -0,0 +1,51 @@ +import type { ScaleSetControllerManifest, ScaleSetServiceConfig } from './config'; +import { ScaleSetControllerHealth } from './health'; +import type { ScaleSetLogger } from './logger'; +import { ScaleSetReconciler, type ScaleSetReconcilerDependencies } from './reconciler'; + +export class ScaleSetController { + readonly health: ScaleSetControllerHealth; + + constructor( + private readonly manifest: ScaleSetControllerManifest, + private readonly serviceConfig: ScaleSetServiceConfig, + private readonly dependencies: ScaleSetReconcilerDependencies, + private readonly controllerLogger: ScaleSetLogger, + ) { + this.health = new ScaleSetControllerHealth( + manifest.groupName, + manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + serviceConfig.healthStaleAfterMs, + ); + } + + async run(signal: AbortSignal): Promise { + this.controllerLogger.debug('scale_set_reconcilers_starting', { + reconcilerCount: this.manifest.reconcilers.length, + runnerConfigNames: this.manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + }); + const completions = this.manifest.reconcilers.map(async (config) => { + const status = this.health.reporter(config.runnerConfigName); + try { + await new ScaleSetReconciler(config, this.serviceConfig, this.dependencies).run(signal, status); + } catch (error) { + status.markFailed(error); + this.controllerLogger.error('scale_set_reconciler_uncaught_failure', { + runnerConfigName: config.runnerConfigName, + scaleSetId: config.scaleSetId, + error, + }); + } + }); + + await Promise.race([Promise.all(completions), waitForAbort(signal)]); + if (!signal.aborted) await waitForAbort(signal); + this.health.markStopping(); + await Promise.all(completions); + } +} + +async function waitForAbort(signal: AbortSignal): Promise { + if (signal.aborted) return; + await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); +} diff --git a/lambdas/services/scale-set/src/credentials.test.ts b/lambdas/services/scale-set/src/credentials.test.ts new file mode 100644 index 0000000000..4536a8e582 --- /dev/null +++ b/lambdas/services/scale-set/src/credentials.test.ts @@ -0,0 +1,159 @@ +const authMocks = vi.hoisted(() => ({ createAppAuth: vi.fn(), requestDefaults: vi.fn() })); +vi.mock('@octokit/auth-app', () => ({ createAppAuth: authMocks.createAppAuth })); +vi.mock('@octokit/request', () => ({ request: { defaults: authMocks.requestDefaults } })); + +import type { ScaleSetFetch } from '@aws-github-runner/github-actions-scale-set'; + +import { createGitHubAppAccessTokenProvider, loadGitHubAppCredentials, type ParameterStore } from './credentials'; + +const references = { + appIdParameterName: '/app/id', + privateKeyParameterName: '/app/key', + installationIdParameterName: '/app/installation', +}; + +function encodedKey(body: string): string { + return Buffer.from(`-----BEGIN PRIVATE KEY-----\n${body}\n-----END PRIVATE KEY-----\n`).toString('base64'); +} + +describe('GitHub App credentials', () => { + beforeEach(() => { + vi.clearAllMocks(); + authMocks.requestDefaults.mockReturnValue(vi.fn()); + }); + + it('validates and decodes referenced SSM values', async () => { + const store: ParameterStore = { + get: vi.fn().mockResolvedValue( + new Map([ + ['/app/id', '123'], + ['/app/installation', '456'], + ['/app/key', encodedKey('abc')], + ]), + ), + put: vi.fn(), + }; + await expect(loadGitHubAppCredentials(references, store)).resolves.toMatchObject({ + appId: '123', + installationId: 456, + privateKey: expect.stringContaining('BEGIN PRIVATE KEY'), + }); + }); + + it('reuses auth for unchanged credentials and recreates it after rotation', async () => { + const values = [encodedKey('first'), encodedKey('first'), encodedKey('second')]; + const store: ParameterStore = { + get: vi.fn( + async () => + new Map([ + ['/app/id', '123'], + ['/app/installation', '456'], + ['/app/key', values.shift() as string], + ]), + ), + }; + const firstAuth = vi.fn().mockResolvedValue({ token: 'token-one', expiresAt: '2099-01-01T00:00:00Z' }); + const secondAuth = vi.fn().mockResolvedValue({ token: 'token-two', expiresAt: '2099-01-01T00:00:00Z' }); + const fetchImplementation = vi.fn(); + authMocks.createAppAuth.mockReturnValueOnce(firstAuth).mockReturnValueOnce(secondAuth); + + const provider = await createGitHubAppAccessTokenProvider( + references, + 'https://github.com/example', + false, + store, + fetchImplementation, + ); + await provider(); + await provider(); + await provider(); + + expect(store.get).toHaveBeenCalledTimes(3); + expect(authMocks.createAppAuth).toHaveBeenCalledTimes(2); + expect(authMocks.requestDefaults).toHaveBeenCalledWith({ + baseUrl: 'https://api.github.com', + request: { fetch: fetchImplementation }, + }); + expect(firstAuth).toHaveBeenCalledTimes(2); + expect(secondAuth).toHaveBeenCalledTimes(1); + }); + + it('discovers the installation ID from the configured organization when SSM does not provide one', async () => { + const store: ParameterStore = { + get: vi.fn().mockResolvedValue( + new Map([ + ['/app/id', '123'], + ['/app/key', encodedKey('abc')], + ]), + ), + put: vi.fn(), + }; + const appAuth = vi.fn().mockResolvedValue({ token: 'app-jwt' }); + const installationAuth = vi + .fn() + .mockResolvedValue({ token: 'installation-token', expiresAt: '2099-01-01T00:00:00Z' }); + const fetchImplementation = vi.fn().mockResolvedValue( + new Response(JSON.stringify([{ id: 456, account: { login: 'example' } }]), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + authMocks.createAppAuth.mockReturnValueOnce(appAuth).mockReturnValueOnce(installationAuth); + + const provider = await createGitHubAppAccessTokenProvider( + references, + 'https://github.com/example', + false, + store, + fetchImplementation, + ); + + await expect(provider()).resolves.toMatchObject({ token: 'installation-token' }); + await expect(provider()).resolves.toMatchObject({ token: 'installation-token' }); + expect(appAuth).toHaveBeenCalledWith({ type: 'app' }); + expect(installationAuth).toHaveBeenCalledWith({ type: 'installation', installationId: 456 }); + expect(appAuth).toHaveBeenCalledTimes(1); + expect(fetchImplementation).toHaveBeenCalledTimes(1); + expect(store.put).toHaveBeenCalledWith('/app/installation', '456'); + expect(fetchImplementation).toHaveBeenCalledWith( + expect.objectContaining({ + href: 'https://api.github.com/app/installations?per_page=100&page=1', + }), + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: 'Bearer app-jwt' }), + }), + ); + }); + + it.each([ + [new Map([['/app/id', '123']]), 'was not returned'], + [ + new Map([ + ['/app/id', 'bad id'], + ['/app/installation', '1'], + ['/app/key', encodedKey('abc')], + ]), + 'App ID', + ], + [ + new Map([ + ['/app/id', '1'], + ['/app/installation', 'zero'], + ['/app/key', encodedKey('abc')], + ]), + 'positive integer', + ], + [ + new Map([ + ['/app/id', '1'], + ['/app/installation', '2'], + ['/app/key', 'not-base64'], + ]), + 'canonical base64', + ], + ])('rejects malformed credential parameters', async (values, message) => { + await expect(loadGitHubAppCredentials(references, { get: vi.fn().mockResolvedValue(values) })).rejects.toThrow( + message, + ); + }); +}); diff --git a/lambdas/services/scale-set/src/credentials.ts b/lambdas/services/scale-set/src/credentials.ts new file mode 100644 index 0000000000..f1178efff2 --- /dev/null +++ b/lambdas/services/scale-set/src/credentials.ts @@ -0,0 +1,209 @@ +import { createAppAuth } from '@octokit/auth-app'; +import { request } from '@octokit/request'; +import { createHash } from 'node:crypto'; + +import { + githubApiUrl, + parseGitHubConfigUrl, + type AccessToken, + type ScaleSetFetch, +} from '@aws-github-runner/github-actions-scale-set'; + +import { ScaleSetConfigurationError, type GitHubAppParameterReferences } from './config'; + +export interface ParameterStore { + get(names: readonly string[]): Promise>; + put?(name: string, value: string): Promise; +} + +interface GitHubAppCredentials { + appId: string; + installationId?: number; + privateKey: string; +} + +interface GitHubAppInstallation { + id?: unknown; + account?: { login?: unknown }; +} + +const MAX_PRIVATE_KEY_BYTES = 64 * 1024; + +function requiredParameter(values: ReadonlyMap, name: string): string { + const value = values.get(name); + if (value === undefined || value === '') { + throw new ScaleSetConfigurationError(`required SSM parameter ${JSON.stringify(name)} was not returned`); + } + return value; +} + +function decodePrivateKey(encoded: string): string { + if ( + encoded.length === 0 || + encoded.length > Math.ceil((MAX_PRIVATE_KEY_BYTES * 4) / 3) + 4 || + encoded.length % 4 !== 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encoded) + ) { + throw new ScaleSetConfigurationError('GitHub App private key parameter must contain canonical base64'); + } + const decoded = Buffer.from(encoded, 'base64').toString('utf8').replace(/\\n/g, '\n'); + if (Buffer.byteLength(decoded, 'utf8') > MAX_PRIVATE_KEY_BYTES) { + throw new ScaleSetConfigurationError('GitHub App private key is too large'); + } + if (!/^-----BEGIN (?:RSA )?PRIVATE KEY-----\n[\s\S]+\n-----END (?:RSA )?PRIVATE KEY-----\n?$/.test(decoded)) { + throw new ScaleSetConfigurationError('GitHub App private key parameter is not a supported PEM private key'); + } + return decoded; +} + +export async function loadGitHubAppCredentials( + references: GitHubAppParameterReferences, + parameterStore: ParameterStore, +): Promise { + const names = [references.appIdParameterName, references.privateKeyParameterName]; + if (references.installationIdParameterName !== undefined) names.splice(1, 0, references.installationIdParameterName); + const values = await parameterStore.get(names); + const appId = requiredParameter(values, references.appIdParameterName).trim(); + if (!/^[A-Za-z0-9_-]{1,128}$/.test(appId)) { + throw new ScaleSetConfigurationError('GitHub App ID parameter is invalid'); + } + let installationId: number | undefined; + if (references.installationIdParameterName !== undefined) { + const installationIdRaw = values.get(references.installationIdParameterName)?.trim(); + if (installationIdRaw !== undefined && installationIdRaw !== '') { + if (!/^\d+$/.test(installationIdRaw)) { + throw new ScaleSetConfigurationError('GitHub App installation ID parameter must be a positive integer'); + } + installationId = Number(installationIdRaw); + if (!Number.isSafeInteger(installationId) || installationId <= 0) { + throw new ScaleSetConfigurationError('GitHub App installation ID parameter must be a positive integer'); + } + } + } + return { + appId, + installationId, + privateKey: decodePrivateKey(requiredParameter(values, references.privateKeyParameterName)), + }; +} + +async function discoverGitHubAppInstallationId( + credentials: Pick, + target: string, + apiBaseUrl: string, + fetchImplementation: ScaleSetFetch, +): Promise { + const appRequest = request.defaults({ baseUrl: apiBaseUrl, request: { fetch: fetchImplementation } }); + const appAuth = createAppAuth({ + appId: credentials.appId, + privateKey: credentials.privateKey, + request: appRequest, + }); + const appAuthentication = await appAuth({ type: 'app' }); + for (let page = 1; page <= 100; page += 1) { + const url = new URL('/app/installations', `${apiBaseUrl}/`); + url.searchParams.set('per_page', '100'); + url.searchParams.set('page', String(page)); + const response = await fetchImplementation(url, { + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${appAuthentication.token}`, + 'User-Agent': 'github-aws-runners/scale-set-controller', + }, + }); + if (!response.ok) { + throw new ScaleSetConfigurationError(`GitHub App installation discovery failed with HTTP ${response.status}`); + } + let payload: unknown; + try { + payload = await response.json(); + } catch (error) { + throw new ScaleSetConfigurationError('GitHub App installation discovery returned invalid JSON', { cause: error }); + } + if (!Array.isArray(payload)) { + throw new ScaleSetConfigurationError('GitHub App installation discovery returned an invalid response'); + } + for (const value of payload as unknown[]) { + if (typeof value !== 'object' || value === null || Array.isArray(value)) continue; + const candidate = value as GitHubAppInstallation; + if ( + Number.isSafeInteger(candidate.id) && + typeof candidate.account?.login === 'string' && + candidate.account.login.toLowerCase() === target.toLowerCase() + ) { + return candidate.id as number; + } + } + if (payload.length < 100) break; + } + throw new ScaleSetConfigurationError(`GitHub App is not installed for ${JSON.stringify(target)}`); +} + +export async function createGitHubAppAccessTokenProvider( + references: GitHubAppParameterReferences, + githubConfigUrl: string, + forceGhes: boolean, + parameterStore: ParameterStore, + fetchImplementation: ScaleSetFetch = globalThis.fetch, +): Promise<() => Promise> { + const parsedConfig = parseGitHubConfigUrl(githubConfigUrl, forceGhes); + const apiBaseUrl = githubApiUrl(parsedConfig, '/').toString().replace(/\/$/, ''); + let cached: + | { + fingerprint: string; + installationId: number; + auth: ReturnType; + } + | undefined; + let discovered: + | { + fingerprint: string; + installationId: number; + } + | undefined; + + return async () => { + // Reload references for rotation visibility, but preserve the Octokit auth + // instance while credentials are unchanged so its installation-token cache + // remains effective. + const credentials = await loadGitHubAppCredentials(references, parameterStore); + const credentialFingerprint = createHash('sha256') + .update(credentials.appId) + .update('\u0000') + .update(credentials.privateKey) + .digest('base64url'); + const target = parsedConfig.organization ?? parsedConfig.enterprise; + if (credentials.installationId === undefined && target === undefined) { + throw new ScaleSetConfigurationError( + 'GitHub App installation discovery requires an organization or enterprise URL', + ); + } + let installationId = credentials.installationId; + if (installationId === undefined) { + if (discovered?.fingerprint === credentialFingerprint) { + installationId = discovered.installationId; + } else { + installationId = await discoverGitHubAppInstallationId(credentials, target!, apiBaseUrl, fetchImplementation); + discovered = { fingerprint: credentialFingerprint, installationId }; + if (references.installationIdParameterName !== undefined && parameterStore.put !== undefined) { + await parameterStore.put(references.installationIdParameterName, String(installationId)); + } + } + } + const fingerprint = `${credentialFingerprint}\u0000${installationId}`; + if (cached?.fingerprint !== fingerprint) { + cached = { + fingerprint, + installationId, + auth: createAppAuth({ + appId: credentials.appId, + installationId, + privateKey: credentials.privateKey, + request: request.defaults({ baseUrl: apiBaseUrl, request: { fetch: fetchImplementation } }), + }), + }; + } + const installation = await cached.auth({ type: 'installation', installationId: cached.installationId }); + return { token: installation.token, expiresAt: installation.expiresAt }; + }; +} diff --git a/lambdas/services/scale-set/src/github-http.test.ts b/lambdas/services/scale-set/src/github-http.test.ts new file mode 100644 index 0000000000..870399ac84 --- /dev/null +++ b/lambdas/services/scale-set/src/github-http.test.ts @@ -0,0 +1,58 @@ +const undiciMocks = vi.hoisted(() => ({ + close: vi.fn().mockResolvedValue(undefined), + createAgent: vi.fn(), +})); + +vi.mock('undici', () => ({ + Agent: class MockAgent { + constructor(options: unknown) { + undiciMocks.createAgent(options); + } + + close = undiciMocks.close; + }, +})); + +import type { ScaleSetFetch } from '@aws-github-runner/github-actions-scale-set'; + +import { createScaleSetGitHubHttp } from './github-http'; + +describe('scale-set GitHub HTTP isolation', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('uses the supplied verified fetch without changing process TLS settings', async () => { + const original = process.env.NODE_TLS_REJECT_UNAUTHORIZED; + const fetchImplementation = vi.fn().mockResolvedValue(new Response(null, { status: 204 })); + const http = createScaleSetGitHubHttp(fetchImplementation); + + await http.fetch(true)('https://github.example/_apis/runtime/runnerscalesets'); + await http.close(); + + expect(fetchImplementation).toHaveBeenCalledWith('https://github.example/_apis/runtime/runnerscalesets'); + expect(undiciMocks.createAgent).not.toHaveBeenCalled(); + expect(process.env.NODE_TLS_REJECT_UNAUTHORIZED).toBe(original); + }); + + it('uses one scoped insecure dispatcher and closes it without mutating global TLS state', async () => { + const original = process.env.NODE_TLS_REJECT_UNAUTHORIZED; + const fetchImplementation = vi.fn().mockResolvedValue(new Response(null, { status: 204 })); + const http = createScaleSetGitHubHttp(fetchImplementation); + const first = http.fetch(false); + const second = http.fetch(false); + + await first('https://github.example/_apis/runtime/runnerscalesets', { method: 'GET' }); + await http.close(); + + expect(first).toBe(second); + expect(undiciMocks.createAgent).toHaveBeenCalledOnce(); + expect(undiciMocks.createAgent).toHaveBeenCalledWith({ connect: { rejectUnauthorized: false } }); + expect(fetchImplementation).toHaveBeenCalledWith( + 'https://github.example/_apis/runtime/runnerscalesets', + expect.objectContaining({ method: 'GET', dispatcher: expect.anything() }), + ); + expect(undiciMocks.close).toHaveBeenCalledOnce(); + expect(process.env.NODE_TLS_REJECT_UNAUTHORIZED).toBe(original); + }); +}); diff --git a/lambdas/services/scale-set/src/github-http.ts b/lambdas/services/scale-set/src/github-http.ts new file mode 100644 index 0000000000..85a60626b1 --- /dev/null +++ b/lambdas/services/scale-set/src/github-http.ts @@ -0,0 +1,34 @@ +import { Agent, type Dispatcher } from 'undici'; + +import type { ScaleSetFetch } from '@aws-github-runner/github-actions-scale-set'; + +type DispatcherRequestInit = RequestInit & { dispatcher: Dispatcher }; + +export interface ScaleSetGitHubHttp { + fetch(sslVerify: boolean): ScaleSetFetch; + close(): Promise; +} + +/** + * Creates fetch implementations whose TLS policy is scoped to one controller + * process. Disabling verification never mutates NODE_TLS_REJECT_UNAUTHORIZED + * or the global Undici dispatcher, so verified and unverified GHES runner + * configurations may safely share one grouped task. + */ +export function createScaleSetGitHubHttp(fetchImplementation: ScaleSetFetch = globalThis.fetch): ScaleSetGitHubHttp { + let insecureAgent: Agent | undefined; + let insecureFetch: ScaleSetFetch | undefined; + + return { + fetch(sslVerify) { + if (sslVerify) return fetchImplementation; + insecureAgent ??= new Agent({ connect: { rejectUnauthorized: false } }); + insecureFetch ??= async (input, init = {}) => + await fetchImplementation(input, { ...init, dispatcher: insecureAgent } as DispatcherRequestInit); + return insecureFetch; + }, + async close() { + await insecureAgent?.close(); + }, + }; +} diff --git a/lambdas/services/scale-set/src/health-server.test.ts b/lambdas/services/scale-set/src/health-server.test.ts new file mode 100644 index 0000000000..16e65ab953 --- /dev/null +++ b/lambdas/services/scale-set/src/health-server.test.ts @@ -0,0 +1,15 @@ +import { startScaleSetHealthServer } from './health-server'; + +describe('health server', () => { + it('separates liveness and readiness on loopback', async () => { + const health = { snapshot: vi.fn(() => ({ live: true, ready: false, state: 'degraded' })) }; + const server = await startScaleSetHealthServer(health, 0); + try { + await expect(fetch(`http://127.0.0.1:${server.port}/healthz`)).resolves.toMatchObject({ status: 200 }); + await expect(fetch(`http://127.0.0.1:${server.port}/readyz`)).resolves.toMatchObject({ status: 503 }); + await expect(fetch(`http://127.0.0.1:${server.port}/other`)).resolves.toMatchObject({ status: 404 }); + } finally { + await server.close(); + } + }); +}); diff --git a/lambdas/services/scale-set/src/health-server.ts b/lambdas/services/scale-set/src/health-server.ts new file mode 100644 index 0000000000..f9b967aaec --- /dev/null +++ b/lambdas/services/scale-set/src/health-server.ts @@ -0,0 +1,53 @@ +import { createServer, type Server } from 'node:http'; + +import type { ScaleSetControllerHealth } from './health'; + +export interface ScaleSetHealthServer { + port: number; + close(): Promise; +} + +export async function startScaleSetHealthServer( + health: Pick, + port: number, +): Promise { + const server = createServer((request, response) => { + response.setHeader('Cache-Control', 'no-store'); + response.setHeader('Connection', 'close'); + response.setHeader('Content-Type', 'application/json; charset=utf-8'); + response.setHeader('X-Content-Type-Options', 'nosniff'); + + if (request.method !== 'GET' || (request.url !== '/healthz' && request.url !== '/readyz')) { + response.statusCode = 404; + response.end(JSON.stringify({ status: 'not-found' })); + return; + } + + const snapshot = health.snapshot(); + const healthy = request.url === '/readyz' ? snapshot.ready : snapshot.live; + response.statusCode = healthy ? 200 : 503; + response.end(JSON.stringify(snapshot)); + }); + await listen(server, port); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('health server did not bind a TCP address'); + return { + port: address.port, + close: async () => { + server.closeAllConnections(); + if (!server.listening) return; + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + }, + }; +} + +async function listen(server: Server, port: number): Promise { + await new Promise((resolve, reject) => { + const onError = (error: Error) => reject(error); + server.once('error', onError); + server.listen(port, '127.0.0.1', () => { + server.removeListener('error', onError); + resolve(); + }); + }); +} diff --git a/lambdas/services/scale-set/src/health.test.ts b/lambdas/services/scale-set/src/health.test.ts new file mode 100644 index 0000000000..b2d5294e11 --- /dev/null +++ b/lambdas/services/scale-set/src/health.test.ts @@ -0,0 +1,46 @@ +import { ScaleSetControllerHealth } from './health'; + +describe('ScaleSetControllerHealth', () => { + it('aggregates independent readiness while reconnect heartbeats stay live', () => { + let now = 0; + const health = new ScaleSetControllerHealth('group', ['a', 'b'], 100, () => now); + const a = health.reporter('a'); + const b = health.reporter('b'); + a.markSessionReady(); + b.markSessionReady(); + a.markProgress(); + b.markProgress(); + expect(health.snapshot()).toMatchObject({ state: 'ready', live: true, ready: true }); + + now = 200; + expect(health.snapshot()).toMatchObject({ state: 'degraded', live: true, ready: false }); + expect(health.snapshot().reconcilers.a).toMatchObject({ state: 'ready', live: true, ready: false }); + + a.markReconnecting(new Error('outage')); + expect(health.snapshot()).toMatchObject({ state: 'degraded', live: true, ready: false }); + expect(health.snapshot().reconcilers.a).toMatchObject({ state: 'reconnecting', live: true, ready: false }); + }); + + it('contains one terminal reconciler failure while another stays ready', () => { + const health = new ScaleSetControllerHealth('group', ['a', 'b'], 100); + health.reporter('a').markFailed(new TypeError('bad config')); + health.reporter('b').markProgress(); + expect(health.snapshot()).toMatchObject({ state: 'degraded', live: true, ready: false }); + expect(health.snapshot().reconcilers.a).toMatchObject({ state: 'failed', lastErrorName: 'TypeError' }); + }); + + it('marks all reporters stopping without reviving failures', () => { + const health = new ScaleSetControllerHealth('group', ['a'], 100); + health.reporter('a').markFailed(); + health.markStopping(); + expect(health.snapshot()).toMatchObject({ state: 'stopping', live: true, ready: false }); + expect(health.snapshot().reconcilers.a.state).toBe('failed'); + }); + + it('rejects duplicate and unknown reporter names', () => { + expect(() => new ScaleSetControllerHealth('g', [], 1)).toThrow('at least one'); + expect(() => new ScaleSetControllerHealth('g', ['a', 'a'], 1)).toThrow('duplicate'); + const health = new ScaleSetControllerHealth('g', ['a'], 1); + expect(() => health.reporter('b')).toThrow('unknown runner config'); + }); +}); diff --git a/lambdas/services/scale-set/src/health.ts b/lambdas/services/scale-set/src/health.ts new file mode 100644 index 0000000000..6cad43d10d --- /dev/null +++ b/lambdas/services/scale-set/src/health.ts @@ -0,0 +1,137 @@ +export type ScaleSetReconcilerState = 'starting' | 'ready' | 'reconnecting' | 'failed' | 'stopping'; +export type ScaleSetControllerState = 'starting' | 'ready' | 'degraded' | 'failed' | 'stopping'; + +export interface ScaleSetReconcilerHealthSnapshot { + state: ScaleSetReconcilerState; + live: boolean; + ready: boolean; + lastActivityAt: string; + consecutiveFailures: number; + lastErrorName?: string; +} + +export interface ScaleSetControllerHealthSnapshot { + groupName: string; + state: ScaleSetControllerState; + live: boolean; + ready: boolean; + reconcilers: Readonly>; +} + +export interface ScaleSetReconcilerStatusReporter { + markSessionReady(): void; + markProgress(): void; + markReconnecting(error?: unknown): void; + markFailed(error?: unknown): void; + markStopping(): void; +} + +interface MutableHealth { + state: ScaleSetReconcilerState; + lastActivityAt: number; + consecutiveFailures: number; + lastErrorName?: string; +} + +function errorName(error: unknown): string | undefined { + if (error === undefined) return undefined; + return error instanceof Error ? error.name : typeof error; +} + +export class ScaleSetControllerHealth { + private readonly states = new Map(); + private stopping = false; + + constructor( + readonly groupName: string, + runnerConfigNames: readonly string[], + private readonly staleAfterMs: number, + private readonly now: () => number = Date.now, + ) { + const startedAt = now(); + for (const name of runnerConfigNames) { + if (this.states.has(name)) throw new Error(`duplicate health reporter for ${JSON.stringify(name)}`); + this.states.set(name, { state: 'starting', lastActivityAt: startedAt, consecutiveFailures: 0 }); + } + if (this.states.size === 0) throw new Error('at least one reconciler health reporter is required'); + } + + reporter(runnerConfigName: string): ScaleSetReconcilerStatusReporter { + const state = this.states.get(runnerConfigName); + if (!state) throw new Error(`unknown runner config ${JSON.stringify(runnerConfigName)}`); + return { + markSessionReady: () => { + if (state.state === 'starting') { + state.state = 'ready'; + state.lastActivityAt = this.now(); + } else if (state.state === 'reconnecting') { + state.state = 'ready'; + } + }, + markProgress: () => { + if (state.state === 'failed' || state.state === 'stopping') return; + state.state = 'ready'; + state.lastActivityAt = this.now(); + state.consecutiveFailures = 0; + state.lastErrorName = undefined; + }, + markReconnecting: (error) => { + if (state.state === 'failed' || state.state === 'stopping') return; + state.state = 'reconnecting'; + state.lastActivityAt = this.now(); + state.consecutiveFailures += 1; + state.lastErrorName = errorName(error); + }, + markFailed: (error) => { + if (state.state === 'stopping') return; + state.state = 'failed'; + state.consecutiveFailures += 1; + state.lastErrorName = errorName(error); + }, + markStopping: () => { + if (state.state !== 'failed') state.state = 'stopping'; + }, + }; + } + + markStopping(): void { + this.stopping = true; + for (const name of this.states.keys()) this.reporter(name).markStopping(); + } + + snapshot(): ScaleSetControllerHealthSnapshot { + const now = this.now(); + const reconcilers: Record = Object.create(null) as Record< + string, + ScaleSetReconcilerHealthSnapshot + >; + for (const [name, state] of this.states) { + const stale = now - state.lastActivityAt > this.staleAfterMs; + // Staleness means the reconciler is not ready, but it is not a process + // liveness failure. A single bounded GitHub/AWS request can legitimately + // outlive the readiness window; restarting the task would only churn its + // message sessions and reset the provider retry policy. + const live = state.state === 'stopping' || state.state !== 'failed'; + reconcilers[name] = { + state: state.state, + live, + ready: state.state === 'ready' && !stale, + lastActivityAt: new Date(state.lastActivityAt).toISOString(), + consecutiveFailures: state.consecutiveFailures, + ...(state.lastErrorName === undefined ? {} : { lastErrorName: state.lastErrorName }), + }; + } + const values = Object.values(reconcilers); + const liveCount = values.filter(({ live }) => live).length; + const readyCount = values.filter(({ ready }) => ready).length; + const live = this.stopping || liveCount > 0; + const ready = !this.stopping && readyCount === values.length; + let state: ScaleSetControllerState; + if (this.stopping) state = 'stopping'; + else if (ready) state = 'ready'; + else if (liveCount === 0) state = 'failed'; + else if (values.every((value) => value.state === 'starting')) state = 'starting'; + else state = 'degraded'; + return { groupName: this.groupName, state, live, ready, reconcilers }; + } +} diff --git a/lambdas/services/scale-set/src/index.ts b/lambdas/services/scale-set/src/index.ts new file mode 100644 index 0000000000..6074b49499 --- /dev/null +++ b/lambdas/services/scale-set/src/index.ts @@ -0,0 +1,9 @@ +export * from './config'; +export * from './controller'; +export * from './credentials'; +export * from './health'; +export * from './health-server'; +export * from './lifecycle'; +export * from './logger'; +export * from './parameter-store'; +export * from './reconciler'; diff --git a/lambdas/services/scale-set/src/lifecycle.test.ts b/lambdas/services/scale-set/src/lifecycle.test.ts new file mode 100644 index 0000000000..a43c31fe5c --- /dev/null +++ b/lambdas/services/scale-set/src/lifecycle.test.ts @@ -0,0 +1,45 @@ +import { ScaleSetServiceRuntime } from './lifecycle'; + +describe('ScaleSetServiceRuntime', () => { + it('starts once and performs idempotent bounded shutdown', async () => { + const health = { markStopping: vi.fn(), snapshot: vi.fn() }; + const run = vi.fn(async (signal: AbortSignal) => { + if (!signal.aborted) + await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); + }); + const runtime = new ScaleSetServiceRuntime({ shutdownTimeoutMs: 100 }, { run, health } as never); + const completion = runtime.run(); + expect(() => runtime.run()).toThrow('already started'); + const shutdown = runtime.shutdown(); + expect(runtime.shutdown()).toBe(shutdown); + await shutdown; + await completion; + expect(health.markStopping).toHaveBeenCalledOnce(); + }); + + it('allows shutdown before start and prevents a later start', async () => { + const health = { markStopping: vi.fn() }; + const runtime = new ScaleSetServiceRuntime({ shutdownTimeoutMs: 100 }, { run: vi.fn(), health } as never); + await runtime.shutdown(); + expect(() => runtime.run()).toThrow('already stopping'); + }); + + it('rejects when a controller ignores cancellation past the timeout', async () => { + vi.useFakeTimers(); + try { + const health = { markStopping: vi.fn() }; + const runtime = new ScaleSetServiceRuntime({ shutdownTimeoutMs: 10 }, { + run: vi.fn(() => new Promise(() => undefined)), + health, + } as never); + void runtime.run(); + await Promise.resolve(); + const shutdown = runtime.shutdown(); + const expectation = expect(shutdown).rejects.toThrow('did not stop within 10ms'); + await vi.advanceTimersByTimeAsync(10); + await expectation; + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/lambdas/services/scale-set/src/lifecycle.ts b/lambdas/services/scale-set/src/lifecycle.ts new file mode 100644 index 0000000000..d4c4a600c0 --- /dev/null +++ b/lambdas/services/scale-set/src/lifecycle.ts @@ -0,0 +1,51 @@ +import type { ScaleSetServiceConfig } from './config'; +import type { ScaleSetController } from './controller'; + +export class ScaleSetServiceRuntime { + private readonly abortController = new AbortController(); + private completion: Promise | undefined; + private shutdownCompletion: Promise | undefined; + + constructor( + private readonly config: Pick, + private readonly controller: Pick, + ) {} + + get health() { + return this.controller.health; + } + + run(): Promise { + if (this.shutdownCompletion !== undefined) throw new Error('Scale-set service runtime is already stopping'); + if (this.completion !== undefined) throw new Error('Scale-set service runtime has already started'); + this.completion = Promise.resolve().then(async () => { + if (!this.abortController.signal.aborted) await this.controller.run(this.abortController.signal); + }); + return this.completion; + } + + shutdown(reason: unknown = new Error('Scale-set service shutdown requested')): Promise { + this.shutdownCompletion ??= this.shutdownOnce(reason); + return this.shutdownCompletion; + } + + private async shutdownOnce(reason: unknown): Promise { + this.controller.health.markStopping(); + this.abortController.abort(reason); + if (this.completion === undefined) return; + let timeout: ReturnType | undefined; + try { + await Promise.race([ + this.completion, + new Promise((_resolve, reject) => { + timeout = setTimeout( + () => reject(new Error(`Scale-set controller did not stop within ${this.config.shutdownTimeoutMs}ms`)), + this.config.shutdownTimeoutMs, + ); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } + } +} diff --git a/lambdas/services/scale-set/src/logger.test.ts b/lambdas/services/scale-set/src/logger.test.ts new file mode 100644 index 0000000000..5e58b47d57 --- /dev/null +++ b/lambdas/services/scale-set/src/logger.test.ts @@ -0,0 +1,56 @@ +import { createScaleSetLogger, logger, sanitizeLogAttributes } from './logger'; +import { ScaleSetConfigurationError } from './config'; + +describe('redacted structured logging', () => { + it('emits debug records when LOG_LEVEL is debug', () => { + const spy = vi.spyOn(console, 'debug').mockImplementation(() => undefined); + createScaleSetLogger({ LOG_LEVEL: 'debug' }).debug('debug_event', { reconcilerCount: 2 }); + expect(spy).toHaveBeenCalledWith(expect.stringContaining('"event":"debug_event"')); + spy.mockRestore(); + }); + + it('does not emit debug records at the default info level', () => { + const spy = vi.spyOn(console, 'debug').mockImplementation(() => undefined); + createScaleSetLogger({}).debug('hidden_debug_event'); + expect(spy).not.toHaveBeenCalled(); + spy.mockRestore(); + }); + + it('redacts nested secrets and strips log-injection characters', () => { + expect( + sanitizeLogAttributes({ + runnerConfig: 'linux\nforged', + privateKey: 'secret', + nested: { authorization: 'Bearer secret', safe: 'ok' }, + }), + ).toEqual({ + runnerConfig: 'linux forged', + privateKey: '[REDACTED]', + nested: { authorization: '[REDACTED]', safe: 'ok' }, + }); + }); + + it('logs errors without their potentially sensitive message', () => { + const spy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + logger.error('failed', { error: new Error('token=secret') }); + expect(spy).toHaveBeenCalledOnce(); + expect(spy.mock.calls[0][0]).not.toContain('token=secret'); + expect(JSON.parse(spy.mock.calls[0][0] as string)).toMatchObject({ level: 'error', event: 'failed' }); + spy.mockRestore(); + }); + + it('includes safe configuration error messages for diagnosis', () => { + expect( + sanitizeLogAttributes({ + error: new ScaleSetConfigurationError( + 'provide exactly one of SCALE_SET_CONTROLLER_MANIFEST or SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH', + ), + }), + ).toEqual({ + error: { + name: 'ScaleSetConfigurationError', + message: 'provide exactly one of SCALE_SET_CONTROLLER_MANIFEST or SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH', + }, + }); + }); +}); diff --git a/lambdas/services/scale-set/src/logger.ts b/lambdas/services/scale-set/src/logger.ts new file mode 100644 index 0000000000..639f8a2177 --- /dev/null +++ b/lambdas/services/scale-set/src/logger.ts @@ -0,0 +1,87 @@ +const REDACTED = '[REDACTED]'; +const SENSITIVE_KEY = /(authorization|credential|encodedjit|jitconfig|password|private.?key|secret|sessionid|token)/i; +const SAFE_ERROR_MESSAGE_NAMES = new Set(['ScaleSetConfigurationError']); +const MAX_LOG_STRING_LENGTH = 1024; +const MAX_LOG_DEPTH = 4; +const LOG_LEVEL_PRIORITY = { debug: 10, info: 20, warn: 30, error: 40 } as const; + +export type ScaleSetLogLevel = keyof typeof LOG_LEVEL_PRIORITY; + +export interface ScaleSetLogger { + debug(event: string, attributes?: Readonly>): void; + info(event: string, attributes?: Readonly>): void; + warn(event: string, attributes?: Readonly>): void; + error(event: string, attributes?: Readonly>): void; +} + +function sanitizeString(value: string): string { + return value.replace(/[\r\n\u2028\u2029]/g, ' ').slice(0, MAX_LOG_STRING_LENGTH); +} + +function sanitize(value: unknown, key: string, depth: number): unknown { + if (SENSITIVE_KEY.test(key)) return REDACTED; + if (depth > MAX_LOG_DEPTH) return '[TRUNCATED]'; + if (value === null || typeof value === 'boolean' || typeof value === 'number') return value; + if (typeof value === 'string') return sanitizeString(value); + if (value instanceof Error) { + const status = 'status' in value && typeof value.status === 'number' ? value.status : undefined; + const code = 'code' in value && typeof value.code === 'string' ? sanitizeString(value.code) : undefined; + const message = SAFE_ERROR_MESSAGE_NAMES.has(value.name) ? sanitizeString(value.message) : undefined; + return { + name: sanitizeString(value.name), + ...(message ? { message } : {}), + ...(status === undefined ? {} : { status }), + ...(code ? { code } : {}), + }; + } + if (Array.isArray(value)) return value.slice(0, 100).map((item) => sanitize(item, key, depth + 1)); + if (typeof value === 'object') { + const result: Record = Object.create(null) as Record; + for (const [childKey, childValue] of Object.entries(value).slice(0, 100)) { + result[sanitizeString(childKey)] = sanitize(childValue, childKey, depth + 1); + } + return result; + } + return sanitizeString(typeof value); +} + +export function sanitizeLogAttributes(attributes: Readonly> = {}): Record { + return sanitize(attributes, '', 0) as Record; +} + +function parseLogLevel(value: string | undefined): ScaleSetLogLevel { + return value !== undefined && value in LOG_LEVEL_PRIORITY ? (value as ScaleSetLogLevel) : 'info'; +} + +function write( + level: ScaleSetLogLevel, + minimumLevel: ScaleSetLogLevel, + event: string, + attributes?: Readonly>, +): void { + if (LOG_LEVEL_PRIORITY[level] < LOG_LEVEL_PRIORITY[minimumLevel]) return; + const record = JSON.stringify({ + timestamp: new Date().toISOString(), + level, + event: sanitizeString(event), + ...sanitizeLogAttributes(attributes), + }); + if (level === 'error') console.error(record); + else if (level === 'warn') console.warn(record); + else if (level === 'info') console.info(record); + else console.debug(record); +} + +export function createScaleSetLogger( + environment: Readonly> = process.env, +): ScaleSetLogger { + const minimumLevel = parseLogLevel(environment.LOG_LEVEL); + return { + debug: (event, attributes) => write('debug', minimumLevel, event, attributes), + info: (event, attributes) => write('info', minimumLevel, event, attributes), + warn: (event, attributes) => write('warn', minimumLevel, event, attributes), + error: (event, attributes) => write('error', minimumLevel, event, attributes), + }; +} + +export const logger = createScaleSetLogger(); diff --git a/lambdas/services/scale-set/src/main.ts b/lambdas/services/scale-set/src/main.ts new file mode 100644 index 0000000000..b97cb112e3 --- /dev/null +++ b/lambdas/services/scale-set/src/main.ts @@ -0,0 +1,100 @@ +import { GitHubActionsScaleSetClient } from '@aws-github-runner/github-actions-scale-set'; +import { createScaleSetComputeProviderRegistry } from '@aws-github-runner/compute-providers/scale-set'; + +import { parseScaleSetServiceConfig } from './config'; +import { ScaleSetController } from './controller'; +import { createGitHubAppAccessTokenProvider } from './credentials'; +import { createScaleSetGitHubHttp } from './github-http'; +import { startScaleSetHealthServer, type ScaleSetHealthServer } from './health-server'; +import { ScaleSetServiceRuntime } from './lifecycle'; +import { logger } from './logger'; +import { createDefaultControllerManifestLoader, defaultParameterStore } from './parameter-store'; +import { abortableSleep, type ScaleSetReconcilerDependencies } from './reconciler'; + +async function main(): Promise { + logger.info('scale_set_controller_configuration_loading', { + manifestConfigured: Boolean(process.env.SCALE_SET_CONTROLLER_MANIFEST?.trim()), + groupConfigConfigured: Boolean(process.env.SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH?.trim()), + }); + const serviceConfig = parseScaleSetServiceConfig(process.env); + const manifest = await createDefaultControllerManifestLoader().load(serviceConfig); + logger.info('scale_set_controller_manifest_loaded', { + groupName: manifest.groupName, + revision: manifest.revision, + reconcilerCount: manifest.reconcilers.length, + runnerConfigNames: manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + }); + logger.debug('scale_set_controller_reconcilers_loaded', { + groupName: manifest.groupName, + reconcilerCount: manifest.reconcilers.length, + runnerConfigNames: manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + }); + const computeProviders = createScaleSetComputeProviderRegistry(); + const githubHttp = createScaleSetGitHubHttp(); + const dependencies: ScaleSetReconcilerDependencies = { + computeProviders, + createAccessTokenProvider: async (config) => + await createGitHubAppAccessTokenProvider( + config.githubApp, + config.githubConfigUrl, + config.forceGhes, + defaultParameterStore, + githubHttp.fetch(config.sslVerify), + ), + createClient: (config, accessTokenProvider) => + new GitHubActionsScaleSetClient({ + gitHubConfigUrl: config.githubConfigUrl, + accessTokenProvider, + fetch: githubHttp.fetch(config.sslVerify), + forceGhes: config.forceGhes, + systemInfo: { + system: config.userAgent ?? 'github-aws-runners', + version: '1', + scaleSetId: config.scaleSetId ?? 0, + subsystem: 'scale-set-controller', + }, + }), + logger, + parameterStore: defaultParameterStore, + sleep: abortableSleep, + random: Math.random, + closeSignal: AbortSignal.timeout, + }; + const controller = new ScaleSetController(manifest, serviceConfig, dependencies, logger); + const runtime = new ScaleSetServiceRuntime(serviceConfig, controller); + let healthServer: ScaleSetHealthServer | undefined; + + const shutdown = (signal: NodeJS.Signals) => { + logger.info('scale_set_controller_shutdown_requested', { signal, groupName: manifest.groupName }); + void runtime.shutdown(new Error(`received ${signal}`)).catch((error) => { + logger.error('scale_set_controller_shutdown_failed', { error, groupName: manifest.groupName }); + process.exitCode = 1; + }); + }; + const onSigterm = () => shutdown('SIGTERM'); + const onSigint = () => shutdown('SIGINT'); + process.once('SIGTERM', onSigterm); + process.once('SIGINT', onSigint); + + try { + healthServer = await startScaleSetHealthServer(runtime.health, serviceConfig.healthPort); + logger.info('scale_set_controller_started', { + groupName: manifest.groupName, + revision: manifest.revision, + reconcilerCount: manifest.reconcilers.length, + healthPort: healthServer.port, + }); + await runtime.run(); + } finally { + await runtime.shutdown().catch(() => undefined); + await healthServer?.close(); + await githubHttp.close(); + process.removeListener('SIGTERM', onSigterm); + process.removeListener('SIGINT', onSigint); + } +} + +void main().catch((error) => { + logger.error('scale_set_controller_fatal_failure', { error }); + process.exitCode = 1; +}); diff --git a/lambdas/services/scale-set/src/parameter-store.test.ts b/lambdas/services/scale-set/src/parameter-store.test.ts new file mode 100644 index 0000000000..f18cc2ef7e --- /dev/null +++ b/lambdas/services/scale-set/src/parameter-store.test.ts @@ -0,0 +1,78 @@ +import { createControllerManifestLoader, type ParametersByPathClient } from './parameter-store'; +import type { ScaleSetServiceConfig } from './config'; + +function leaf(name: string, id: number): string { + return JSON.stringify({ + schemaVersion: 1, + runnerConfigName: name, + githubConfigUrl: 'https://github.com/example', + scaleSetId: id, + expectedScaleSetName: name, + expectedRunnerGroupId: null, + minRunners: 0, + maxRunners: 10, + sslVerify: true, + githubApp: { + appIdParameterName: '/app/id', + privateKeyParameterName: '/app/key', + installationIdParameterName: '/app/installation', + }, + computeProvider: { type: 'ec2', configuration: {} }, + }); +} + +const config: ScaleSetServiceConfig = { + groupName: 'group', + groupConfigPath: '/groups/group', + groupRevision: 'rev-1', + healthPort: 8080, + healthStaleAfterMs: 1000, + shutdownTimeoutMs: 1000, + sessionCloseTimeoutMs: 1000, + reconnectInitialBackoffMs: 100, + reconnectMaxBackoffMs: 1000, +}; + +describe('createControllerManifestLoader', () => { + it('paginates direct children, sorts them, and returns a versioned group', async () => { + const send = vi + .fn() + .mockResolvedValueOnce({ Parameters: [{ Name: '/groups/group/b', Value: leaf('b', 2) }], NextToken: 'next' }) + .mockResolvedValueOnce({ Parameters: [{ Name: '/groups/group/a', Value: leaf('a', 1) }] }); + const manifest = await createControllerManifestLoader({ send } as ParametersByPathClient).load(config); + expect(manifest).toMatchObject({ version: 1, groupName: 'group', revision: 'rev-1' }); + expect(manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName)).toEqual(['a', 'b']); + expect(send).toHaveBeenCalledTimes(2); + }); + + it('uses the injected inline manifest loader path for local tests', async () => { + const inline = JSON.stringify({ version: 1, groupName: 'local', reconcilers: [JSON.parse(leaf('a', 1))] }); + const send = vi.fn(); + await expect( + createControllerManifestLoader({ send } as ParametersByPathClient).load({ ...config, manifest: inline }), + ).resolves.toMatchObject({ + groupName: 'local', + }); + expect(send).not.toHaveBeenCalled(); + }); + + it.each([ + [{ Parameters: [] }, 'contains no runner configs'], + [{ Parameters: [{ Name: '/groups/group/nested/a', Value: leaf('a', 1) }] }, 'outside the direct group path'], + [{ Parameters: [{ Name: '/groups/group/wrong', Value: leaf('a', 1) }] }, 'must match runnerConfigName'], + [{ Parameters: [{ Name: '/groups/group/a', Value: '{' }] }, 'contains invalid JSON'], + [{ Parameters: [{ Name: undefined, Value: leaf('a', 1) }] }, 'incomplete parameter'], + ])('rejects malformed SSM group pages', async (page, message) => { + await expect( + createControllerManifestLoader({ send: vi.fn().mockResolvedValue(page) }).load(config), + ).rejects.toThrow(message); + }); + + it('rejects repeated pagination tokens', async () => { + const send = vi + .fn() + .mockResolvedValueOnce({ Parameters: [{ Name: '/groups/group/a', Value: leaf('a', 1) }], NextToken: 'same' }) + .mockResolvedValueOnce({ Parameters: [], NextToken: 'same' }); + await expect(createControllerManifestLoader({ send }).load(config)).rejects.toThrow('repeated token'); + }); +}); diff --git a/lambdas/services/scale-set/src/parameter-store.ts b/lambdas/services/scale-set/src/parameter-store.ts new file mode 100644 index 0000000000..3df49818b8 --- /dev/null +++ b/lambdas/services/scale-set/src/parameter-store.ts @@ -0,0 +1,141 @@ +import { GetParametersByPathCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; + +import { getParameters, ssmClient } from '@aws-github-runner/aws-ssm-util'; + +import { + MAX_MANIFEST_BYTES, + SCALE_SET_CONTROLLER_MANIFEST_VERSION, + ScaleSetConfigurationError, + parseScaleSetControllerManifest, + parseScaleSetReconcilerConfig, + validateUniqueReconcilers, + type ScaleSetControllerManifest, + type ScaleSetServiceConfig, +} from './config'; +import type { ParameterStore } from './credentials'; + +const MAX_GROUP_PARAMETERS = 1000; +const MAX_PARAMETER_BYTES = 64 * 1024; +const MAX_GROUP_BYTES = 4 * 1024 * 1024; + +export const defaultParameterStore: ParameterStore = { + get: async (names) => await getParameters([...names]), + put: async (name, value) => { + await ssmClient().send( + new PutParameterCommand({ + Name: name, + Value: value, + Type: 'String', + Overwrite: true, + }), + ); + }, +}; + +export interface ControllerManifestLoader { + load(config: ScaleSetServiceConfig): Promise; +} + +export interface ParametersByPathClient { + send(command: GetParametersByPathCommand): Promise<{ + Parameters?: Array<{ Name?: string; Value?: string }>; + NextToken?: string; + }>; +} + +export function createControllerManifestLoader(client: ParametersByPathClient): ControllerManifestLoader { + return { + load: async (config) => { + if (config.manifest !== undefined) return parseScaleSetControllerManifest(config.manifest); + if (!config.groupConfigPath || !config.groupName || !config.groupRevision) { + throw new ScaleSetConfigurationError('SSM group configuration source is incomplete'); + } + const prefix = `${config.groupConfigPath.replace(/\/$/, '')}/`; + const parameters: Array<{ name: string; value: string }> = []; + const seenTokens = new Set(); + let nextToken: string | undefined; + let totalBytes = 0; + do { + if (nextToken !== undefined && seenTokens.has(nextToken)) { + throw new ScaleSetConfigurationError('SSM pagination returned a repeated token'); + } + if (nextToken !== undefined) seenTokens.add(nextToken); + const response = await client.send( + new GetParametersByPathCommand({ + Path: config.groupConfigPath, + Recursive: false, + WithDecryption: false, + MaxResults: 10, + ...(nextToken === undefined ? {} : { NextToken: nextToken }), + }), + ); + for (const parameter of response.Parameters ?? []) { + if (!parameter.Name || parameter.Value === undefined) { + throw new ScaleSetConfigurationError('SSM group configuration returned an incomplete parameter'); + } + if (!parameter.Name.startsWith(prefix) || parameter.Name.slice(prefix.length).includes('/')) { + throw new ScaleSetConfigurationError( + 'SSM group configuration returned a parameter outside the direct group path', + ); + } + const size = Buffer.byteLength(parameter.Value, 'utf8'); + if (size > MAX_PARAMETER_BYTES || size > MAX_MANIFEST_BYTES) { + throw new ScaleSetConfigurationError( + `runner config parameter ${JSON.stringify(parameter.Name)} is too large`, + ); + } + totalBytes += size; + if (totalBytes > MAX_GROUP_BYTES) + throw new ScaleSetConfigurationError('SSM controller group configuration is too large'); + parameters.push({ name: parameter.Name, value: parameter.Value }); + if (parameters.length > MAX_GROUP_PARAMETERS) { + throw new ScaleSetConfigurationError(`SSM controller group exceeds ${MAX_GROUP_PARAMETERS} runner configs`); + } + } + nextToken = response.NextToken; + } while (nextToken !== undefined); + + if (parameters.length === 0) + throw new ScaleSetConfigurationError('SSM controller group contains no runner configs'); + parameters.sort((left, right) => left.name.localeCompare(right.name)); + const reconcilers = parameters.map(({ name, value }, index) => { + let parsed: unknown; + try { + parsed = JSON.parse(value) as unknown; + } catch (error) { + throw new ScaleSetConfigurationError( + `runner config parameter ${JSON.stringify(name)} contains invalid JSON`, + { + cause: error, + }, + ); + } + const reconciler = parseScaleSetReconcilerConfig(parsed, index, config.groupName as string, 'ssmRunnerConfigs'); + const leafName = name.slice(prefix.length); + if (leafName !== reconciler.runnerConfigName) { + throw new ScaleSetConfigurationError( + `runner config parameter ${JSON.stringify(name)} must match runnerConfigName ${JSON.stringify(reconciler.runnerConfigName)}`, + ); + } + return reconciler; + }); + validateUniqueReconcilers(reconcilers); + return { + version: SCALE_SET_CONTROLLER_MANIFEST_VERSION, + groupName: config.groupName, + revision: config.groupRevision, + reconcilers, + }; + }, + }; +} + +export function createDefaultControllerManifestLoader(): ControllerManifestLoader { + return createControllerManifestLoader( + new SSMClient({ + region: process.env.AWS_REGION, + maxAttempts: 10, + retryMode: 'adaptive', + }), + ); +} diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts new file mode 100644 index 0000000000..e9de83afaf --- /dev/null +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -0,0 +1,441 @@ +import type { MessageSessionClient, RunnerScaleSetMessage } from '@aws-github-runner/github-actions-scale-set'; +import type { ScaleSetComputeProvider, ScaleSetReconcileResult } from '@aws-github-runner/compute-providers/scale-set'; +import { describe, expect, it, vi } from 'vitest'; + +import type { ScaleSetReconcilerConfig, ScaleSetServiceConfig } from './config'; +import type { ScaleSetReconcilerStatusReporter } from './health'; +import { + ScaleSetReconciler, + calculateDesiredRunners, + validateProviderResult, + type ScaleSetReconcilerClient, + type ScaleSetReconcilerDependencies, +} from './reconciler'; + +const config: ScaleSetReconcilerConfig = { + schemaVersion: 1, + runnerConfigName: 'linux', + scaleSetId: 42, + scaleSetName: 'linux', + githubConfigUrl: 'https://github.com/example', + githubApp: { + appIdParameterName: '/app/id', + privateKeyParameterName: '/app/key', + installationIdParameterName: '/app/installation', + }, + computeProvider: { type: 'ec2', configuration: {} }, + minRunners: 0, + maxRunners: 10, + bootTimeoutMinutes: 10, + sessionOwner: 'group.linux', + workFolder: '_work', + forceGhes: false, +}; + +const serviceConfig: Pick< + ScaleSetServiceConfig, + 'sessionCloseTimeoutMs' | 'reconnectInitialBackoffMs' | 'reconnectMaxBackoffMs' +> = { sessionCloseTimeoutMs: 100, reconnectInitialBackoffMs: 1, reconnectMaxBackoffMs: 10 }; + +function result(overrides: Partial = {}): ScaleSetReconcileResult { + return { + status: 'converged', + desiredRunners: 1, + currentRunners: 1, + actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + ...overrides, + }; +} + +function reporter(): ScaleSetReconcilerStatusReporter { + return { + markSessionReady: vi.fn(), + markProgress: vi.fn(), + markReconnecting: vi.fn(), + markFailed: vi.fn(), + markStopping: vi.fn(), + }; +} + +function message(): RunnerScaleSetMessage { + return { + messageId: 7, + statistics: { + totalAvailableJobs: 1, + totalAcquiredJobs: 0, + totalAssignedJobs: 1, + totalRunningJobs: 0, + totalRegisteredRunners: 1, + totalBusyRunners: 0, + totalIdleRunners: 1, + }, + jobAvailableMessages: [{ runnerRequestId: 99 } as RunnerScaleSetMessage['jobAvailableMessages'][number]], + jobAssignedMessages: [], + jobStartedMessages: [ + { runnerId: 5, runnerName: 'runner-5' } as RunnerScaleSetMessage['jobStartedMessages'][number], + ], + jobCompletedMessages: [], + }; +} + +function fixture(options: { + session: Partial & { session: MessageSessionClient['session'] }; + reconcile?: ScaleSetComputeProvider['reconcile']; +}) { + const computeProvider: ScaleSetComputeProvider = { + reconcile: options.reconcile ?? vi.fn().mockResolvedValue(result()), + }; + const client: ScaleSetReconcilerClient = { + getRunnerScaleSetById: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), + getRunnerScaleSet: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), + createRunnerScaleSet: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), + getRunnerGroupByName: vi.fn().mockResolvedValue({ + id: 7, + name: 'runner-group', + size: 0, + isDefaultGroup: false, + }), + createMessageSessionClient: vi.fn().mockResolvedValue(options.session as MessageSessionClient), + generateJitRunnerConfig: vi.fn(), + getRunnerByName: vi.fn(), + removeRunner: vi.fn(), + systemInfo: { scaleSetId: 42 }, + setSystemInfo: vi.fn(), + }; + const dependencies: ScaleSetReconcilerDependencies = { + createAccessTokenProvider: vi.fn().mockResolvedValue(async () => ({ token: 'not-a-real-token' })), + createClient: vi.fn().mockReturnValue(client), + computeProviders: { create: vi.fn().mockReturnValue(computeProvider) }, + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + sleep: vi.fn(async (_delay, signal) => { + if (!signal.aborted) + await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); + }), + random: () => 0, + closeSignal: () => new AbortController().signal, + parameterStore: { get: vi.fn().mockResolvedValue(new Map()), put: vi.fn() }, + }; + return { client, computeProvider, dependencies }; +} + +describe('ScaleSetReconciler', () => { + it('resolves the GitHub runner-group and scale-set IDs from their names', async () => { + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + deleteMessage: vi.fn(), + acquireJobs: vi.fn(), + close: vi.fn(), + }; + const { client, dependencies } = fixture({ + session, + reconcile: vi.fn(async () => { + abort.abort(); + return result(); + }), + }); + vi.mocked(client.getRunnerScaleSet).mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }); + + await new ScaleSetReconciler( + { + ...config, + scaleSetId: undefined, + runnerGroupName: 'runner-group', + runnerGroupIdParameterName: '/runner/group-id', + scaleSetName: 'linux', + }, + serviceConfig, + dependencies, + ).run(abort.signal, reporter()); + + expect(client.getRunnerGroupByName).toHaveBeenCalledWith('runner-group', { signal: abort.signal }); + expect(client.getRunnerScaleSet).toHaveBeenCalledWith(7, 'linux', { signal: abort.signal }); + expect(client.setSystemInfo).toHaveBeenCalledWith(expect.objectContaining({ scaleSetId: 42 })); + expect(dependencies.parameterStore.put).toHaveBeenCalledWith('/runner/group-id', '7'); + expect(dependencies.logger.info).toHaveBeenCalledWith( + 'scale_set_compute_provider_created', + expect.objectContaining({ computeProviderType: 'ec2' }), + ); + expect(dependencies.logger.info).toHaveBeenCalledWith( + 'scale_set_compute_provider_reconcile_started', + expect.objectContaining({ computeProviderType: 'ec2', desiredRunners: 1, busyRunners: 0 }), + ); + }); + + it('registers a missing scale set in the resolved runner group', async () => { + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + deleteMessage: vi.fn(), + acquireJobs: vi.fn(), + close: vi.fn(), + }; + const { client, dependencies } = fixture({ + session, + reconcile: vi.fn(async () => { + abort.abort(); + return result(); + }), + }); + vi.mocked(client.getRunnerScaleSet).mockResolvedValueOnce(null); + + await new ScaleSetReconciler( + { ...config, scaleSetId: undefined, runnerGroupName: 'runner-group', scaleSetName: 'linux' }, + serviceConfig, + dependencies, + ).run(abort.signal, reporter()); + + expect(client.createRunnerScaleSet).toHaveBeenCalledWith( + { + name: 'linux', + runnerGroupId: 7, + labels: [{ name: 'linux' }], + runnerSetting: {}, + }, + { signal: abort.signal }, + ); + }); + + it('acknowledges before acquisition, lifecycle observation, and reconciliation', async () => { + const order: string[] = []; + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn(async () => { + order.push('acquire'); + return [99]; + }), + deleteMessage: vi.fn(async () => { + order.push('delete'); + }), + close: vi.fn(), + }; + const reconcile = vi.fn(async (request) => { + order.push('reconcile'); + expect(request.busyRunners).toBe(0); + expect(request.bootTimeoutMinutes).toBe(10); + expect(request.runnerStates).toContainEqual( + expect.objectContaining({ runnerId: 5, runnerName: 'runner-5', lifecycle: 'started' }), + ); + abort.abort(); + return result(); + }); + const { dependencies } = fixture({ session, reconcile }); + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); + expect(order).toEqual(['delete', 'acquire', 'reconcile']); + expect(session.deleteMessage).toHaveBeenCalledWith(7, { signal: abort.signal }); + expect(dependencies.computeProviders.create).toHaveBeenCalledWith('ec2', { + runnerConfigName: 'linux', + scaleSetId: 42, + githubScope: 'https://github.com/example', + configuration: {}, + }); + }); + + it('acknowledges and stops when reconciliation rejects', async () => { + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn().mockResolvedValue([99]), + deleteMessage: vi.fn(), + close: vi.fn(), + }; + const { dependencies } = fixture({ session, reconcile: vi.fn().mockRejectedValue(new Error('provider failed')) }); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(session.deleteMessage).toHaveBeenCalledOnce(); + expect(status.markFailed).toHaveBeenCalledWith( + expect.objectContaining({ name: 'ScaleSetProviderReconciliationError' }), + ); + expect(status.markReconnecting).not.toHaveBeenCalled(); + expect(dependencies.sleep).not.toHaveBeenCalled(); + }); + + it('does not process a message when acknowledgement fails', async () => { + const abort = new AbortController(); + const reconcile = vi.fn(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn(), + deleteMessage: vi.fn().mockRejectedValue(new Error('acknowledgement failed')), + close: vi.fn(), + }; + const { dependencies } = fixture({ session, reconcile }); + dependencies.sleep = vi.fn(async () => abort.abort()); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(session.deleteMessage).toHaveBeenCalledOnce(); + expect(session.acquireJobs).not.toHaveBeenCalled(); + expect(reconcile).not.toHaveBeenCalled(); + expect(status.markReconnecting).toHaveBeenCalledOnce(); + }); + + it('acknowledges and stops when the provider returns an error result', async () => { + const abort = new AbortController(); + const order: string[] = []; + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn().mockResolvedValue([99]), + deleteMessage: vi.fn(async () => { + order.push('delete'); + }), + close: vi.fn(), + }; + const reconcile = vi.fn(async () => { + order.push('reconcile'); + return result({ + status: 'error', + currentRunners: 0, + errors: [{ operation: 'launch', code: 'ThrottlingException' }], + }); + }); + const { dependencies } = fixture({ session, reconcile }); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(order).toEqual(['delete', 'reconcile']); + expect(status.markFailed).toHaveBeenCalledWith( + expect.objectContaining({ name: 'ScaleSetProviderReconciliationError' }), + ); + expect(status.markReconnecting).not.toHaveBeenCalled(); + expect(dependencies.sleep).not.toHaveBeenCalled(); + }); + + it('does not call public GitHub runner inventory after a provider error result', async () => { + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn().mockResolvedValue([99]), + deleteMessage: vi.fn(), + close: vi.fn(), + }; + const reconcile = vi.fn().mockResolvedValue( + result({ + status: 'error', + currentRunners: 0, + errors: [{ operation: 'launch', code: 'EC2_LAUNCH_FAILED' }], + }), + ); + const { dependencies } = fixture({ session, reconcile }); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(new AbortController().signal, status); + + expect(reconcile).toHaveBeenCalledOnce(); + expect(status.markFailed).toHaveBeenCalledWith( + expect.objectContaining({ name: 'ScaleSetProviderReconciliationError' }), + ); + }); + + it('uses the Actions-service identity check without public runner verification', async () => { + const abort = new AbortController(); + const order: string[] = []; + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn(async () => { + order.push('acquire'); + return [99]; + }), + deleteMessage: vi.fn(async () => { + order.push('delete'); + }), + close: vi.fn(), + }; + const reconcile = vi.fn(async (request) => { + order.push('reconcile'); + await expect(request.removeRunner({ runnerId: 5, runnerName: 'runner-5', scaleSetId: 42 })).resolves.toEqual({ + status: 'removed', + }); + abort.abort(); + return result({ + status: 'retained', + currentRunners: 2, + actions: { launched: 0, terminated: 0, retainedBusy: 1, retainedUnknown: 0 }, + }); + }); + const { client, dependencies } = fixture({ session, reconcile }); + vi.mocked(client.getRunnerByName).mockImplementation(async () => { + order.push('actions-refetch'); + return { id: 5, name: 'runner-5', runnerScaleSetId: 42 }; + }); + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); + + expect(order).toEqual(['delete', 'acquire', 'reconcile', 'actions-refetch']); + expect(client.removeRunner).toHaveBeenCalledWith(5, { signal: abort.signal }); + expect(session.deleteMessage).toHaveBeenCalledOnce(); + expect(session.acquireJobs).toHaveBeenCalledTimes(1); + }); + + it('bounds the lifecycle cache per reconciler', () => { + const { dependencies } = fixture({ session: { session: {}, close: vi.fn() } }); + const reconciler = new ScaleSetReconciler(config, serviceConfig, dependencies) as unknown as { + rememberLifecycle(id: number, name: string, lifecycle: 'started'): void; + lifecycle: Map; + resolvedScaleSetId: number; + }; + reconciler.resolvedScaleSetId = 42; + for (let index = 0; index < 1100; index += 1) reconciler.rememberLifecycle(index + 1, `runner-${index}`, 'started'); + expect(reconciler.lifecycle.size).toBe(1000); + expect(reconciler.lifecycle.has('runner-0')).toBe(false); + }); +}); + +describe('reconciler helpers', () => { + it('calculates bounded desired capacity', () => { + expect(calculateDesiredRunners(5, 2, 6)).toBe(6); + expect(calculateDesiredRunners(8, 2, 5)).toBe(8); + expect(() => calculateDesiredRunners(-1, 0, 1)).toThrow('non-negative integer'); + }); + + it.each([ + { status: 'unexpected' }, + { status: 'retryable_error' }, + { status: 'non_retryable_error' }, + { retryable: true }, + { actions: { launched: 0, terminated: 0, retainedBusy: -1, retainedUnknown: 0 } }, + { actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0, retryable: true } }, + { status: 'converged', errors: [{ operation: 'list', code: 'UNEXPECTED_ERROR' }] }, + { status: 'retained', errors: [{ operation: 'list', code: 'UNEXPECTED_ERROR' }] }, + { status: 'error', errors: [] }, + { currentRunners: 0 }, + { currentRunners: 2 }, + { status: 'retained' }, + { errors: [{ operation: 'shell', code: 'BAD' }] }, + { errors: [{ operation: 'list', code: 'contains spaces' }] }, + { errors: [{ operation: 'list', code: 'BAD!CODE' }] }, + { errors: [{ operation: 'list', code: 'BAD\nCODE' }] }, + { errors: [{ operation: 'list', code: 'BAD', retryable: true }] }, + ])('rejects malformed compute-provider result metadata: %o', (overrides) => { + expect(() => validateProviderResult({ ...result(), ...overrides } as ScaleSetReconcileResult, 1)).toThrow( + /scale-set compute provider returned (?:an? )?invalid/, + ); + }); + + it('accepts bounded provider and AWS error codes', () => { + expect(() => + validateProviderResult( + result({ + status: 'error', + errors: [ + { operation: 'list', code: 'AccessDeniedException' }, + { operation: 'launch', code: 'ThrottlingException' }, + ], + }), + 1, + ), + ).not.toThrow(); + }); +}); diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts new file mode 100644 index 0000000000..69c56e9cda --- /dev/null +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -0,0 +1,668 @@ +import { + GitHubActionsScaleSetClient, + isScaleSetHttpError, + ScaleSetProtocolError, + type AccessToken, + type MessageSessionClient, + type RunnerScaleSetMessage, + type RunnerScaleSetStatistic, +} from '@aws-github-runner/github-actions-scale-set'; +import type { + ScaleSetComputeProvider, + ScaleSetComputeProviderFactoryInput, + ScaleSetReconcileRequest, + ScaleSetReconcileResult, + ScaleSetRunnerLifecycle, + ScaleSetRunnerState, +} from '@aws-github-runner/compute-providers/scale-set'; + +import { ScaleSetConfigurationError, type ScaleSetReconcilerConfig, type ScaleSetServiceConfig } from './config'; +import type { ParameterStore } from './credentials'; +import type { ScaleSetReconcilerStatusReporter } from './health'; +import type { ScaleSetLogger } from './logger'; + +const MAX_JIT_CONFIGURATION_BYTES = 1024 * 1024; + +export interface ScaleSetComputeProviderFactory { + create(type: string, input: ScaleSetComputeProviderFactoryInput): ScaleSetComputeProvider; +} + +export type ScaleSetReconcilerClient = Pick< + GitHubActionsScaleSetClient, + | 'createMessageSessionClient' + | 'generateJitRunnerConfig' + | 'getRunnerGroupByName' + | 'getRunnerScaleSet' + | 'getRunnerScaleSetById' + | 'getRunnerByName' + | 'removeRunner' + | 'createRunnerScaleSet' + | 'setSystemInfo' + | 'systemInfo' +>; + +export interface ScaleSetReconcilerDependencies { + createAccessTokenProvider(config: ScaleSetReconcilerConfig): Promise<() => Promise>; + createClient(config: ScaleSetReconcilerConfig, provider: () => Promise): ScaleSetReconcilerClient; + computeProviders: ScaleSetComputeProviderFactory; + logger: ScaleSetLogger; + sleep(delayMs: number, signal: AbortSignal): Promise; + random(): number; + closeSignal(timeoutMs: number): AbortSignal; + parameterStore: ParameterStore; +} + +interface LifecycleObservation { + runnerId: number; + runnerName: string; + scaleSetId: number; + lifecycle: ScaleSetRunnerLifecycle; +} + +export class ScaleSetProviderReconciliationError extends Error { + constructor( + readonly result?: ScaleSetReconcileResult, + options?: ErrorOptions, + ) { + super( + result === undefined + ? 'scale-set compute provider reconciliation failed' + : `scale-set compute provider returned ${result.status}`, + options, + ); + this.name = 'ScaleSetProviderReconciliationError'; + } +} + +export class ScaleSetReconciler { + private readonly lifecycle = new Map(); + private readonly lifecycleLimit: number; + private resolvedScaleSetId?: number; + private resolvedRunnerGroupId?: number; + + constructor( + private readonly config: ScaleSetReconcilerConfig, + private readonly serviceConfig: Pick< + ScaleSetServiceConfig, + 'sessionCloseTimeoutMs' | 'reconnectInitialBackoffMs' | 'reconnectMaxBackoffMs' + >, + private readonly dependencies: ScaleSetReconcilerDependencies, + ) { + this.lifecycleLimit = Math.min(20_000, Math.max(1000, config.maxRunners * 4)); + } + + async run(signal: AbortSignal, status: ScaleSetReconcilerStatusReporter): Promise { + let provider: ScaleSetComputeProvider; + let client: ScaleSetReconcilerClient; + try { + const accessTokenProvider = await this.dependencies.createAccessTokenProvider(this.config); + client = this.dependencies.createClient(this.config, accessTokenProvider); + const resolved = await this.resolveScaleSet(client, signal); + this.resolvedScaleSetId = resolved.scaleSetId; + this.resolvedRunnerGroupId = resolved.runnerGroupId; + client.setSystemInfo({ ...client.systemInfo, scaleSetId: resolved.scaleSetId }); + this.log('debug', 'scale_set_compute_provider_loading', { + computeProviderType: this.config.computeProvider.type, + }); + provider = this.dependencies.computeProviders.create(this.config.computeProvider.type, { + runnerConfigName: this.config.runnerConfigName, + scaleSetId: resolved.scaleSetId, + githubScope: this.config.githubConfigUrl, + configuration: this.config.computeProvider.configuration, + }); + this.log('info', 'scale_set_compute_provider_created', { + computeProviderType: this.config.computeProvider.type, + }); + this.log('debug', 'scale_set_compute_provider_loaded', { + computeProviderType: this.config.computeProvider.type, + }); + } catch (error) { + status.markFailed(error); + this.log('error', 'scale_set_reconciler_initialization_failed', { + computeProviderType: this.config.computeProvider.type, + ...httpErrorLogAttributes(error), + error, + }); + return; + } + + let consecutiveFailures = 0; + while (!signal.aborted) { + let session: MessageSessionClient | undefined; + let madeProgress = false; + try { + const configuredScaleSet = await client.getRunnerScaleSetById(this.scaleSetId, { signal }); + if ( + configuredScaleSet === null || + configuredScaleSet.id !== this.scaleSetId || + configuredScaleSet.name !== this.config.scaleSetName || + (this.resolvedRunnerGroupId !== undefined && configuredScaleSet.runnerGroupId !== this.resolvedRunnerGroupId) + ) { + throw new ScaleSetConfigurationError('configured GitHub runner scale set identity does not match'); + } + session = await client.createMessageSessionClient(this.scaleSetId, this.config.sessionOwner, { signal }); + status.markSessionReady(); + this.log('info', 'scale_set_session_created'); + this.log('debug', 'scale_set_session_scale_set_loaded', { + scaleSetName: session.session.runnerScaleSet?.name, + scaleSetLabels: session.session.runnerScaleSet?.labels?.map(({ name, type }) => ({ name, type })), + }); + let latestStatistics = session.session.statistics ?? undefined; + let lastMessageId = 0; + if (latestStatistics !== undefined) { + await this.reconcile(client, provider, latestStatistics, signal); + madeProgress = true; + consecutiveFailures = 0; + status.markProgress(); + } + + while (!signal.aborted) { + const message = await session.getMessage(lastMessageId, this.config.maxRunners, { signal }); + if (message === null) { + if (latestStatistics === undefined) { + throw new ScaleSetProtocolError('message session returned no message and no statistics snapshot'); + } + await this.reconcile(client, provider, latestStatistics, signal); + } else { + if (message.statistics === null) { + throw new ScaleSetProtocolError(`scale-set message ${message.messageId} contains no statistics`); + } + latestStatistics = message.statistics; + lastMessageId = message.messageId; + await session.deleteMessage(message.messageId, { signal }); + const requestIds = uniqueRequestIds(message); + if (requestIds.length > 0) await session.acquireJobs(requestIds, { signal }); + this.observeLifecycle(message); + await this.reconcile(client, provider, latestStatistics, signal); + this.pruneCompletedLifecycle(message); + } + madeProgress = true; + consecutiveFailures = 0; + status.markProgress(); + } + } catch (error) { + if (signal.aborted) break; + if (isFatalReconcilerError(error)) { + status.markFailed(error); + this.log('error', 'scale_set_reconciler_failed', { + ...httpErrorLogAttributes(error), + error, + }); + return; + } + consecutiveFailures = madeProgress ? 1 : consecutiveFailures + 1; + status.markReconnecting(error); + this.log('warn', 'scale_set_reconciler_reconnecting', { consecutiveFailures, error }); + } finally { + if (session !== undefined) await this.closeSession(session); + } + + if (!signal.aborted) { + await this.dependencies.sleep( + calculateReconnectDelay( + consecutiveFailures, + this.serviceConfig.reconnectInitialBackoffMs, + this.serviceConfig.reconnectMaxBackoffMs, + this.dependencies.random, + ), + signal, + ); + } + } + status.markStopping(); + } + + private async resolveScaleSet( + client: ScaleSetReconcilerClient, + signal: AbortSignal, + ): Promise<{ scaleSetId: number; runnerGroupId?: number }> { + let runnerGroupId = this.config.expectedRunnerGroupId; + if (this.config.runnerGroupName !== undefined) { + const cachedRunnerGroupId = await this.loadCachedRunnerGroupId(); + const runnerGroup = + cachedRunnerGroupId === undefined + ? await client.getRunnerGroupByName(this.config.runnerGroupName, { signal }) + : { id: cachedRunnerGroupId }; + if (runnerGroupId !== undefined && runnerGroupId !== runnerGroup.id) { + throw new ScaleSetConfigurationError( + `runner group ${JSON.stringify(this.config.runnerGroupName)} resolved to ID ${runnerGroup.id}, expected ${runnerGroupId}`, + ); + } + runnerGroupId = runnerGroup.id; + if (cachedRunnerGroupId === undefined && this.config.runnerGroupIdParameterName !== undefined) { + await this.dependencies.parameterStore.put?.(this.config.runnerGroupIdParameterName, String(runnerGroupId)); + } + this.log('info', 'scale_set_runner_group_resolved', { + runnerConfigName: this.config.runnerConfigName, + runnerGroupName: this.config.runnerGroupName, + runnerGroupId, + }); + } + + if (this.config.scaleSetId === undefined && runnerGroupId === undefined) { + throw new ScaleSetConfigurationError('runner group ID was not resolved'); + } + let configuredScaleSet = + this.config.scaleSetId === undefined + ? await client.getRunnerScaleSet(runnerGroupId as number, this.config.scaleSetName, { signal }) + : await client.getRunnerScaleSetById(this.config.scaleSetId, { signal }); + if (configuredScaleSet === null && runnerGroupId !== undefined && this.config.scaleSetId === undefined) { + this.log('info', 'scale_set_registering', { + runnerConfigName: this.config.runnerConfigName, + scaleSetName: this.config.scaleSetName, + runnerGroupId, + }); + try { + configuredScaleSet = await client.createRunnerScaleSet( + { + name: this.config.scaleSetName, + runnerGroupId, + labels: [{ name: this.config.scaleSetName }], + runnerSetting: {}, + }, + { signal }, + ); + } catch (error) { + const existingScaleSet = await client.getRunnerScaleSet(runnerGroupId, this.config.scaleSetName, { signal }); + if (existingScaleSet === null) throw error; + configuredScaleSet = existingScaleSet; + } + } + if (configuredScaleSet === null || configuredScaleSet.id === undefined) { + throw new ScaleSetConfigurationError( + `GitHub runner scale set ${JSON.stringify(this.config.scaleSetName)} was not found`, + ); + } + if ( + configuredScaleSet.name !== this.config.scaleSetName || + (runnerGroupId !== undefined && configuredScaleSet.runnerGroupId !== runnerGroupId) || + (this.config.scaleSetId !== undefined && configuredScaleSet.id !== this.config.scaleSetId) + ) { + throw new ScaleSetConfigurationError('configured GitHub runner scale set identity does not match'); + } + this.log('info', 'scale_set_resolved', { + runnerConfigName: this.config.runnerConfigName, + scaleSetName: this.config.scaleSetName, + scaleSetId: configuredScaleSet.id, + runnerGroupId, + }); + this.log('debug', 'scale_set_labels_resolved', { + scaleSetName: configuredScaleSet.name, + scaleSetLabels: configuredScaleSet.labels?.map(({ name, type }) => ({ name, type })), + }); + return { scaleSetId: configuredScaleSet.id, runnerGroupId }; + } + + private async loadCachedRunnerGroupId(): Promise { + const parameterName = this.config.runnerGroupIdParameterName; + if (parameterName === undefined) return undefined; + const values = await this.dependencies.parameterStore.get([parameterName]); + const raw = values.get(parameterName)?.trim(); + if (raw === undefined || raw === '') return undefined; + if (!/^\d+$/.test(raw)) { + throw new ScaleSetConfigurationError( + `runner group ID parameter ${JSON.stringify(parameterName)} must contain a positive integer`, + ); + } + const id = Number(raw); + if (!Number.isSafeInteger(id) || id <= 0) { + throw new ScaleSetConfigurationError( + `runner group ID parameter ${JSON.stringify(parameterName)} must contain a positive integer`, + ); + } + return id; + } + + private get scaleSetId(): number { + if (this.resolvedScaleSetId === undefined) { + throw new ScaleSetConfigurationError('scale set ID was not resolved'); + } + return this.resolvedScaleSetId; + } + + private async reconcile( + client: ScaleSetReconcilerClient, + provider: ScaleSetComputeProvider, + statistics: RunnerScaleSetStatistic, + signal: AbortSignal, + ): Promise { + const desiredRunners = calculateDesiredRunners( + statistics.totalAssignedJobs, + this.config.minRunners, + this.config.maxRunners, + ); + const callbacks = this.createReconcileCallbacks(client, signal); + const result = await this.reconcileProvider(provider, { + desiredRunners, + busyRunners: statistics.totalBusyRunners, + bootTimeoutMinutes: this.config.bootTimeoutMinutes, + runnerStates: this.lifecycleStates(), + ...callbacks, + }); + validateProviderResult(result, desiredRunners); + throwIfProviderError(result); + this.logReconciliationResult(result, desiredRunners); + } + + private createReconcileCallbacks(client: ScaleSetReconcilerClient, signal: AbortSignal) { + return { + signal, + generateJitConfiguration: async ({ + runnerName, + signal: callbackSignal, + }: { + runnerName: string; + signal?: AbortSignal; + }) => { + const jit = await client.generateJitRunnerConfig( + { name: runnerName, workFolder: this.config.workFolder }, + this.scaleSetId, + { signal: callbackSignal ?? signal }, + ); + if ( + jit.runner === null || + jit.runner.name !== runnerName || + jit.runner.runnerScaleSetId !== this.scaleSetId || + !Number.isSafeInteger(jit.runner.id) || + jit.runner.id <= 0 + ) { + throw new ScaleSetProtocolError('GitHub returned a mismatched runner identity for JIT configuration'); + } + if ( + typeof jit.encodedJITConfig !== 'string' || + jit.encodedJITConfig === '' || + Buffer.byteLength(jit.encodedJITConfig, 'utf8') > MAX_JIT_CONFIGURATION_BYTES + ) { + throw new ScaleSetProtocolError('GitHub returned an invalid JIT configuration'); + } + return { + encodedJitConfiguration: jit.encodedJITConfig, + runnerId: jit.runner.id, + runnerName: jit.runner.name, + scaleSetId: jit.runner.runnerScaleSetId, + }; + }, + removeRunner: async (expected: { + runnerId: number; + runnerName: string; + scaleSetId: number; + signal?: AbortSignal; + }) => { + const callbackSignal = expected.signal ?? signal; + const runner = await client.getRunnerByName(expected.runnerName, { signal: callbackSignal }); + if (runner === null) return { status: 'retained_unknown' as const }; + if ( + runner.id !== expected.runnerId || + runner.name !== expected.runnerName || + runner.runnerScaleSetId !== expected.scaleSetId || + expected.scaleSetId !== this.scaleSetId + ) { + return { status: 'retained_unknown' as const }; + } + // Busy state comes from the aggregate scale-set statistics. The + // Actions-service runner reference above remains the exact identity + // check; no public GitHub REST runner call is required. + try { + await client.removeRunner(runner.id, { signal: callbackSignal }); + } catch (error) { + if (isScaleSetHttpError(error) && error.status === 404) return { status: 'removed' as const }; + throw error; + } + return { status: 'removed' as const }; + }, + }; + } + + private logReconciliationResult(result: ScaleSetReconcileResult, desiredRunners: number): void { + this.log('info', 'scale_set_reconciled', { + computeProviderType: this.config.computeProvider.type, + desiredRunners, + currentRunners: result.currentRunners, + status: result.status, + actions: result.actions, + errorCount: result.errors.length, + }); + if (result.status === 'retained') { + this.log('warn', 'scale_set_capacity_retained', { + computeProviderType: this.config.computeProvider.type, + desiredRunners, + currentRunners: result.currentRunners, + retainedBusy: result.actions.retainedBusy, + retainedUnknown: result.actions.retainedUnknown, + }); + } + } + + private async reconcileProvider( + provider: ScaleSetComputeProvider, + request: ScaleSetReconcileRequest, + ): Promise { + this.log('info', 'scale_set_compute_provider_reconcile_started', { + computeProviderType: this.config.computeProvider.type, + desiredRunners: request.desiredRunners, + busyRunners: request.busyRunners, + }); + try { + return await provider.reconcile(request); + } catch (error) { + request.signal.throwIfAborted(); + this.log('error', 'scale_set_compute_provider_reconcile_failed', { + computeProviderType: this.config.computeProvider.type, + desiredRunners: request.desiredRunners, + busyRunners: request.busyRunners, + error, + }); + throw new ScaleSetProviderReconciliationError(undefined, { cause: error }); + } + } + + private observeLifecycle(message: RunnerScaleSetMessage): void { + for (const runner of message.jobStartedMessages) + this.rememberLifecycle(runner.runnerId, runner.runnerName, 'started'); + for (const runner of message.jobCompletedMessages) { + this.rememberLifecycle(runner.runnerId, runner.runnerName, 'completed'); + } + } + + private rememberLifecycle(runnerId: number, runnerName: string, lifecycle: ScaleSetRunnerLifecycle): void { + if (!Number.isSafeInteger(runnerId) || runnerId <= 0 || runnerName === '') return; + const current = this.lifecycle.get(runnerName); + if (current !== undefined && current.runnerId !== runnerId) { + this.lifecycle.delete(runnerName); + return; + } + this.lifecycle.set(runnerName, { runnerId, runnerName, scaleSetId: this.scaleSetId, lifecycle }); + while (this.lifecycle.size > this.lifecycleLimit) { + const oldest = this.lifecycle.keys().next().value as string | undefined; + if (oldest === undefined) break; + this.lifecycle.delete(oldest); + } + } + + private pruneCompletedLifecycle(message: RunnerScaleSetMessage): void { + for (const runner of message.jobCompletedMessages) { + const observation = this.lifecycle.get(runner.runnerName); + if (observation?.runnerId === runner.runnerId && observation.lifecycle === 'completed') { + this.lifecycle.delete(runner.runnerName); + } + } + } + + private lifecycleStates(): ScaleSetRunnerState[] { + return [...this.lifecycle.values()].map((observation) => ({ + ...observation, + status: 'unknown', + busy: undefined, + })); + } + + private async closeSession(session: Pick): Promise { + try { + await session.close({ signal: this.dependencies.closeSignal(this.serviceConfig.sessionCloseTimeoutMs) }); + } catch (error) { + this.log('warn', 'scale_set_session_close_failed', { error }); + } + } + + private log( + level: 'debug' | 'info' | 'warn' | 'error', + event: string, + attributes: Record = {}, + ): void { + this.dependencies.logger[level](event, { + groupRunnerConfig: this.config.runnerConfigName, + scaleSetId: this.resolvedScaleSetId, + ...attributes, + }); + } +} + +export function calculateDesiredRunners(totalAssignedJobs: number, minRunners: number, maxRunners: number): number { + if (!Number.isSafeInteger(totalAssignedJobs) || totalAssignedJobs < 0) { + throw new ScaleSetProtocolError('statistics.totalAssignedJobs must be a non-negative integer'); + } + // maxRunners bounds newly requested idle capacity, but an operator reducing + // it must never make already-assigned work a scale-down target. + return Math.max(totalAssignedJobs, Math.min(maxRunners, minRunners + totalAssignedJobs)); +} + +export function calculateReconnectDelay( + attempt: number, + initialBackoffMs: number, + maxBackoffMs: number, + random: () => number = Math.random, +): number { + if (!Number.isSafeInteger(attempt) || attempt <= 0) throw new Error('attempt must be a positive integer'); + const ceiling = Math.min(maxBackoffMs, initialBackoffMs * 2 ** Math.min(attempt - 1, 30)); + const value = Math.max(0, Math.min(1, random())); + return Math.floor(ceiling / 2 + (ceiling / 2) * value); +} + +export async function abortableSleep(delayMs: number, signal: AbortSignal): Promise { + if (signal.aborted || delayMs <= 0) return; + await new Promise((resolve) => { + const done = () => { + clearTimeout(timeout); + signal.removeEventListener('abort', done); + resolve(); + }; + const timeout = setTimeout(done, delayMs); + signal.addEventListener('abort', done, { once: true }); + }); +} + +function uniqueRequestIds(message: RunnerScaleSetMessage): number[] { + return [...new Set(message.jobAvailableMessages.map(({ runnerRequestId }) => runnerRequestId))]; +} + +export function validateProviderResult(result: ScaleSetReconcileResult, desiredRunners: number): void { + const value = result as unknown; + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new ScaleSetProtocolError('scale-set compute provider returned an invalid reconciliation result'); + } + const record = value as Record; + const resultFields = new Set(['status', 'desiredRunners', 'currentRunners', 'actions', 'errors']); + if (Object.keys(record).some((key) => !resultFields.has(key))) { + throw new ScaleSetProtocolError('scale-set compute provider returned an invalid reconciliation result'); + } + const statuses = new Set(['converged', 'retained', 'error']); + if (!statuses.has(record.status as string)) { + throw new ScaleSetProtocolError('scale-set compute provider returned an invalid status'); + } + if (record.desiredRunners !== desiredRunners || !boundedCount(record.currentRunners)) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid capacity counts'); + } + const actions = record.actions; + if (typeof actions !== 'object' || actions === null || Array.isArray(actions)) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid actions'); + } + const actionFields = new Set(['launched', 'terminated', 'retainedBusy', 'retainedUnknown']); + if (Object.keys(actions).some((key) => !actionFields.has(key))) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid action counts'); + } + for (const key of ['launched', 'terminated', 'retainedBusy', 'retainedUnknown']) { + if (!boundedCount((actions as Record)[key])) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid action counts'); + } + } + if (!Array.isArray(record.errors) || record.errors.length > 1000) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid errors'); + } + const operations = new Set([ + 'validate', + 'reconcile', + 'list', + 'launch', + 'generate_jit_configuration', + 'publish_jit_configuration', + 'remove_runner', + 'terminate', + ]); + const errorFields = new Set(['operation', 'code', 'runnerName', 'resourceId']); + for (const error of record.errors) { + if (typeof error !== 'object' || error === null || Array.isArray(error)) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid error metadata'); + } + const metadata = error as Record; + if ( + Object.keys(metadata).some((key) => !errorFields.has(key)) || + !operations.has(metadata.operation as string) || + typeof metadata.code !== 'string' || + !/^[A-Za-z][A-Za-z0-9._-]{0,127}$/.test(metadata.code) || + !optionalBoundedMetadata(metadata.runnerName) || + !optionalBoundedMetadata(metadata.resourceId) + ) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid error metadata'); + } + } + if ((record.status === 'error') !== record.errors.length > 0) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid error status'); + } + let expectedStatus: ScaleSetReconcileResult['status'] = 'converged'; + if (record.errors.length > 0 || (record.currentRunners as number) < desiredRunners) { + expectedStatus = 'error'; + } else if ((record.currentRunners as number) > desiredRunners) { + expectedStatus = 'retained'; + } + if (record.status !== expectedStatus) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid reconciliation status'); + } +} + +function throwIfProviderError(result: ScaleSetReconcileResult): void { + if (result.status === 'error') { + throw new ScaleSetProviderReconciliationError(result); + } +} + +function boundedCount(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0 && (value as number) <= 2_147_483_647; +} + +function optionalBoundedMetadata(value: unknown): value is string | undefined { + return value === undefined || (typeof value === 'string' && value.length <= 256 && !hasAsciiControlCharacter(value)); +} + +function hasAsciiControlCharacter(value: string): boolean { + return [...value].some((character) => { + const code = character.charCodeAt(0); + return code <= 31 || code === 127; + }); +} + +function isFatalReconcilerError(error: unknown): boolean { + if (error instanceof ScaleSetConfigurationError || error instanceof ScaleSetProtocolError) return true; + if (error instanceof ScaleSetProviderReconciliationError) return true; + if (!isScaleSetHttpError(error)) return false; + return error.status >= 400 && error.status < 500 && ![408, 409, 425, 429].includes(error.status); +} + +function httpErrorLogAttributes(error: unknown): Record { + if (!isScaleSetHttpError(error)) return {}; + return { + requestMethod: error.method, + requestUrl: error.url, + requestStatus: error.status, + requestCode: error.code, + }; +} diff --git a/lambdas/services/scale-set/tsconfig.json b/lambdas/services/scale-set/tsconfig.json new file mode 100644 index 0000000000..714aa27b6b --- /dev/null +++ b/lambdas/services/scale-set/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/lambdas/services/scale-set/vitest.config.ts b/lambdas/services/scale-set/vitest.config.ts new file mode 100644 index 0000000000..28a41aa2aa --- /dev/null +++ b/lambdas/services/scale-set/vitest.config.ts @@ -0,0 +1,18 @@ +import { mergeConfig } from 'vitest/config'; +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + root: __dirname, + test: { + coverage: { + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts', 'src/index.ts', 'src/main.ts'], + thresholds: { + statements: 80, + branches: 70, + functions: 80, + lines: 80, + }, + }, + }, +}); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 811757d346..c295e1c330 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -138,6 +138,7 @@ __metadata: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-sdk/client-ec2": "npm:^3.1009.0" + "@aws-sdk/client-ssm": "npm:^3.1009.0" "@octokit/rest": "npm:22.0.1" aws-sdk-client-mock: "npm:^4.1.0" aws-sdk-client-mock-jest: "npm:^4.1.0" @@ -199,6 +200,32 @@ __metadata: languageName: unknown linkType: soft +"@aws-github-runner/github-actions-scale-set@npm:*, @aws-github-runner/github-actions-scale-set@workspace:libs/github-actions-scale-set": + version: 0.0.0-use.local + resolution: "@aws-github-runner/github-actions-scale-set@workspace:libs/github-actions-scale-set" + dependencies: + "@types/node": "npm:^22.19.3" + typescript: "npm:^5.9.3" + languageName: unknown + linkType: soft + +"@aws-github-runner/scale-set-service@workspace:services/scale-set": + version: 0.0.0-use.local + resolution: "@aws-github-runner/scale-set-service@workspace:services/scale-set" + dependencies: + "@aws-github-runner/aws-ssm-util": "npm:*" + "@aws-github-runner/compute-providers": "npm:*" + "@aws-github-runner/github-actions-scale-set": "npm:*" + "@aws-sdk/client-ssm": "npm:^3.1009.0" + "@octokit/auth-app": "npm:8.2.0" + "@octokit/request": "npm:^9.2.2" + "@types/node": "npm:^22.19.3" + "@vercel/ncc": "npm:0.38.4" + typescript: "npm:^5.9.3" + undici: "npm:^6.19.2" + languageName: unknown + linkType: soft + "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher": version: 0.0.0-use.local resolution: "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher" @@ -11000,6 +11027,13 @@ __metadata: languageName: node linkType: hard +"undici@npm:^6.19.2": + version: 6.28.0 + resolution: "undici@npm:6.28.0" + checksum: 10c0/3029a70df06b38b5b2f30732932a1e92544c753cd82c8abdf0d35afad48e0ba91612e79fe3a442dbbb9434d6a9eba2b714d5ea28984c903dda2b5d5444f38354 + languageName: node + linkType: hard + "unicode-canonical-property-names-ecmascript@npm:^2.0.0": version: 2.0.0 resolution: "unicode-canonical-property-names-ecmascript@npm:2.0.0"