feat(scale-set): wire orchestration through runner config

modules/compute-providers/aws/ec2/outputs.tf

@@ -16,6 +16,8 @@ output "resources" {
 output "provider" {
   description = "Nested EC2 compute-provider contract consumed by runner-config."
   value = {
+    type                  = "ec2"
+    capabilities          = { scale_set = local.scale_set_capability }
     environment_variables = local.provider_environment_variables
     policies              = local.provider_policies
     resources             = local.provider_resources

modules/compute-providers/aws/ec2/scale-set.tf

@@ -0,0 +1,255 @@
+# Provider-owned runtime and IAM fragments for the additive scale-set
+# orchestration capability. GitHub credentials, GitHub scope, desired capacity,
+# and boot timeout remain orchestration-owned and are not serialized here.
+locals {
+  scale_set_ec2_instance_criteria = merge(
+    {
+      instanceTypes              = var.config.instance_types
+      targetCapacityType         = var.config.instance_target_capacity_type
+      instanceAllocationStrategy = var.config.instance_allocation_strategy
+    },
+    var.config.instance_type_priorities == null ? {} : {
+      instanceTypePriorities = var.config.instance_type_priorities
+    },
+    var.config.instance_max_spot_price == null ? {} : {
+      maxSpotPrice = var.config.instance_max_spot_price
+    },
+  )
+
+  scale_set_runtime_configuration = merge(
+    {
+      region                  = var.aws_region
+      environment             = var.prefix
+      runnerNamePrefix        = var.runner.name_prefix
+      jitConfigParameterPath  = "${var.ssm.paths.root}/${var.ssm.paths.tokens}"
+      subnets                 = var.config.subnet_ids
+      launchTemplateName      = aws_launch_template.runner.name
+      ec2instanceCriteria     = local.scale_set_ec2_instance_criteria
+      onDemandFailoverOnError = var.config.on_demand_failover_for_errors
+      scaleErrors             = var.config.scale_errors
+      useDedicatedHost        = var.config.use_dedicated_host
+      ssmParameterTags = [
+        for key in sort(keys(local.ssm_parameter_tags)) : {
+          Key   = key
+          Value = local.ssm_parameter_tags[key]
+        }
+      ]
+    },
+    local.ami_id_ssm_external ? {
+      amiIdSsmParameterName = local.ami_id_ssm_parameter_name
+    } : {},
+  )
+
+  scale_set_owned_instance_conditions = [
+    {
+      test     = "StringEquals"
+      variable = "ec2:ResourceTag/ghr:Application"
+      values   = toset(["github-action-runner"])
+    },
+    {
+      test     = "StringEquals"
+      variable = "ec2:ResourceTag/ghr:created_by"
+      values   = toset(["scale-set-service"])
+    },
+    {
+      test     = "StringEquals"
+      variable = "ec2:ResourceTag/ghr:environment"
+      values   = toset([var.prefix])
+    },
+  ]
+
+  scale_set_owned_request_conditions = [
+    {
+      test     = "StringEquals"
+      variable = "aws:RequestTag/ghr:Application"
+      values   = toset(["github-action-runner"])
+    },
+    {
+      test     = "StringEquals"
+      variable = "aws:RequestTag/ghr:created_by"
+      values   = toset(["scale-set-service"])
+    },
+    {
+      test     = "StringEquals"
+      variable = "aws:RequestTag/ghr:environment"
+      values   = toset([var.prefix])
+    },
+  ]
+
+  scale_set_launch_dependency_resources = toset(concat(
+    [
+      "arn:${var.aws_partition}:ec2:${var.aws_region}::image/*",
+      "arn:${var.aws_partition}:ec2:${var.aws_region}:*:snapshot/*",
+      "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:dedicated-host/*",
+      "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:network-interface/*",
+      "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:placement-group/*",
+      "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:security-group/*",
+      aws_launch_template.runner.arn,
+    ],
+    [
+      for subnet_id in var.config.subnet_ids :
+      "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:subnet/${subnet_id}"
+    ],
+    var.config.key_name == null ? [] : [
+      "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:key-pair/${var.config.key_name}",
+    ],
+  ))
+
+  scale_set_create_fleet_dependency_resources = toset(concat(
+    [
+      "arn:${var.aws_partition}:ec2:${var.aws_region}::image/*",
+      "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:placement-group/*",
+      aws_launch_template.runner.arn,
+    ],
+    [
+      for subnet_id in var.config.subnet_ids :
+      "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:subnet/${subnet_id}"
+    ],
+  ))
+
+  scale_set_iam_statements = merge(
+    {
+      describe_ec2 = {
+        actions = toset([
+          "ec2:DescribeInstances",
+          "ec2:DescribeLaunchTemplateVersions",
+          "ec2:DescribeTags",
+        ])
+        # These EC2 Describe APIs do not support resource-level permissions.
+        resources  = toset(["*"])
+        conditions = []
+      }
+      create_fleet_dependencies = {
+        actions    = toset(["ec2:CreateFleet"])
+        resources  = local.scale_set_create_fleet_dependency_resources
+        conditions = []
+      }
+      create_owned_fleet_capacity = {
+        actions = toset(["ec2:CreateFleet"])
+        resources = toset([
+          "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:fleet/*",
+          "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:instance/*",
+          "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:volume/*",
+        ])
+        conditions = local.scale_set_owned_request_conditions
+      }
+      run_instances_dependencies = {
+        actions    = toset(["ec2:RunInstances"])
+        resources  = local.scale_set_launch_dependency_resources
+        conditions = []
+      }
+      run_owned_instances = {
+        actions = toset(["ec2:RunInstances"])
+        resources = toset([
+          "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:instance/*",
+          "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:volume/*",
+        ])
+        conditions = local.scale_set_owned_request_conditions
+      }
+      tag_runners_on_create = {
+        actions   = toset(["ec2:CreateTags"])
+        resources = toset(["arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:*/*"])
+        conditions = [
+          {
+            test     = "StringEquals"
+            variable = "ec2:CreateAction"
+            values   = toset(["CreateFleet", "RunInstances"])
+          },
+        ]
+      }
+      update_owned_runner_tags = {
+        actions   = toset(["ec2:CreateTags"])
+        resources = toset(["arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:instance/*"])
+        conditions = concat(local.scale_set_owned_instance_conditions, [
+          {
+            test     = "ForAllValues:StringEquals"
+            variable = "aws:TagKeys"
+            values = toset([
+              "ghr:github_runner_id",
+              "ghr:runner_name",
+              "ghr:scale_set_state",
+            ])
+          },
+        ])
+      }
+      terminate_owned_runners = {
+        actions    = toset(["ec2:TerminateInstances"])
+        resources  = toset(["arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:instance/*"])
+        conditions = local.scale_set_owned_instance_conditions
+      }
+      pass_runner_role = {
+        actions   = toset(["iam:PassRole"])
+        resources = toset([var.runner.iam.role.arn])
+        conditions = [
+          {
+            test     = "StringEquals"
+            variable = "iam:PassedToService"
+            values   = toset(["ec2.amazonaws.com"])
+          },
+        ]
+      }
+      publish_runner_jit_configuration = {
+        actions = toset([
+          "ssm:AddTagsToResource",
+          "ssm:DeleteParameter",
+          "ssm:PutParameter",
+        ])
+        resources = toset([
+          "${local.ssm_parameter_arn_prefix}${var.ssm.paths.root}/${var.ssm.paths.tokens}/*",
+        ])
+        conditions = []
+      }
+    },
+    local.ami_id_ssm_external ? {
+      read_external_ami_parameter = {
+        actions    = toset(["ssm:GetParameter"])
+        resources  = toset([local.ami_id_ssm_parameter_arn])
+        conditions = []
+      }
+    } : {},
+    local.ami_kms_key_enabled ? {
+      use_ami_kms_key = {
+        actions = toset([
+          "kms:Decrypt",
+          "kms:DescribeKey",
+          "kms:ReEncryptFrom",
+          "kms:ReEncryptTo",
+        ])
+        resources  = toset([local.ami_kms_key_arn])
+        conditions = []
+      }
+      create_ami_kms_grant = {
+        actions   = toset(["kms:CreateGrant"])
+        resources = toset([local.ami_kms_key_arn])
+        conditions = [
+          {
+            test     = "Bool"
+            variable = "kms:GrantIsForAWSResource"
+            values   = toset(["true"])
+          },
+        ]
+      }
+    } : {},
+    var.config.create_service_linked_role_spot ? {
+      create_spot_service_linked_role = {
+        actions = toset(["iam:CreateServiceLinkedRole"])
+        resources = toset([
+          "arn:${var.aws_partition}:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot",
+        ])
+        conditions = [
+          {
+            test     = "StringEquals"
+            variable = "iam:AWSServiceName"
+            values   = toset(["spot.amazonaws.com"])
+          },
+        ]
+      }
+    } : {},
+  )
+
+  scale_set_capability = {
+    configuration_json    = jsonencode(local.scale_set_runtime_configuration)
+    environment_variables = {}
+    iam_statements        = local.scale_set_iam_statements
+  }
+}

modules/multi-runner/config.experimental.effective.tf

@@ -35,6 +35,7 @@ locals {
               artifact = local.normalized_config.orchestration_provider.webhook.lambda.artifact
             })
           })
+          scale_set = v.orchestration_provider.scale_set
         }
 
         ssm = merge(v.ssm, {

modules/multi-runner/config.experimental.resolved.tf

@@ -33,7 +33,10 @@ locals {
   # A non-empty experimental map selects v2 normalization.
   use_v2_config = length(var.experimental_multi_runner_config) > 0
 
-  normalized_config = local.use_v2_config ? local.experimental : local.stable_to_experimental
+  normalized_config = merge(
+    local.stable_to_experimental,
+    { for k, v in local.experimental : k => v if local.use_v2_config },
+  )
 }
 
 locals {
@@ -278,6 +281,7 @@ locals {
               tags = merge(local.normalized_config.orchestration_provider.webhook.queue.tags, v.orchestration_provider.webhook.queue.tags)
             })
           })
+          scale_set = v.orchestration_provider.scale_set
         }
 
         ssm = merge(v.ssm, {

modules/multi-runner/config.experimental.translation.tf

@@ -140,6 +140,7 @@ locals {
         encryption = var.queue_encryption
       }
     }
+    scale_set = null
   }
 
   stable_to_experimental_ssm = {
@@ -425,6 +426,7 @@ locals {
             }
           }
         }
+        scale_set = null
       }
 
       ssm = {

modules/multi-runner/orchestration-provider.scale-set.tf

@@ -0,0 +1,74 @@
+locals {
+  scale_set_runner_config = {
+    for runner_name, runner_config in local.effective_config.multi_runner_config :
+    runner_name => runner_config
+    if runner_config.orchestration_provider.scale_set != null
+  }
+
+  scale_set_runner_configs = {
+    for runner_name, runner_config in local.scale_set_runner_config : runner_name => {
+      github = {
+        config_url = runner_config.orchestration_provider.scale_set.github.config_url
+        app = {
+          app_id = {
+            name        = local.primary_app_id.name
+            arn         = local.primary_app_id.arn
+            kms_key_arn = local.effective_config.ssm.kms_key_id
+          }
+          private_key = {
+            name        = local.primary_app_key_base64.name
+            arn         = local.primary_app_key_base64.arn
+            kms_key_arn = local.effective_config.ssm.kms_key_id
+          }
+          installation_id = {
+            name        = runner_config.orchestration_provider.scale_set.github.installation_id_ssm.name
+            arn         = runner_config.orchestration_provider.scale_set.github.installation_id_ssm.arn
+            kms_key_arn = runner_config.orchestration_provider.scale_set.github.installation_id_ssm.kms_key_arn
+          }
+        }
+        force_ghes = try(coalesce(
+          runner_config.orchestration_provider.scale_set.github.force_ghes,
+          local.effective_config.github.enterprise_server.url != null,
+        ), false)
+        ssl_verify = local.effective_config.github.enterprise_server.ssl_verify
+        user_agent = local.effective_config.github.user_agent
+      }
+      scale_set = {
+        name                 = runner_config.orchestration_provider.scale_set.name
+        id                   = runner_config.orchestration_provider.scale_set.id
+        runner_group_id      = runner_config.orchestration_provider.scale_set.runner_group_id
+        min_runners          = runner_config.orchestration_provider.scale_set.min_runners
+        max_runners          = runner_config.orchestration_provider.scale_set.max_runners
+        boot_time_in_minutes = runner_config.orchestration_provider.scale_set.boot_time_in_minutes
+        session_owner        = runner_config.orchestration_provider.scale_set.session_owner
+      }
+      work_folder = runner_config.orchestration_provider.scale_set.work_folder
+    }
+  }
+
+  scale_set_compute_provider_contracts = {
+    for runner_name in keys(local.scale_set_runner_configs) :
+    runner_name => module.runner_configs[runner_name].compute_provider_contract
+  }
+}
+
+module "orchestration_scale_set" {
+  source = "../orchestration-providers/scale-set"
+  count  = length(local.scale_set_runner_configs) > 0 ? 1 : 0
+
+  prefix                     = var.prefix
+  runner_configs             = local.scale_set_runner_configs
+  compute_provider_contracts = local.scale_set_compute_provider_contracts
+
+  grouping     = try(local.effective_config.orchestration_provider.scale_set.grouping, {})
+  container    = try(local.effective_config.orchestration_provider.scale_set.container, {})
+  config_store = try(local.effective_config.orchestration_provider.scale_set.config_store, {})
+  ecs          = try(local.effective_config.orchestration_provider.scale_set.ecs, {})
+  network      = try(local.effective_config.orchestration_provider.scale_set.network, {})
+  logging      = try(local.effective_config.orchestration_provider.scale_set.logging, {})
+  tags = merge(
+    local.effective_config.tags,
+    try(local.effective_config.orchestration_provider.scale_set.tags, {}),
+    { "ghr:environment" = var.prefix },
+  )
+}