diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index b21ecc544f..059be32d9c 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -85,6 +85,8 @@ jobs: "download-lambda", "lambda", "multi-runner", + "compute-providers/aws/microvm", + "compute-providers/aws/microvm/trust-policy", "runner-binaries-syncer", "runners", "setup-iam-permissions", @@ -155,6 +157,7 @@ jobs: "ephemeral", "termination-watcher", "multi-runner", + "multi-runner-v2", "external-managed-ssm-secrets" ] defaults: @@ -214,6 +217,9 @@ jobs: matrix: module: - modules/runners + - modules/multi-runner + - modules/compute-providers/aws/microvm + - modules/compute-providers/aws/microvm/trust-policy defaults: run: working-directory: ${{ matrix.module }} diff --git a/docs/adr/002-runner-orchestration-provider-boundary.md b/docs/adr/002-runner-orchestration-provider-boundary.md new file mode 100644 index 0000000000..0b2cc5886a --- /dev/null +++ b/docs/adr/002-runner-orchestration-provider-boundary.md @@ -0,0 +1,405 @@ +# ADR-002: Runner Orchestration Provider Boundary + +## Status + +Proposed + +## Date + +2026-09-03 + +## Context + +The multi-runner module currently receives workflow-job demand through a shared +GitHub webhook. A build queue invokes scale-up, schedules invoke scale-down +and the runner pool, and an optional retry queue checks queued jobs. These +components evolved together, while their settings were spread across shared +module inputs and each runner configuration. + +That layout assumes every runner configuration uses the same demand-control +model. It also makes the runner configuration responsible for webhook-specific +resources. Adding another model would require provider conditionals throughout +the module or a second copy of the common runner and compute-provider wiring. + +GitHub Actions Runner Scale Sets require a different control model. A future +implementation is expected to use the runner scale-set and agent APIs: + +- `_apis/runtime/runnerscalesets` +- `_apis/distributedtask/pools/0/agents` + +Unlike event- and schedule-driven Lambda components, a scale-set controller +maintains reconciliation state and long-lived coordination with GitHub. It may +therefore need a containerized service, with ECS as a candidate deployment +target, rather than another independent Lambda handler. + +The Terraform contract must allow that future addition without moving webhook +fields a second time. This ADR defines the boundary. It does not implement the +scale-set API client, controller, container image, or ECS resources. + +## Terminology + +- **Runner configuration**: One entry in `experimental_multi_runner_config`, + including common runner behavior, one orchestration provider, and one compute + provider. +- **Orchestration provider**: The implementation that receives or reconciles + runner demand and owns the controls that turn demand into capacity actions. +- **Compute provider**: The implementation that creates and manages runner + capacity, such as AWS EC2. It supplies capabilities to orchestration. +- **Webhook orchestration**: The existing webhook, queue, scale-up, scale-down, + pool, and job-retry implementation. +- **Scale-set orchestration**: A future stateful controller built on GitHub's + runner scale-set APIs. + +## Decision + +We will use typed orchestration-provider and compute-provider boundaries in the +experimental multi-runner interface. Every runner configuration selects +exactly one provider of each type. + +### The experimental contract uses split global variables + +The experimental interface is intentionally represented by separate Terraform +variables rather than one monolithic `experimental` object: + +- `experimental_global_config` contains common global defaults such as tags, + roles, and runner identity. +- `experimental_global_config_github` contains shared GitHub settings. +- `experimental_global_config_lambda` contains provider-neutral Lambda + substrate and the shared artifact bucket. +- `experimental_global_config_orchestration_provider` contains global webhook + defaults and shared webhook settings. +- `experimental_global_config_ssm` contains global SSM settings. +- `experimental_global_config_observability` contains logs, tracing, and + metrics defaults. +- `experimental_global_config_compute_provider` contains global compute + provider settings. +- `experimental_multi_runner_config` contains per-runner configuration + overrides and provider selections. + +For example: + +```hcl +experimental_global_config = { + tags = { + Environment = "ci" + } +} + +experimental_global_config_observability = { + metrics = { + enabled = true + metric = { + github_app_rate_limit = { enabled = true } + job_retry = { enabled = true } + } + } +} + +experimental_global_config_orchestration_provider = { + webhook = { + eventbridge = { + enabled = true + } + } +} + +experimental_multi_runner_config = { + linux_arm64 = { + orchestration_provider = { + webhook = { + runner = { + boot_time_in_minutes = 5 + ephemeral = true + jit_config_enabled = null + maximum_count = 4 + } + + github = { + organization_runners = true + } + + matcherConfig = { + labelMatchers = [["linux", "arm64"]] + } + } + } + + compute_provider = { + aws = { + ec2 = { + instance_types = ["m7g.large"] + on_demand_failover_for_errors = ["InsufficientInstanceCapacity"] + instance_termination_watcher = { + features = { + runner_deregistration = { enabled = true } + spot_termination_handler = { enabled = true } + spot_termination_notification_watcher = { enabled = true } + } + } + } + } + } + } +} +``` + +The populated provider wrapper selects the provider; selection is not based on +a string discriminator. The wrapper's nullness and every value that controls +resource shape must be known during planning. Other values inside the selected +provider may remain unknown until apply. + +### Provider selection is per runner configuration + +Every entry in `experimental_multi_runner_config` must contain exactly one +non-null typed `orchestration_provider` block and exactly one non-null typed +`compute_provider` block. In this phase the supported blocks are: + +- `orchestration_provider.webhook` +- `compute_provider.aws.ec2` + +Validation counts non-null provider blocks rather than naming one special case. +A future provider can therefore be added as a sibling without changing the +selection rule. Different runner configurations may select different +providers once more than one exists, but one runner configuration cannot combine +providers. + +### Global provider blocks provide defaults; they do not select providers + +`experimental_global_config_orchestration_provider.webhook` is the global +defaults and shared-component namespace for webhook orchestration. Its +presence does not select webhook orchestration for every runner configuration. +Selection remains under +`experimental_multi_runner_config..orchestration_provider`. + +The global webhook namespace owns queue selection, EventBridge routing, +matcher-parameter tier, repository filtering, build-queue defaults, redrive +behavior, encryption, shared webhook Lambda settings, the runner-control +artifact, and default scale-up, scale-down, and pool settings. + +Job-retry remains a per-runner webhook setting in this phase. Its typed block +supplies its own defaults rather than inheriting a global job-retry block. + +For a selected webhook provider, resolution follows: + +```text +runner configuration override > experimental global webhook default +``` + +Tag maps merge from broad to narrow. A runner-specific override affects only +that runner configuration; it does not configure a shared singleton. + +### Canonical names describe ownership and enablement + +Nested feature groups use an `enabled` field: + +- `observability.metrics.enabled` +- `observability.metrics.metric.github_app_rate_limit.enabled` +- `observability.metrics.metric.job_retry.enabled` +- `observability.metrics.metric.spot_termination_warning.enabled` +- `instance_termination_watcher.features.runner_deregistration.enabled` +- `instance_termination_watcher.features.spot_termination_handler.enabled` +- `instance_termination_watcher.features.spot_termination_notification_watcher.enabled` + +Standalone settings remain descriptive names ending in `_enabled`, for +example `managed_security_group_enabled`, `jit_config_enabled`, +`job_queued_check_enabled`, `detailed_monitoring_enabled`, and `ssm_enabled`. +The EC2 failover list is named `on_demand_failover_for_errors`. + +Runner-binary configuration is owned by the compute provider's +`runner_binaries` block. It does not publish a global `targets` map. Binary +targets are derived from the resolved runner configurations that enable binary +synchronization, so the binary module does not depend on the effective +configuration it helps produce. + +### Module ownership follows the provider boundary + +| Layer | Responsibility | +| --- | --- | +| `modules/multi-runner` | Translates stable inputs, resolves global and per-runner values, owns shared ingress and queues, and routes typed provider objects. | +| `modules/runner-config` | Composes provider-neutral runner resources, selects exactly one orchestration provider and one compute provider, and connects provider capabilities. | +| `modules/orchestration-providers/webhook` | Owns webhook orchestration composition, defaults, tag layering, and scale, pool, and retry leaf modules. | +| `modules/orchestration-providers/webhook/scale-runners` | Owns scale-up and scale-down Lambdas, schedules, queue integration, IAM, and outputs. | +| `modules/orchestration-providers/webhook/pool` | Owns optional scheduled pool resources and IAM. | +| `modules/orchestration-providers/webhook/job-retry` | Owns optional queued-job retry resources and IAM. | +| `modules/runner-config/ssm-housekeeper` | Owns provider-neutral cleanup of runner token and configuration parameters. | +| `modules/compute-providers//` | Owns provider-specific capacity resources and returns policy, environment, managed-policy, and resource capabilities. | + +Provider leaf modules live below `modules/orchestration-providers/`, +not below `modules/runner-config`. This keeps the common composition module +small and prevents provider-owned resources from becoming part of the common +contract. + +```mermaid +flowchart TD + Multi["multi-runner: translate and resolve"] --> Config["runner-config: compose one runner config"] + Config --> Orchestration{"exactly one orchestration provider"} + Orchestration --> Webhook["orchestration-providers/webhook"] + Orchestration -. future .-> ScaleSet["orchestration-providers/scale-set"] + Config --> Compute{"exactly one compute provider"} + Compute --> EC2["compute-providers/aws/ec2"] + EC2 --> Capabilities["compute capabilities"] + Capabilities --> Webhook + Webhook --> Scale["scale-runners"] + Webhook --> Pool["pool"] + Webhook --> Retry["job-retry"] +``` + +### Compute providers expose capabilities, not orchestration resources + +The selected compute provider remains independent from the selected +orchestration provider. It owns capacity resources and supplies the policy, +environment-variable, managed-policy, trust-policy, and resource capabilities +needed by the selected orchestration implementation. + +`runner-config` adapts those outputs into the capabilities consumed by webhook +orchestration. The webhook provider owns its Lambda roles and attaches the +capability fragments it needs. The compute provider does not create webhook +resources. + +This direction keeps the dependency graph one-way: + +```text +runner-config -> compute provider -> capability contract -> orchestration provider +``` + +A future scale-set controller may require a different subset or extension of +the capability contract. That extension belongs at the provider boundary; it +must not add scale-set conditionals to webhook leaves. + +### Compatibility and state are explicit + +Stable inputs are translated into the same internal canonical representation so +defaults and shared singleton values have one resolution path. Stable runner +configurations continue to use the existing `modules/runners` implementation. +Opting into experimental v2 is module-wide: a non-empty +`experimental_multi_runner_config` replaces, rather than merges with, the +stable `multi_runner_config` map. + +The canonical v2 output groups orchestration resources under +`orchestration_provider.webhook` and compute resources under the selected +namespace and provider, currently `provider.aws.ec2`. Compatibility aliases +may remain during the experimental transition, but new consumers must use the +canonical paths. + +This ADR does not define automatic stable-v1-to-v2 state migration. Existing +deployments remain on the stable path until that migration is separately +designed and documented. + +### Existing shared modules stay unchanged + +The orchestration-provider boundary does not change the public contracts or +resource addresses of `modules/webhook` or `modules/ssm`. + +The shared webhook remains at its existing module address. The shared SSM module +continues to create or reference the webhook secret even when no runner +configuration selects webhook orchestration. That singleton contract is +independent of per-runner exact-one provider selection. + +Any later proposal to make those shared modules conditional is a separate +compatibility and state decision. + +### Scale-set implementation is deferred + +No `scale_set` field is added in this phase. The typed object and module layout +reserve the extension point without publishing an incomplete contract. + +A follow-up design must decide the public SDK surface, authentication and API +versioning, reconciliation and persistence, controller recovery and +concurrency, container release, ECS networking and scaling, compute-provider +capabilities, and Terraform migration/coexistence behavior. + +The intended end state permits webhook and scale-set orchestration in the same +multi-runner module instance when different runner configurations select them. +It does not permit both controllers to own the same runner configuration. + +## Consequences + +### Positive + +- A future orchestration provider becomes a sibling module instead of a + cross-cutting conditional. +- Common runner settings and compute-provider configuration remain reusable. +- Provider-owned queue, Lambda, artifact, IAM, and output settings have one + discoverable namespace. +- Exact-one validation prevents ambiguous ownership of a runner configuration. +- Stable behavior and shared singleton addresses remain unchanged. + +### Negative + +- The experimental input is more deeply nested than the existing flat + interface and is split across several global variables. +- Global webhook defaults and per-runner webhook selection have similarly named + blocks with different purposes. +- Adapter objects and capability contracts require maintenance. +- A stateful provider will still require separate runtime, deployment, + observability, and failure-recovery design. + +## Alternatives Considered + +### Add a flat orchestration mode string + +A value such as `orchestration_type = "webhook"` plus flat settings would make +unrelated fields valid for every provider and require cross-field validation. + +**Decision**: Use typed nullable sibling blocks. The populated block both +selects and configures the provider. + +### Put provider conditionals directly in `runner-config` + +This would keep fewer directories initially, but every provider would add +resources, variables, IAM branches, and outputs to the common module. + +**Decision**: Keep `runner-config` as selector and composer. Put concrete +resources under `modules/orchestration-providers/`. + +### Keep webhook leaves under `runner-config` + +Scale, pool, and retry are webhook orchestration behavior. Leaving them under +the common module would blur ownership and make a future provider appear to +support components it does not use. + +**Decision**: Keep those leaves under the webhook provider root. + +### Add the scale-set schema and ECS service now + +Publishing placeholders would lock in names and types before the API client, +reconciliation semantics, and runtime model have been validated. + +**Decision**: Publish only the provider-neutral extension point now. + +### Make the shared webhook and webhook secret conditional + +That would alter existing singleton addresses and conflate module-level ingress +with per-runner provider selection. + +**Decision**: Leave `modules/webhook` and `modules/ssm` unchanged. + +## Migration and Verification + +Implementation and review must verify the boundary at several levels: + +- A runner configuration with exactly one provider of each type plans + successfully; zero or multiple selections fail with focused messages. +- Provider-wrapper nullness and other graph-shaping values are plan-known. +- Per-runner values override global defaults, and omitted nullable values inherit + them. +- Shared singleton resources consume global values rather than arbitrary + per-runner overrides. +- Stable inputs preserve stable resource addresses and output shape. +- Runner-config routes only the selected orchestration provider and compute + capabilities reach the correct provider component. +- Canonical nested metric and feature settings are consumed by all provider + modules. +- Existing `modules/webhook` and `modules/ssm` contracts and addresses remain + unchanged. +- Stable and experimental Terraform tests, formatting, documentation + generation, and repository checks pass. + +Before an existing experimental deployment adopts a module rename, operators +must migrate affected state explicitly and confirm that the plan contains no +unintended replacements. Stable deployments must not enable v2 until a +stable-to-v2 migration procedure exists. + +## References + +- [GitHub Actions Runner Scale Set client](https://github.com/actions/scaleset) diff --git a/docs/examples/index.md b/docs/examples/index.md index aee1d868b0..f0558966bd 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -5,6 +5,7 @@ Examples are located in the [examples](https://github.com/github-aws-runners/ter - _[Default](default.md)_: The default example of the module - _[Ephemeral](ephemeral.md)_: Example usages of ephemeral runners based on the default example. - _[Multi Runner](multi-runner.md)_ : Example usage of creating a multi runner which creates multiple runners/ configurations with a single deployment. The examples including: "arm64", "windows", and "ubuntu" runners. +- _[Multi Runner v2](multi-runner-v2.md)_ : Example usage of the experimental v2 multi-runner configuration interface with shared defaults and per-lane overrides. - _[Permissions boundary](permissions-boundary.md)_: Example usages of permissions boundaries. - _[Prebuilt Images](prebuilt.md)_: Example usages of deploying runners with a custom prebuilt image. - _[Termination watcher](termination-watcher.md)_: Example usages of termination watcher. diff --git a/docs/examples/multi-runner-v2.md b/docs/examples/multi-runner-v2.md new file mode 100644 index 0000000000..565b601ecb --- /dev/null +++ b/docs/examples/multi-runner-v2.md @@ -0,0 +1 @@ +--8<-- "examples/multi-runner-v2/README.md" diff --git a/examples/multi-runner-v2/.terraform.lock.hcl b/examples/multi-runner-v2/.terraform.lock.hcl new file mode 100644 index 0000000000..877f39a156 --- /dev/null +++ b/examples/multi-runner-v2/.terraform.lock.hcl @@ -0,0 +1,89 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.63.0" + constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" + hashes = [ + "h1:dRlYHkc+r6fgzF57WC7Zjcmb6sF/6TTGDEgwGK+LAZY=", + "zh:005d56736afd17d963998c405cee6f434dbc23a415109f9435ff1542879ae611", + "zh:026ef126321a86ad7080b5d858e2527f96f5289678cbcd8856296e229c43339d", + "zh:06e0b58b2d1eddb5137fc86bee7ad2d07953c0bc3f57cccfc5ae0d2456068a3a", + "zh:07221735d61ababed84734e5ffcfc5bd59d01f29f029166ba5f2175895dceed1", + "zh:1a72db00583112bdb8c19b213a78a3f5de754fffc08f07e061f4e326289fab7d", + "zh:32968e74a53b03e97a084dc7050c22ef661fb5b3ea8a44f5a63e47bc45ad0e7c", + "zh:4b357dfe4b820e3e4acd2881cff8288b2186491e63416751f0d12692ba478ceb", + "zh:81e30884d7de686265e7d87bb92527e802878c65a378470ede2a1e9f4e40ccc9", + "zh:82e137297f6a5a08b9ce2138f7aabea245ad99495d9d9eff502f752d6ca90dbd", + "zh:8eb83b67099f0ea9df238a979dff933ff50ce06a2e3ff05a48556a10f10dd204", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:d0ba30886cbe41850fee689f51ef9088578f323cfd21817bb409951d43c465eb", + "zh:dd48e7089784454bc03d713e9057f5ca0ea1613bd402125054a51894957b7925", + "zh:f250fa81e54cf60fcb0e9c0fc4ac043f1ecc2ac24967f628b3609364fcab3d04", + "zh:f38fc09fc25a8d2cf89a4d4cd6a5ef7cb1aad72798dbdcad58b8876b6a551a54", + "zh:f7c7380fdf126e1901f2084588dbfd724c76cb131ccfa795a541219111103c06", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.9.0" + constraints = "~> 2.0" + hashes = [ + "h1:m24fjcInWvTVZ1XSo2MaNuKPe+X/gfG8SIi09rA7a7M=", + "zh:0baa4566cf77f1ff52f4293d1c8536202dd23edc197c3196413a28343c3ac3a0", + "zh:16b5559c3c07088ddad11a9bb9e9c0799999363c2958e9a5be2bcbbf2cd9ca64", + "zh:197c79015a10d1cce904a8ea722cbc750c42aeae2da53f44a6a0751d9fd1aa90", + "zh:29d0b03e5343a80677ebfeb2e2c31cbe4b1f65e736e53417454a4277fec2544c", + "zh:4896bfa6cf1d2fd562b47ef2e87f47862ae92a04f8ad5d764380f0c6653473b8", + "zh:531f8529cbca49f681883e57761a05a8398afaef6d1ab0d205d26bf12f4428e8", + "zh:6aaf5011d83161c86d2bfb80c0923ec934e578288758da2f37acb7aec129004b", + "zh:7430275253d3d3c40aa6179e0ec0d63212874dbbc06c5a51b9d07ec590f9756c", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:be17dc611e95e26cdf6cad79dfccf1064f0e32032a2efeb939a9bbe7fb1cbfe9", + "zh:f0e3b0aa644202e1d79d2000dca91f6019425da71e9800fa23f27e51c034f195", + "zh:f62bae4519e4ead49182ddc8afe8cf61e2a4c3ba3973b0fbba967736a2696aa3", + "zh:fcafa360a5b0b96244f26f4e3a6d642b716a376557142c2442ff2fb12d11da18", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.3.1" + constraints = "~> 3.0, ~> 3.2" + hashes = [ + "h1:m5FqidbIgh+E9OigiZh8/xbkvpUQFSj3hZo/jqNLCLQ=", + "zh:08c59776542ea16e5a8545752787b17ff412922182b4cfabe16139197be8ac44", + "zh:123109cc7e5ed6d515787fbc212f2a3fd5e75647bb24ab7c801ccd4d4ed42451", + "zh:14b3fa4372754b54844b41d5dbd4671a292d8d6828b90169061feb4d7b15dd05", + "zh:56a4daaa3212f57b764bf3d1f333141c6610c5f21abb240e0111221f7c7fa4d4", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7e888a026dbacd2474a42264227ae35f639780f0f0c613529d10a95cd61988b3", + "zh:85a53646267e87d600df7124e4767ffde9bba3b6356d45d961618bdd68131cc7", + "zh:8ffa0e9c7c39b2ab0905b472465d6e35ef0b776b3f6273bb34c150340b61bff1", + "zh:9846510a1841530d4403f4818e233f91e3b3bade7441047599fbf800742f65be", + "zh:afa98d44860875f037c6def0a7e6ff208e042712ba771f620482b143cd336891", + "zh:bdca130d9ef27488ae0b13bc8fd8019e8bbdd4f2ceff29da066bd333165d68c5", + "zh:cb3b94cbca88210dd0d1f11e2b8a89333f48c3857faf8f70f589072ce7c28610", + "zh:f0c0ba87925fe32f84b80f7513b1efb1b0866f51f899ba825e95ad59ff09b018", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.9.0" + constraints = "~> 3.0" + hashes = [ + "h1:OO+IuvQJSPmWdN8AyyIEvPJbLvDQpgX/zbktoa9KsJE=", + "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", + "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", + "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", + "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", + "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", + "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", + "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", + "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", + "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", + "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", + "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", + ] +} diff --git a/examples/multi-runner-v2/README.md b/examples/multi-runner-v2/README.md new file mode 100644 index 0000000000..30edd11cdd --- /dev/null +++ b/examples/multi-runner-v2/README.md @@ -0,0 +1,74 @@ +# Multi-runner v2 example + +This example demonstrates the experimental multi-runner v2 interface. Shared +defaults are configured with `experimental_global_config*` variables, while +each runner lane uses `experimental_multi_runner_config` for its matcher, +runner lifecycle, and compute-provider settings. + +The example creates three lanes from one deployment: + +- Linux ARM64 Amazon Linux runners. +- Ephemeral Linux x64 Amazon Linux runners with job retry enabled. +- Windows x64 Server Core 2022 runners. + +The v2 interface keeps provider-owned settings inside the selected provider +configuration. For example, VPC and subnet settings are under +`experimental_global_config_compute_provider.aws.ec2`, while the per-lane +instance types and AMI filter are under each lane's compute provider block. + +Configure the GitHub App variables before applying: + +```bash +terraform init +terraform apply \ + -var='github_app={id="123456",key_base64="..."}' +``` + +The `github_app` value is sensitive and should be supplied through a secure +variable source in real deployments rather than committed to configuration. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | +| [local](#requirement\_local) | ~> 2.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [random](#provider\_random) | 3.9.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [base](#module\_base) | ../base | n/a | +| [runners](#module\_runners) | ../../modules/multi-runner | n/a | +| [webhook\_github\_app](#module\_webhook\_github\_app) | ../../modules/webhook-github-app | n/a | + +## Resources + +| Name | Type | +|------|------| +| [random_id.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no | +| [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no | +| [github\_app](#input\_github\_app) | GitHub App ID and base64-encoded private key. |
object({
id = string
key_base64 = string
})
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [webhook\_endpoint](#output\_webhook\_endpoint) | n/a | +| [webhook\_secret](#output\_webhook\_secret) | n/a | + diff --git a/examples/multi-runner-v2/main.tf b/examples/multi-runner-v2/main.tf new file mode 100644 index 0000000000..de63ac962b --- /dev/null +++ b/examples/multi-runner-v2/main.tf @@ -0,0 +1,173 @@ +locals { + environment = var.environment != null ? var.environment : "multi-runner-v2" + aws_region = var.aws_region +} + +resource "random_id" "random" { + byte_length = 20 +} + +module "base" { + source = "../base" + + prefix = local.environment + aws_region = local.aws_region +} + +module "runners" { + source = "../../modules/multi-runner" + + prefix = local.environment + aws_region = local.aws_region + + experimental_global_config = { + tags = { + Example = local.environment + Project = "ProjectX" + } + runner = { + os = "linux" + architecture = "x64" + extra_labels = ["v2"] + } + } + + experimental_global_config_github = { + app = { + key_base64 = var.github_app.key_base64 + id = var.github_app.id + webhook_secret = random_id.random.hex + } + } + + experimental_global_config_lambda = { + architecture = "arm64" + } + + experimental_global_config_orchestration_provider = { + webhook = { + eventbridge = { + enabled = true + accept_events = ["workflow_job"] + } + } + } + + experimental_global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = module.base.vpc.vpc_id + subnet_ids = module.base.vpc.private_subnets + ssm_enabled = true + runner_binaries = { + enabled = true + } + } + } + } + + experimental_multi_runner_config = { + linux-arm64 = { + runner = { + architecture = "arm64" + name_prefix = "amazon-arm64-" + extra_labels = ["amazon"] + } + orchestration_provider = { + webhook = { + runner = { + maximum_count = 1 + } + matcherConfig = { + exactMatch = true + labelMatchers = [["self-hosted", "linux", "arm64", "amazon"]] + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = ["t4g.large", "c6g.large"] + } + } + } + } + + linux-x64 = { + runner = { + name_prefix = "amazon-x64-" + extra_labels = ["amazon"] + } + orchestration_provider = { + webhook = { + runner = { + ephemeral = true + maximum_count = 1 + } + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64", "amazon"]] + exactMatch = false + priority = 1 + } + queue = { + delay_webhook_event = 0 + } + job_retry = { + enabled = true + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = ["m5a.large", "m5ad.large"] + } + } + } + } + + windows-x64 = { + runner = { + os = "windows" + name_prefix = "windows-x64-" + } + orchestration_provider = { + webhook = { + runner = { + boot_time_in_minutes = 20 + maximum_count = 1 + } + matcherConfig = { + exactMatch = true + labelMatchers = [["self-hosted", "windows", "x64", "servercore-2022"]] + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = ["m5.large", "c5.large"] + ami = { + filter = { + name = ["Windows_Server-2022-English-Full-ECS_Optimized-*"] + state = ["available"] + } + } + } + } + } + } + } +} + +module "webhook_github_app" { + source = "../../modules/webhook-github-app" + depends_on = [module.runners] + + github_app = { + key_base64 = var.github_app.key_base64 + id = var.github_app.id + webhook_secret = random_id.random.hex + } + webhook_endpoint = module.runners.webhook.endpoint +} diff --git a/examples/multi-runner-v2/outputs.tf b/examples/multi-runner-v2/outputs.tf new file mode 100644 index 0000000000..1feaf2e671 --- /dev/null +++ b/examples/multi-runner-v2/outputs.tf @@ -0,0 +1,8 @@ +output "webhook_endpoint" { + value = module.runners.webhook.endpoint +} + +output "webhook_secret" { + sensitive = true + value = random_id.random.hex +} diff --git a/examples/multi-runner-v2/providers.tf b/examples/multi-runner-v2/providers.tf new file mode 100644 index 0000000000..eca2fe96a7 --- /dev/null +++ b/examples/multi-runner-v2/providers.tf @@ -0,0 +1,9 @@ +provider "aws" { + region = local.aws_region + + default_tags { + tags = { + Example = local.environment + } + } +} diff --git a/examples/multi-runner-v2/variables.tf b/examples/multi-runner-v2/variables.tf new file mode 100644 index 0000000000..2a7f7eda54 --- /dev/null +++ b/examples/multi-runner-v2/variables.tf @@ -0,0 +1,23 @@ +variable "github_app" { + description = "GitHub App ID and base64-encoded private key." + + type = object({ + id = string + key_base64 = string + }) + sensitive = true +} + +variable "environment" { + description = "Environment name, used as prefix." + + type = string + default = null +} + +variable "aws_region" { + description = "AWS region to deploy to." + + type = string + default = "eu-west-1" +} diff --git a/examples/multi-runner-v2/versions.tf b/examples/multi-runner-v2/versions.tf new file mode 100644 index 0000000000..1dfb3e5774 --- /dev/null +++ b/examples/multi-runner-v2/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + local = { + source = "hashicorp/local" + version = "~> 2.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + required_version = ">= 1.4.0" +} diff --git a/examples/multi-runner/README.md b/examples/multi-runner/README.md index a899015568..d87b344e8b 100644 --- a/examples/multi-runner/README.md +++ b/examples/multi-runner/README.md @@ -56,7 +56,7 @@ terraform output -raw webhook_secret | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [terraform](#requirement\_terraform) | >= 1.4.0 | | [aws](#requirement\_aws) | >= 6.33 | | [local](#requirement\_local) | ~> 2.0 | | [random](#requirement\_random) | ~> 3.0 | diff --git a/examples/multi-runner/versions.tf b/examples/multi-runner/versions.tf index 666b978aac..1dfb3e5774 100644 --- a/examples/multi-runner/versions.tf +++ b/examples/multi-runner/versions.tf @@ -13,5 +13,5 @@ terraform { version = "~> 3.0" } } - required_version = ">= 1.3.0" + required_version = ">= 1.4.0" } diff --git a/mkdocs.yaml b/mkdocs.yaml index d892cb3a58..1b190a4a67 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -75,6 +75,7 @@ nav: - Overview: examples/index.md - Default: examples/default.md - Multi Runner: examples/multi-runner.md + - Multi Runner v2: examples/multi-runner-v2.md - Ephemeral: examples/ephemeral.md - External managed secrets: examples/external-managed-ssm-secrets.md - Custom AMI: examples/prebuilt.md diff --git a/modules/compute-providers/aws/ec2/README.md b/modules/compute-providers/aws/ec2/README.md new file mode 100644 index 0000000000..aa2ad83651 --- /dev/null +++ b/modules/compute-providers/aws/ec2/README.md @@ -0,0 +1,80 @@ +# EC2 runner provider + +This internal module owns the EC2 compute implementation used by the common runner configuration. It creates the runner launch template, security group, instance profile, EC2 bootstrap parameters, and runner log groups. + +The module returns one nested `provider` contract. It groups EC2-specific Lambda settings under `environment_variables`, permission requirements under `policies.runner`, `policies.scale_up`, `policies.scale_down`, and `policies.pool`, and EC2 artifacts under `resources`. The parent runner configuration owns the shared runner role, provider-policy attachments, Lambda functions, execution roles, schedules, queues, retry flow, and SSM housekeeper. + +EC2 is the only active compute provider. The parent runner configuration selects it when `aws.ec2` is the one populated typed leaf under `compute_provider`; no separate namespace or type input is required. Runner-config dispatches this module at `module.compute_aws_ec2[0]` from the `modules/compute-providers/aws/ec2` source and publishes its resources under `provider.aws.ec2`. A future provider must add its own typed namespace and provider leaf and implement the same contracts before it can be selected. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.gh_runners](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_instance_profile.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_policy.ami_id_ssm_parameter_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_launch_template.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_security_group.runner_sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_ssm_parameter.cloudwatch_agent_config_runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_ami_id](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_config_run_as](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_enable_cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [terraform_data.validate_config](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.validate_runner](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [aws_ami.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.ami_id_ssm_parameter_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.create_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.describe_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.distribution_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.service_linked_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.session_manager](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.ssm_parameters](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.terminate_self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | AWS partition used to construct IAM ARNs. | `string` | `"aws"` | no | +| [aws\_region](#input\_aws\_region) | AWS region used by compute-provider resources and policy documents. | `string` | n/a | yes | +| [config](#input\_config) | EC2 compute-provider configuration. Paths match `compute_provider.aws.ec2` in the runner configuration.

- `ami`: Optional AMI discovery and encryption configuration. Null selects defaults for `runner.os` and `runner.architecture`.
- `ami.filter`: AMI filter names mapped to accepted values and merged over the provider defaults.
- `ami.owners`: AWS account IDs or aliases allowed to own the selected AMI.
- `ami.id_ssm_parameter`: Optional externally managed SSM parameter containing the AMI ID. Its object presence is the plan-time ownership discriminator.
- `ami.id_ssm_parameter.arn`: ARN of the external AMI-ID parameter. The ARN may remain unknown until apply.
- `ami.kms_key`: Optional customer-managed KMS key required for encrypted AMIs or snapshots. Its object presence is the plan-time policy discriminator.
- `ami.kms_key.arn`: ARN of the AMI KMS key. The ARN may remain unknown until apply.
- `vpc_id`: VPC in which runner networking resources are created.
- `subnet_ids`: Subnets from which the control plane may launch runners.
- `overrides.name_runner`: Optional Name tag override for runner compute resources.
- `overrides.name_sg`: Optional Name tag override for the managed security group.
- `instance_profile`: Optional externally managed instance profile. Its object presence is the plan-time ownership discriminator.
- `instance_profile.name`: Name of the external instance profile. The name may remain unknown until apply.
- `instance_profile_path`: IAM path for the provider-managed instance profile. Null derives the path from `prefix`.
- `binaries_syncer.enabled`: Uses the synchronized runner distribution from S3 during bootstrap.
- `binaries_syncer.s3`: S3 object containing the synchronized runner distribution. Required when synchronization is enabled.
- `binaries_syncer.s3.arn`: Runner-distribution bucket ARN used by IAM policies.
- `binaries_syncer.s3.id`: Runner-distribution bucket name used in the bootstrap URI.
- `binaries_syncer.s3.key`: Runner-distribution object key.
- `block_device_mappings`: EBS mappings added to the launch template.
- `block_device_mappings[].delete_on_termination`: Deletes the volume when its runner terminates.
- `block_device_mappings[].device_name`: Device name exposed to the runner instance.
- `block_device_mappings[].encrypted`: Enables EBS encryption.
- `block_device_mappings[].iops`: Provisioned IOPS for volume types that support configurable IOPS.
- `block_device_mappings[].kms_key_id`: KMS key ID or ARN used to encrypt the volume.
- `block_device_mappings[].snapshot_id`: Snapshot used to initialize the volume.
- `block_device_mappings[].throughput`: Provisioned throughput for volume types that support it.
- `block_device_mappings[].volume_initialization_rate`: Fixed initialization rate for supported snapshot-backed volumes.
- `block_device_mappings[].volume_size`: EBS volume size in GiB.
- `block_device_mappings[].volume_type`: EBS volume type.
- `ebs_optimized`: Requests EBS-optimized instances.
- `instance_target_capacity_type`: Primary capacity type, either `spot` or `on-demand`.
- `instance_allocation_strategy`: EC2 Fleet allocation strategy.
- `instance_type_priorities`: Optional numeric priorities keyed by instance type.
- `instance_max_spot_price`: Optional maximum hourly Spot price.
- `instance_types`: EC2 instance types available to the control plane.
- `user_data`: Runner bootstrap user-data configuration.
- `user_data.enabled`: Enables launch-template user data.
- `user_data.template`: Optional path to a custom user-data template.
- `user_data.content`: Optional complete user-data content used instead of a template.
- `user_data.pre_install`: Script inserted before runner installation.
- `user_data.post_install`: Script inserted after runner installation.
- `user_data.debug_logging_enabled`: Enables verbose user-data tracing, which can expose secrets.
- `ssm_enabled`: Includes Session Manager permissions in the provider's runner policy group.
- `create_service_linked_role_spot`: Allows scale-up to create the EC2 Spot service-linked role.
- `cloudwatch_agent.enabled`: Enables CloudWatch agent configuration for runner instances.
- `cloudwatch_agent.config`: Optional complete CloudWatch agent configuration.
- `managed_security_group_enabled`: Creates and attaches the provider-managed security group.
- `log_files`: Optional files collected by the CloudWatch agent. Null uses provider defaults.
- `log_files[].log_group_name`: CloudWatch log-group name before optional prefixing.
- `log_files[].prefix_log_group`: Prefixes the log-group name with the runner configuration path.
- `log_files[].file_path`: File or glob read by the CloudWatch agent.
- `log_files[].log_stream_name`: CloudWatch log-stream name template.
- `log_files[].log_class`: CloudWatch log-group class for the collected file.
- `key_name`: Optional EC2 key-pair name.
- `additional_security_group_ids`: Existing security groups attached to runners.
- `detailed_monitoring_enabled`: Enables detailed EC2 monitoring.
- `egress_rules`: Rules created on the managed security group.
- `egress_rules[].cidr_blocks`: IPv4 CIDR destinations.
- `egress_rules[].ipv6_cidr_blocks`: IPv6 CIDR destinations.
- `egress_rules[].prefix_list_ids`: AWS prefix-list destinations.
- `egress_rules[].from_port`: First destination port in the permitted range.
- `egress_rules[].protocol`: IP protocol name or number. Use `-1` for all protocols.
- `egress_rules[].security_groups`: Destination security-group IDs.
- `egress_rules[].self`: Allows traffic to the managed security group itself.
- `egress_rules[].to_port`: Last destination port in the permitted range.
- `egress_rules[].description`: Optional rule description.
- `tags`: Runner instance, volume, network-interface, and eligible Spot-request tags. Provider-required bootstrap tags take final precedence.
- `metadata_options`: Instance Metadata Service configuration.
- `metadata_options.instance_metadata_tags`: Exposes instance tags through Instance Metadata Service when enabled.
- `metadata_options.http_endpoint`: Enables or disables the Instance Metadata Service endpoint.
- `metadata_options.http_tokens`: Controls whether IMDSv2 session tokens are optional or required.
- `metadata_options.http_put_response_hop_limit`: Network hop limit for Instance Metadata Service token responses.
- `credit_specification`: CPU credit mode for burstable instance types.
- `cpu_options`: CPU topology and processor-feature configuration.
- `cpu_options.core_count`: Number of CPU cores exposed to the runner instance.
- `cpu_options.threads_per_core`: Number of hardware threads exposed per CPU core.
- `cpu_options.amd_sev_snp`: Enables or disables AMD SEV-SNP on supported instance types.
- `cpu_options.nested_virtualization`: Enables or disables nested virtualization on supported instance types.
- `placement`: EC2 placement configuration.
- `placement.affinity`: Dedicated Host affinity setting.
- `placement.availability_zone`: Availability Zone in which runner instances are placed.
- `placement.group_id`: Placement-group ID.
- `placement.group_name`: Placement-group name.
- `placement.host_id`: Dedicated Host ID.
- `placement.host_resource_group_arn`: ARN of the host resource group used for placement.
- `placement.spread_domain`: Spread-domain placement value.
- `placement.tenancy`: Instance tenancy, such as `default`, `dedicated`, or `host`.
- `placement.partition_number`: Placement-group partition number.
- `license_specifications`: License Manager configurations added to the launch template.
- `license_specifications[].license_configuration_arn`: ARN of an AWS License Manager license configuration.
- `associate_public_ipv4_address`: Associates a public IPv4 address with runner network interfaces.
- `on_demand_failover_for_errors`: EC2 errors that trigger on-demand fallback after a Spot failure.
- `scale_errors`: EC2 errors treated as retryable scale-up failures.
- `use_dedicated_host`: Enables the dedicated-host launch path. |
object({
ami = optional(object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter = optional(object({
arn = string
}), null)
kms_key = optional(object({
arn = string
}), null)
}), null)
vpc_id = string
subnet_ids = list(string)
overrides = optional(object({
name_runner = optional(string, "")
name_sg = optional(string, "")
}), {})
instance_profile = optional(object({
name = string
}), null)
instance_profile_path = optional(string, null)
binaries_syncer = optional(object({
enabled = optional(bool, true)
s3 = optional(object({
arn = string
id = string
key = string
}), null)
}), {})
block_device_mappings = optional(list(object({
delete_on_termination = optional(bool, true)
device_name = optional(string, "/dev/xvda")
encrypted = optional(bool, true)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = number
volume_type = optional(string, "gp3")
})), [{ volume_size = 30 }])
ebs_optimized = optional(bool, false)
instance_target_capacity_type = optional(string, "spot")
instance_allocation_strategy = optional(string, "lowest-price")
instance_type_priorities = optional(map(number), null)
instance_max_spot_price = optional(string, null)
instance_types = list(string)
user_data = optional(object({
enabled = optional(bool, true)
template = optional(string, null)
content = optional(string, null)
pre_install = optional(string, "")
post_install = optional(string, "")
debug_logging_enabled = optional(bool, false)
}), {})
ssm_enabled = optional(bool, false)
create_service_linked_role_spot = optional(bool, false)
cloudwatch_agent = optional(object({
enabled = optional(bool, true)
config = optional(string, null)
}), {})
managed_security_group_enabled = optional(bool, true)
log_files = optional(list(object({
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
log_class = optional(string, "STANDARD")
})), null)
key_name = optional(string, null)
additional_security_group_ids = optional(list(string), [])
detailed_monitoring_enabled = optional(bool, false)
egress_rules = optional(list(object({
cidr_blocks = list(string)
ipv6_cidr_blocks = list(string)
prefix_list_ids = list(string)
from_port = number
protocol = string
security_groups = list(string)
self = bool
to_port = number
description = string
})), [{
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
prefix_list_ids = null
from_port = 0
protocol = "-1"
security_groups = null
self = null
to_port = 0
description = null
}])
tags = optional(map(string), {})
metadata_options = optional(object({
instance_metadata_tags = optional(string, "enabled")
http_endpoint = optional(string, "enabled")
http_tokens = optional(string, "required")
http_put_response_hop_limit = optional(number, 1)
}), {})
credit_specification = optional(string, null)
cpu_options = optional(object({
core_count = optional(number)
threads_per_core = optional(number)
amd_sev_snp = optional(string)
nested_virtualization = optional(string)
}), null)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_id = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
spread_domain = optional(string)
tenancy = optional(string)
partition_number = optional(number)
}), null)
license_specifications = optional(list(object({
license_configuration_arn = string
})), [])
associate_public_ipv4_address = optional(bool, false)
on_demand_failover_for_errors = optional(list(string), [])
scale_errors = optional(list(string), [
"UnfulfillableCapacity",
"MaxSpotInstanceCountExceeded",
"TargetCapacityLimitExceededException",
"RequestLimitExceeded",
"ResourceLimitExceeded",
"MaxSpotInstanceCountExceeded",
"MaxSpotFleetRequestCountExceeded",
"InsufficientInstanceCapacity",
"InsufficientCapacityOnHost",
])
use_dedicated_host = optional(bool, false)
})
| n/a | yes | +| [github](#input\_github) | GitHub Enterprise Server settings available to compute-provider bootstrap data.

- `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com.
- `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server. |
object({
enterprise_server = optional(object({
url = optional(string, null)
ssl_verify = optional(bool, true)
}), {})
})
| `{}` | no | +| [observability](#input\_observability) | CloudWatch Logs settings available to compute-provider runner log groups.

- `logs.retention_in_days`: Retention period for provider-owned runner log groups.
- `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt runner log groups.
- `logs.tags`: Shared log-group tags that override module-level `tags`. |
object({
logs = optional(object({
retention_in_days = optional(number, 180)
kms_key_id = optional(string, null)
tags = optional(map(string), {})
}), {})
})
| `{}` | no | +| [prefix](#input\_prefix) | Prefix used to identify resources created for the runner configuration. | `string` | `"github-actions"` | no | +| [runner](#input\_runner) | Provider-neutral runner settings consumed by compute providers.

- `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`.
- `architecture`: Runner distribution architecture.
- `name_prefix`: Prefix added to registered runner names.
- `run_as_root`: Runs the runner service as root.
- `run_as`: Operating-system user used when `run_as_root` is false.
- `hooks.job_started`: Script installed as the runner job-started hook.
- `hooks.job_completed`: Script installed as the runner job-completed hook.
- `iam.role.arn`: Resolved runner-role ARN referenced by provider policies and resources.
- `iam.role.name`: Resolved runner-role name used by provider resources.
- `iam.role.managed`: Whether runner-config manages the resolved runner role.
- `iam.managed_policy_arns`: Common managed-policy ARNs returned with the provider-specific runner policies for attachment by runner-config.
- `iam.path`: IAM path available to provider-managed IAM resources. Null derives the path from `prefix`. |
object({
os = optional(string, "linux")
architecture = optional(string, "x64")
name_prefix = optional(string, "")
run_as_root = optional(bool, false)
run_as = optional(string, "ec2-user")
hooks = optional(object({
job_started = optional(string, "")
job_completed = optional(string, "")
}), {})
iam = object({
role = object({
arn = string
name = string
managed = optional(bool, true)
})
managed_policy_arns = optional(map(string), {})
path = optional(string, null)
})
})
| n/a | yes | +| [ssm](#input\_ssm) | Parameter Store paths and tag scopes available to compute-provider bootstrap resources.

- `paths.root`: Root Parameter Store path for the runner configuration.
- `paths.tokens`: Path segment used for registration tokens and just-in-time configuration.
- `paths.config`: Path segment used for persistent runner and provider configuration.
- `tags`: Shared SSM tags that override module-level `tags`.
- `parameters.tags`: Parameter-specific tags that override module-level and shared SSM tags. |
object({
paths = object({
root = string
tokens = string
config = string
})
tags = optional(map(string), {})
parameters = optional(object({
tags = optional(map(string), {})
}), {})
})
| n/a | yes | +| [tags](#input\_tags) | Base tags available to taggable compute-provider resources. Provider-specific tags override this map within their documented scopes. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [environment\_variables](#output\_environment\_variables) | Provider-specific Lambda environment variable fragments consumed by runner-config. | +| [policies](#output\_policies) | Provider-specific IAM policy fragments consumed by runner-config. | +| [provider](#output\_provider) | Nested EC2 compute-provider contract consumed by runner-config. | +| [resources](#output\_resources) | Provider-specific EC2 resources exposed by runner-config. | + diff --git a/modules/compute-providers/aws/ec2/control-plane.tf b/modules/compute-providers/aws/ec2/control-plane.tf new file mode 100644 index 0000000000..1b25442032 --- /dev/null +++ b/modules/compute-providers/aws/ec2/control-plane.tf @@ -0,0 +1,216 @@ +# EC2-specific IAM and environment fragments consumed by the common control +# plane in runner-config. +data "aws_iam_policy_document" "ami_id_ssm_parameter_read" { + count = local.ami_id_ssm_external ? 1 : 0 + + statement { + effect = "Allow" + actions = ["ssm:GetParameter"] + resources = [local.ami_id_ssm_parameter_arn] + } +} + +resource "aws_iam_policy" "ami_id_ssm_parameter_read" { + count = local.ami_id_ssm_external ? 1 : 0 + name = "${var.prefix}-ami-id-ssm-parameter-read" + path = local.role_path + description = "Allows for reading ${var.prefix} GitHub runner AMI ID from an SSM parameter" + tags = local.provider_tags + policy = data.aws_iam_policy_document.ami_id_ssm_parameter_read[0].json +} + +data "aws_iam_policy_document" "scale_up" { + statement { + effect = "Allow" + actions = [ + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeTags", + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateTags", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["ec2:TerminateInstances"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/ghr:Application" + values = ["github-action-runner"] + } + } + + statement { + effect = "Allow" + actions = ["ec2:TerminateInstances"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/ghr:environment" + values = [var.prefix] + } + } + + statement { + effect = "Allow" + actions = ["iam:PassRole"] + resources = [var.runner.iam.role.arn] + } + + statement { + effect = "Allow" + actions = ["ssm:GetParameter", "ssm:GetParameters"] + resources = [local.ami_id_ssm_module_managed ? aws_ssm_parameter.runner_ami_id[0].arn : local.ami_id_ssm_parameter_arn] + } + + dynamic "statement" { + for_each = local.ami_kms_key_enabled ? [local.ami_kms_key_arn] : [] + + content { + effect = "Allow" + actions = ["kms:DescribeKey", "kms:ReEncrypt*", "kms:Decrypt"] + resources = [statement.value] + } + } + + dynamic "statement" { + for_each = local.ami_kms_key_enabled ? [local.ami_kms_key_arn] : [] + + content { + effect = "Allow" + actions = ["kms:CreateGrant"] + resources = [statement.value] + + condition { + test = "Bool" + variable = "aws:ViaAWSService" + values = ["true"] + } + } + } +} + +data "aws_iam_policy_document" "scale_down" { + statement { + effect = "Allow" + actions = ["ec2:DescribeInstances", "ec2:DescribeTags"] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["ec2:TerminateInstances", "ec2:CreateTags", "ec2:DeleteTags"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/ghr:Application" + values = ["github-action-runner"] + } + } + + statement { + effect = "Allow" + actions = ["ec2:TerminateInstances", "ec2:CreateTags", "ec2:DeleteTags"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/ghr:environment" + values = [var.prefix] + } + } +} + +data "aws_iam_policy_document" "pool" { + statement { + effect = "Allow" + actions = [ + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateTags", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["iam:PassRole"] + resources = [var.runner.iam.role.arn] + } + + statement { + effect = "Allow" + actions = ["ssm:GetParameters"] + resources = [local.ami_id_ssm_module_managed ? aws_ssm_parameter.runner_ami_id[0].arn : local.ami_id_ssm_parameter_arn] + } + + dynamic "statement" { + for_each = local.ami_kms_key_enabled ? [local.ami_kms_key_arn] : [] + + content { + effect = "Allow" + actions = ["kms:DescribeKey", "kms:ReEncrypt*", "kms:Decrypt"] + resources = [statement.value] + } + } + + dynamic "statement" { + for_each = local.ami_kms_key_enabled ? [local.ami_kms_key_arn] : [] + + content { + effect = "Allow" + actions = ["kms:CreateGrant"] + resources = [statement.value] + + condition { + test = "Bool" + variable = "aws:ViaAWSService" + values = ["true"] + } + } + } +} + +data "aws_iam_policy_document" "service_linked_role" { + count = var.config.create_service_linked_role_spot ? 1 : 0 + + statement { + effect = "Allow" + actions = ["iam:CreateServiceLinkedRole"] + resources = ["arn:${var.aws_partition}:iam::*:role/aws-service-role/*"] + } +} + +locals { + scale_up_environment_variables = { + AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name + INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy + INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price + INSTANCE_TARGET_CAPACITY_TYPE = var.config.instance_target_capacity_type + INSTANCE_TYPE_PRIORITIES = var.config.instance_type_priorities != null ? jsonencode(var.config.instance_type_priorities) : "" + INSTANCE_TYPES = join(",", var.config.instance_types) + LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name + SUBNET_IDS = join(",", var.config.subnet_ids) + ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.config.on_demand_failover_for_errors) + SCALE_ERRORS = jsonencode(var.config.scale_errors) + USE_DEDICATED_HOST = var.config.use_dedicated_host + } + + scale_down_environment_variables = {} + + pool_environment_variables = local.scale_up_environment_variables + + scale_up_iam_policy_json = data.aws_iam_policy_document.scale_up.json + scale_down_iam_policy_json = data.aws_iam_policy_document.scale_down.json + pool_iam_policy_json = data.aws_iam_policy_document.pool.json + service_linked_role_policy_json = var.config.create_service_linked_role_spot ? data.aws_iam_policy_document.service_linked_role[0].json : null +} diff --git a/modules/compute-providers/aws/ec2/instance-profile.tf b/modules/compute-providers/aws/ec2/instance-profile.tf new file mode 100644 index 0000000000..f01a865f73 --- /dev/null +++ b/modules/compute-providers/aws/ec2/instance-profile.tf @@ -0,0 +1,9 @@ +# The common runner configuration owns the role; EC2 owns the profile consumed by its +# launch template. +resource "aws_iam_instance_profile" "runner" { + count = var.config.instance_profile == null ? 1 : 0 + name = "${var.prefix}-runner-profile" + role = var.runner.iam.role.name + path = local.instance_profile_path + tags = local.provider_tags +} diff --git a/modules/compute-providers/aws/ec2/logging.tf b/modules/compute-providers/aws/ec2/logging.tf new file mode 100644 index 0000000000..00ae952e4d --- /dev/null +++ b/modules/compute-providers/aws/ec2/logging.tf @@ -0,0 +1,75 @@ +# EC2 runner log collection and CloudWatch resources. +locals { + runner_log_files = ( + var.config.log_files != null + ? var.config.log_files + : [ + { + "prefix_log_group" : true, + "file_path" : "/var/log/messages", + "log_group_name" : "messages", + "log_stream_name" : "{instance_id}", + "log_class" : "STANDARD" + }, + { + "log_group_name" : "user_data", + "prefix_log_group" : true, + "file_path" : var.runner.os == "windows" ? "C:/UserData.log" : "/var/log/user-data.log", + "log_stream_name" : "{instance_id}", + "log_class" : "STANDARD" + }, + { + "log_group_name" : "runner", + "prefix_log_group" : true, + "file_path" : var.runner.os == "windows" ? "C:/actions-runner/_diag/Runner_*.log" : "/opt/actions-runner/_diag/Runner_**.log", + "log_stream_name" : "{instance_id}", + "log_class" : "STANDARD" + }, + { + "log_group_name" : "runner-startup", + "prefix_log_group" : true, + "file_path" : var.runner.os == "windows" ? "C:/runner-startup.log" : "/var/log/runner-startup.log", + "log_stream_name" : "{instance_id}", + "log_class" : "STANDARD" + } + ] + ) + # CloudWatch agent collect_list schema expects log_group_class, not log_class + logfiles = var.config.cloudwatch_agent.enabled ? [for l in local.runner_log_files : { + "log_group_name" : l.prefix_log_group ? "/github-self-hosted-runners/${var.prefix}/${l.log_group_name}" : "/${l.log_group_name}" + "log_stream_name" : l.log_stream_name + "file_path" : l.file_path + "log_group_class" : l.log_class + }] : [] + + loggroups_names = distinct([for l in local.logfiles : l.log_group_name]) + # Create a list of unique log classes corresponding to each log group name + # This maintains the same order as loggroups_names for use with count + loggroups_classes = [ + for name in local.loggroups_names : [ + for l in local.logfiles : l.log_group_class + if l.log_group_name == name + ][0] + ] + +} + + +resource "aws_ssm_parameter" "cloudwatch_agent_config_runner" { + count = var.config.cloudwatch_agent.enabled ? 1 : 0 + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/cloudwatch_agent_config_runner" + type = "String" + value = var.config.cloudwatch_agent.config != null ? var.config.cloudwatch_agent.config : templatefile("${path.module}/templates/cloudwatch_config.json", { + logfiles = jsonencode(local.logfiles) + }) + tags = local.ssm_parameter_tags +} + +resource "aws_cloudwatch_log_group" "gh_runners" { + count = length(local.loggroups_names) + name = local.loggroups_names[count.index] + retention_in_days = var.observability.logs.retention_in_days + kms_key_id = var.observability.logs.kms_key_id + log_group_class = local.loggroups_classes[count.index] + tags = local.log_group_tags +} diff --git a/modules/compute-providers/aws/ec2/outputs.tf b/modules/compute-providers/aws/ec2/outputs.tf new file mode 100644 index 0000000000..422383df0f --- /dev/null +++ b/modules/compute-providers/aws/ec2/outputs.tf @@ -0,0 +1,23 @@ +output "environment_variables" { + description = "Provider-specific Lambda environment variable fragments consumed by runner-config." + value = local.provider_environment_variables +} + +output "policies" { + description = "Provider-specific IAM policy fragments consumed by runner-config." + value = local.provider_policies +} + +output "resources" { + description = "Provider-specific EC2 resources exposed by runner-config." + value = local.provider_resources +} + +output "provider" { + description = "Nested EC2 compute-provider contract consumed by runner-config." + value = { + environment_variables = local.provider_environment_variables + policies = local.provider_policies + resources = local.provider_resources + } +} diff --git a/modules/compute-providers/aws/ec2/policies-runner.tf b/modules/compute-providers/aws/ec2/policies-runner.tf new file mode 100644 index 0000000000..c16077debc --- /dev/null +++ b/modules/compute-providers/aws/ec2/policies-runner.tf @@ -0,0 +1,206 @@ +# EC2 runner permission documents returned to runner-config for attachment to +# the common runner role. +data "aws_caller_identity" "current" {} + +locals { + ssm_parameter_arn_prefix = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter" + ssm_config_arn = "${local.ssm_parameter_arn_prefix}${var.ssm.paths.root}/${var.ssm.paths.config}" + cloudwatch_config_arn = "${local.ssm_config_arn}/cloudwatch_agent_config_runner" +} + +data "aws_iam_policy_document" "ssm_parameters" { + statement { + effect = "Allow" + actions = [ + "ssm:DeleteParameter", + "ssm:GetParameters", + "ssm:GetParameter", + ] + resources = [ + "${local.ssm_parameter_arn_prefix}${var.ssm.paths.root}/${var.ssm.paths.tokens}/*", + ] + + condition { + test = "StringLike" + variable = "ec2:SourceInstanceARN" + values = ["*/&{aws:ResourceTag/InstanceId}"] + } + } + + statement { + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:GetParametersByPath", + ] + resources = [ + local.ssm_config_arn, + "${local.ssm_config_arn}/*", + ] + } +} + +data "aws_iam_policy_document" "session_manager" { + statement { + effect = "Allow" + actions = [ + "ssm:DescribeAssociation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:DescribeDocument", + "ssm:GetManifest", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + ] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "distribution_bucket" { + count = var.config.binaries_syncer.enabled ? 1 : 0 + + statement { + sid = "githubActionDist" + effect = "Allow" + actions = ["s3:GetObject", "s3:GetObjectAcl"] + resources = ["${try(var.config.binaries_syncer.s3.arn, "")}/${try(var.config.binaries_syncer.s3.key, "")}"] + } +} + +data "aws_iam_policy_document" "describe_tags" { + statement { + effect = "Allow" + actions = ["ec2:DescribeTags"] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "create_tags" { + statement { + effect = "Allow" + actions = ["ec2:CreateTags"] + resources = ["arn:*:ec2:*:*:instance/*"] + + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = ["ghr:github_runner_id"] + } + + condition { + test = "StringEquals" + variable = "aws:ARN" + values = ["&{ec2:SourceInstanceARN}"] + } + } +} + +data "aws_iam_policy_document" "terminate_self" { + statement { + effect = "Allow" + actions = ["ec2:TerminateInstances"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:ARN" + values = ["&{ec2:SourceInstanceARN}"] + } + } +} + +data "aws_iam_policy_document" "cloudwatch" { + count = var.config.cloudwatch_agent.enabled ? 1 : 0 + + statement { + effect = "Allow" + actions = [ + "cloudwatch:PutMetricData", + "ec2:DescribeVolumes", + "ec2:DescribeTags", + "logs:PutLogEvents", + "logs:DescribeLogStreams", + "logs:DescribeLogGroups", + "logs:CreateLogStream", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["ssm:GetParameter"] + resources = ["${local.cloudwatch_config_arn}/*"] + } +} + +locals { + runner_inline_policies = merge( + { + ssm_parameters = { + name = "runner-ssm-parameters" + policy_json = data.aws_iam_policy_document.ssm_parameters.json + } + describe_tags = { + name = "runner-describe-tags" + policy_json = data.aws_iam_policy_document.describe_tags.json + } + create_tags = { + name = "runner-create-tags" + policy_json = data.aws_iam_policy_document.create_tags.json + } + terminate_self = { + name = "ec2" + policy_json = data.aws_iam_policy_document.terminate_self.json + } + }, + var.config.ssm_enabled ? { + session_manager = { + name = "runner-ssm-session" + policy_json = data.aws_iam_policy_document.session_manager.json + } + } : {}, + var.config.binaries_syncer.enabled ? { + distribution_bucket = { + name = "distribution-bucket" + policy_json = data.aws_iam_policy_document.distribution_bucket[0].json + } + } : {}, + var.config.cloudwatch_agent.enabled ? { + cloudwatch = { + name = "CloudWatchLogginAndMetrics" + policy_json = data.aws_iam_policy_document.cloudwatch[0].json + } + } : {}, + ) +} diff --git a/modules/compute-providers/aws/ec2/provider-contract.tf b/modules/compute-providers/aws/ec2/provider-contract.tf new file mode 100644 index 0000000000..5682496d78 --- /dev/null +++ b/modules/compute-providers/aws/ec2/provider-contract.tf @@ -0,0 +1,34 @@ +locals { + provider_environment_variables = { + scale_up = local.scale_up_environment_variables + scale_down = local.scale_down_environment_variables + pool = local.pool_environment_variables + } + + provider_policies = { + runner = { + inline_policies = local.runner_inline_policies + managed_policy_arns = var.runner.iam.managed_policy_arns + } + scale_up = { + iam_policy_json = local.scale_up_iam_policy_json + additional_iam_policy_json = local.service_linked_role_policy_json + managed_policy_enabled = local.ami_id_ssm_external + managed_policy_arn = local.ami_id_ssm_external ? aws_iam_policy.ami_id_ssm_parameter_read[0].arn : null + } + scale_down = { + iam_policy_json = local.scale_down_iam_policy_json + } + pool = { + iam_policy_json = local.pool_iam_policy_json + managed_policy_enabled = local.ami_id_ssm_external + managed_policy_arn = local.ami_id_ssm_external ? aws_iam_policy.ami_id_ssm_parameter_read[0].arn : null + } + } + + provider_resources = { + launch_template = aws_launch_template.runner + runners_log_groups = try(aws_cloudwatch_log_group.gh_runners, []) + logfiles = local.logfiles + } +} diff --git a/modules/compute-providers/aws/ec2/runner-config.tf b/modules/compute-providers/aws/ec2/runner-config.tf new file mode 100644 index 0000000000..f1d859581c --- /dev/null +++ b/modules/compute-providers/aws/ec2/runner-config.tf @@ -0,0 +1,13 @@ +resource "aws_ssm_parameter" "runner_config_run_as" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/run_as" + type = "String" + value = var.runner.run_as_root ? "root" : var.runner.run_as + tags = local.ssm_parameter_tags +} + +resource "aws_ssm_parameter" "runner_enable_cloudwatch" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/enable_cloudwatch" + type = "String" + value = var.config.cloudwatch_agent.enabled + tags = local.ssm_parameter_tags +} diff --git a/modules/compute-providers/aws/ec2/runner-instances.tf b/modules/compute-providers/aws/ec2/runner-instances.tf new file mode 100644 index 0000000000..8b88251e85 --- /dev/null +++ b/modules/compute-providers/aws/ec2/runner-instances.tf @@ -0,0 +1,325 @@ +# AMI selection, bootstrap rendering, launch template, and security group for +# EC2 runner instances. +locals { + provider_tags = merge( + { + "Name" = format("%s-action-runner", var.prefix) + }, + var.tags, + ) + + ssm_parameter_tags = merge( + local.provider_tags, + var.ssm.tags, + var.ssm.parameters.tags, + ) + + log_group_tags = merge( + local.provider_tags, + var.observability.logs.tags, + ) + + name_sg = var.config.overrides.name_sg == "" ? local.provider_tags["Name"] : var.config.overrides.name_sg + name_runner = var.config.overrides.name_runner == "" ? local.provider_tags["Name"] : var.config.overrides.name_runner + runner_tags = merge( + local.provider_tags, + { + "Name" = local.name_runner + }, + var.config.tags, + { + "ghr:environment" = var.prefix + "ghr:ssm_config_path" = "${var.ssm.paths.root}/${var.ssm.paths.config}" + "ghr:runner_name_prefix" = var.runner.name_prefix + }, + ) + + role_path = var.runner.iam.path == null ? "/${var.prefix}/" : var.runner.iam.path + instance_profile_path = var.config.instance_profile_path == null ? "/${var.prefix}/" : var.config.instance_profile_path + userdata_template = var.config.user_data.template == null ? local.default_userdata_template[var.runner.os] : var.config.user_data.template + s3_location_runner_distribution = var.config.binaries_syncer.enabled ? "s3://${try(var.config.binaries_syncer.s3.id, "")}/${try(var.config.binaries_syncer.s3.key, "")}" : "" + default_ami = { + "windows" = { name = ["Windows_Server-2022-English-Full-ECS_Optimized-*"] } + "linux" = var.runner.architecture == "arm64" ? { name = ["al2023-ami-2023.*-kernel-6.*-arm64"] } : { name = ["al2023-ami-2023.*-kernel-6.*-x86_64"] } + "osx" = var.runner.architecture == "arm64" ? { name = ["amzn-ec2-macos-15.*-arm64"] } : { name = ["amzn-ec2-macos-15.*"] } + } + + default_userdata_template = { + "windows" = "${path.module}/templates/user-data.ps1" + "linux" = "${path.module}/templates/user-data.sh" + "osx" = "${path.module}/templates/user-data-osx.sh" + } + + userdata_install_runner = { + "windows" = "${path.module}/templates/install-runner.ps1" + "linux" = "${path.module}/templates/install-runner.sh" + "osx" = "${path.module}/templates/install-runner-osx.sh" + } + + userdata_start_runner = { + "windows" = "${path.module}/templates/start-runner.ps1" + "linux" = "${path.module}/templates/start-runner.sh" + "osx" = "${path.module}/templates/start-runner-osx.sh" + } + + # Handle AMI configuration + ami_config = var.config.ami != null ? var.config.ami : { + filter = local.default_ami[var.runner.os] + owners = ["amazon"] + id_ssm_parameter = null + kms_key = null + } + ami_kms_key_enabled = local.ami_config.kms_key != null + ami_kms_key_arn = local.ami_kms_key_enabled ? local.ami_config.kms_key.arn : null + ami_filter = merge(local.default_ami[var.runner.os], local.ami_config.filter) + ami_id_ssm_external = local.ami_config.id_ssm_parameter != null + ami_id_ssm_module_managed = !local.ami_id_ssm_external + ami_id_ssm_parameter_arn = local.ami_id_ssm_external ? local.ami_config.id_ssm_parameter.arn : null + # Extract parameter name from ARN (format: arn:aws:ssm:region:account:parameter/path/to/param) + ami_id_ssm_parameter_name = local.ami_id_ssm_external ? try(regex("parameter(/.+)$", local.ami_id_ssm_parameter_arn)[0], null) : null + + user_data = var.config.user_data.enabled ? (var.config.user_data.content == null ? templatefile(local.userdata_template, { + enable_debug_logging = var.config.user_data.debug_logging_enabled + s3_location_runner_distribution = local.s3_location_runner_distribution + pre_install = var.config.user_data.pre_install + install_runner = templatefile(local.userdata_install_runner[var.runner.os], { + S3_LOCATION_RUNNER_DISTRIBUTION = local.s3_location_runner_distribution + RUNNER_ARCHITECTURE = var.runner.architecture + }) + post_install = var.config.user_data.post_install + hook_job_started = var.runner.hooks.job_started + hook_job_completed = var.runner.hooks.job_completed + start_runner = templatefile(local.userdata_start_runner[var.runner.os], { + metadata_tags = var.config.metadata_options != null ? var.config.metadata_options.instance_metadata_tags : "enabled" + }) + ghes_url = var.github.enterprise_server.url + ghes_ssl_verify = var.github.enterprise_server.ssl_verify + + ## retain these for backwards compatibility + environment = var.prefix + enable_cloudwatch_agent = var.config.cloudwatch_agent.enabled + ssm_key_cloudwatch_agent_config = var.config.cloudwatch_agent.enabled ? aws_ssm_parameter.cloudwatch_agent_config_runner[0].name : "" + }) : var.config.user_data.content) : "" + + encoded_user_data = ( + var.runner.os == "linux" ? base64gzip(local.user_data) : + var.runner.os == "windows" ? base64encode(local.user_data) : + var.runner.os == "osx" ? base64encode(local.user_data) : + null + ) +} + +data "aws_ami" "runner" { + most_recent = "true" + + dynamic "filter" { + for_each = local.ami_filter + content { + name = filter.key + values = filter.value + } + } + + owners = local.ami_config.owners +} + +resource "aws_ssm_parameter" "runner_ami_id" { + count = local.ami_id_ssm_module_managed ? 1 : 0 + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/ami_id" + type = "String" + data_type = "aws:ec2:image" + value = data.aws_ami.runner.id + + tags = merge( + local.provider_tags, + local.ssm_parameter_tags, + { + # Remove parentheses from AMI name to comply with AWS tag constraints + "ghr:ami_name" = replace(data.aws_ami.runner.name, "/[()]/", "") + }, + { + "ghr:ami_creation_date" = data.aws_ami.runner.creation_date + }, + { + "ghr:ami_deprecation_time" = data.aws_ami.runner.deprecation_time + } + ) +} + +resource "aws_launch_template" "runner" { + name = "${var.prefix}-action-runner" + + dynamic "block_device_mappings" { + for_each = var.config.block_device_mappings != null ? var.config.block_device_mappings : [] + content { + device_name = block_device_mappings.value.device_name + + ebs { + delete_on_termination = block_device_mappings.value.delete_on_termination + encrypted = block_device_mappings.value.encrypted + iops = block_device_mappings.value.iops + kms_key_id = block_device_mappings.value.kms_key_id + snapshot_id = block_device_mappings.value.snapshot_id + throughput = block_device_mappings.value.throughput + volume_initialization_rate = block_device_mappings.value.volume_initialization_rate + volume_size = block_device_mappings.value.volume_size + volume_type = block_device_mappings.value.volume_type + } + } + } + + dynamic "metadata_options" { + for_each = var.config.metadata_options != null ? [var.config.metadata_options] : [] + + content { + http_endpoint = metadata_options.value.http_endpoint + http_tokens = metadata_options.value.http_tokens + http_put_response_hop_limit = metadata_options.value.http_put_response_hop_limit + instance_metadata_tags = metadata_options.value.instance_metadata_tags + } + } + + dynamic "metadata_options" { + for_each = var.config.metadata_options != null ? [] : [0] + + content { + instance_metadata_tags = "enabled" + } + } + + dynamic "credit_specification" { + for_each = var.config.credit_specification != null ? [var.config.credit_specification] : [] + content { + cpu_credits = credit_specification.value + } + } + + dynamic "cpu_options" { + for_each = var.config.cpu_options != null ? [var.config.cpu_options] : [] + content { + core_count = try(cpu_options.value.core_count, null) + threads_per_core = try(cpu_options.value.threads_per_core, null) + amd_sev_snp = try(cpu_options.value.amd_sev_snp, null) + nested_virtualization = try(cpu_options.value.nested_virtualization, null) + } + } + + dynamic "placement" { + for_each = var.config.placement != null ? [var.config.placement] : [] + content { + affinity = try(placement.value.affinity, null) + availability_zone = try(placement.value.availability_zone, null) + group_id = try(placement.value.group_id, null) + group_name = try(placement.value.group_name, null) + host_id = try(placement.value.host_id, null) + host_resource_group_arn = try(placement.value.host_resource_group_arn, null) + spread_domain = try(placement.value.spread_domain, null) + tenancy = try(placement.value.tenancy, null) + partition_number = try(placement.value.partition_number, null) + } + } + + dynamic "license_specification" { + for_each = var.config.license_specifications + content { + license_configuration_arn = license_specification.value.license_configuration_arn + } + } + + monitoring { + enabled = var.config.detailed_monitoring_enabled + } + + iam_instance_profile { + name = var.config.instance_profile != null ? var.config.instance_profile.name : aws_iam_instance_profile.runner[0].name + } + + instance_initiated_shutdown_behavior = "terminate" + image_id = "resolve:ssm:${local.ami_id_ssm_module_managed ? aws_ssm_parameter.runner_ami_id[0].arn : local.ami_id_ssm_parameter_arn}" + key_name = var.config.key_name + ebs_optimized = var.config.ebs_optimized + + vpc_security_group_ids = !var.config.associate_public_ipv4_address ? compact(concat( + var.config.managed_security_group_enabled ? [aws_security_group.runner_sg[0].id] : [], + var.config.additional_security_group_ids, + )) : [] + + tag_specifications { + resource_type = "instance" + tags = local.runner_tags + } + + tag_specifications { + resource_type = "volume" + tags = local.runner_tags + } + + # We avoid including the "spot-instances-request" tag_specifications block when on_demand_failover_for_errors is defined, + # because when using on-demand fallback, the spot instance request resource is not created and thus the tags would not apply. + # Additionally, tagging spot requests via the CreateFleetCommand in the Lambda function does not work as expected, + # so we rely on Terraform to manage these tags only when spot is exclusively used without on-demand failover. + dynamic "tag_specifications" { + for_each = var.config.instance_target_capacity_type == "spot" && length(var.config.on_demand_failover_for_errors) == 0 ? [1] : [] # Include the block only if the value is "spot" and on_demand_failover_for_errors is not enabled + content { + resource_type = "spot-instances-request" + tags = local.runner_tags + } + } + + tag_specifications { + resource_type = "network-interface" + tags = local.runner_tags + } + + user_data = local.encoded_user_data + + tags = local.provider_tags + + update_default_version = true + + dynamic "network_interfaces" { + for_each = var.config.associate_public_ipv4_address ? [var.config.associate_public_ipv4_address] : [] + iterator = associate_public_ipv4_address + content { + associate_public_ip_address = associate_public_ipv4_address.value + security_groups = compact(concat( + var.config.managed_security_group_enabled ? [aws_security_group.runner_sg[0].id] : [], + var.config.additional_security_group_ids, + )) + } + } +} + +resource "aws_security_group" "runner_sg" { + count = var.config.managed_security_group_enabled ? 1 : 0 + name_prefix = "${var.prefix}-github-actions-runner-sg" + description = "Github Actions Runner security group" + + vpc_id = var.config.vpc_id + + ingress = [] + + dynamic "egress" { + for_each = var.config.egress_rules + iterator = each + + content { + cidr_blocks = each.value.cidr_blocks + ipv6_cidr_blocks = each.value.ipv6_cidr_blocks + prefix_list_ids = each.value.prefix_list_ids + from_port = each.value.from_port + protocol = each.value.protocol + security_groups = each.value.security_groups + self = each.value.self + to_port = each.value.to_port + description = each.value.description + } + } + + tags = merge( + local.provider_tags, + { + "Name" = format("%s", local.name_sg) + }, + ) +} diff --git a/modules/compute-providers/aws/ec2/templates/cloudwatch_config.json b/modules/compute-providers/aws/ec2/templates/cloudwatch_config.json new file mode 100644 index 0000000000..47b9bede8a --- /dev/null +++ b/modules/compute-providers/aws/ec2/templates/cloudwatch_config.json @@ -0,0 +1,12 @@ +{ + "agent": { + "metrics_collection_interval": 5 + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": ${logfiles} + } + } + } +} diff --git a/modules/compute-providers/aws/ec2/templates/install-runner-osx.sh b/modules/compute-providers/aws/ec2/templates/install-runner-osx.sh new file mode 100644 index 0000000000..ed848dad27 --- /dev/null +++ b/modules/compute-providers/aws/ec2/templates/install-runner-osx.sh @@ -0,0 +1,61 @@ +# shellcheck shell=bash + +set -euo pipefail + +## install the runner (macOS) + +s3_location=${S3_LOCATION_RUNNER_DISTRIBUTION} +architecture=${RUNNER_ARCHITECTURE} + +if [ -z "$RUNNER_TARBALL_URL" ] && [ -z "$s3_location" ]; then + echo "Neither RUNNER_TARBALL_URL or s3_location are set" + exit 1 +fi + +file_name="actions-runner.tar.gz" + +echo "Setting up GH Actions runner tool cache" +mkdir -p /Users/runner/hostedtoolcache + +echo "Creating actions-runner directory for the GH Action installation" +sudo mkdir -p /opt/actions-runner +cd /opt/actions-runner || exit 1 + +if [[ -n "$runner_tarball_url" ]]; then + echo "Downloading the GH Action runner from $runner_tarball_url to $file_name" + curl -s -o "$file_name" -L "$runner_tarball_url" +else + echo "Retrieving REGION from AWS API" + token="$(curl -s -f -X PUT "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 180")" + + region="$(curl -s -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)" + echo "Retrieved REGION from AWS API ($region)" + + echo "Downloading the GH Action runner from s3 bucket $s3_location" + aws s3 cp "$s3_location" "$file_name" --region "$region" --no-progress +fi + +echo "Un-tar action runner" +tar xzf "./$file_name" +echo "Delete tar file" +rm -rf "$file_name" + +os_name=$(sw_vers -productName 2>/dev/null || echo "macOS") +os_version=$(sw_vers -productVersion 2>/dev/null || echo "unknown") +arch_name=$(uname -m) + +echo "OS: $os_name $os_version ($arch_name)" + +if ! command -v brew >/dev/null 2>&1; then + echo "Homebrew not found; skipping dependency installation via brew" +else + echo "Homebrew detected; install any macOS-specific dependencies here if needed" + # Example: brew install jq awscli +fi + +echo "Set file ownership of action runner" +sudo chown -R "$user_name":staff /opt/actions-runner +sudo chmod 755 "/Users/runner" +sudo chown -R "$user_name":staff /Users/runner/hostedtoolcache diff --git a/modules/compute-providers/aws/ec2/templates/install-runner.ps1 b/modules/compute-providers/aws/ec2/templates/install-runner.ps1 new file mode 100644 index 0000000000..a13f91a65b --- /dev/null +++ b/modules/compute-providers/aws/ec2/templates/install-runner.ps1 @@ -0,0 +1,13 @@ +## install the runner + +Write-Host "Creating actions-runner directory for the GH Action installation" +New-Item -ItemType Directory -Path C:\actions-runner ; Set-Location C:\actions-runner + +Write-Host "Downloading the GH Action runner from s3 bucket $s3_location" +aws s3 cp ${S3_LOCATION_RUNNER_DISTRIBUTION} actions-runner.zip + +Write-Host "Un-zip action runner" +Expand-Archive -Path actions-runner.zip -DestinationPath . + +Write-Host "Delete zip file" +Remove-Item actions-runner.zip diff --git a/modules/compute-providers/aws/ec2/templates/install-runner.sh b/modules/compute-providers/aws/ec2/templates/install-runner.sh new file mode 100644 index 0000000000..5ed5897e7c --- /dev/null +++ b/modules/compute-providers/aws/ec2/templates/install-runner.sh @@ -0,0 +1,73 @@ +# shellcheck shell=bash + +## install the runner + +s3_location=${S3_LOCATION_RUNNER_DISTRIBUTION} + +if [ -z "$RUNNER_TARBALL_URL" ] && [ -z "$s3_location" ]; then + echo "Neither RUNNER_TARBALL_URL or s3_location are set" + exit 1 +fi + +file_name="actions-runner.tar.gz" + +echo "Setting up GH Actions runner tool cache" +# Required for various */setup-* actions to work, location is also know by various environment +# variable names in the actions/runner software : RUNNER_TOOL_CACHE / RUNNER_TOOLSDIRECTORY / AGENT_TOOLSDIRECTORY +# Warning, not all setup actions support the env vars and so this specific path must be created regardless +mkdir -p /opt/hostedtoolcache + +echo "Creating actions-runner directory for the GH Action installation" +cd /opt/ +mkdir -p actions-runner && cd actions-runner + + +if [[ -n "$RUNNER_TARBALL_URL" ]]; then + echo "Downloading the GH Action runner from $RUNNER_TARBALL_URL to $file_name" + curl -s -o $file_name -L "$RUNNER_TARBALL_URL" +else + echo "Retrieving TOKEN from AWS API" + token="$(curl -s -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180")" + + region="$(curl -s -f -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)" + echo "Retrieved REGION from AWS API ($region)" + + echo "Downloading the GH Action runner from s3 bucket $s3_location" + aws s3 cp "$s3_location" "$file_name" --region "$region" --no-progress +fi + +echo "Un-tar action runner" +tar xzf ./$file_name +echo "Delete tar file" +rm -rf $file_name + +os_id=$(awk -F= '/^ID=/{print $2}' /etc/os-release) +echo OS: $os_id + +# Install libicu on non-ubuntu, non-debian +if [[ ! "$os_id" =~ ^(ubuntu|debian).* ]]; then + max_attempts=5 + attempt_count=0 + success=false + while [ $success = false ] && [ $attempt_count -le $max_attempts ]; do + echo "Attempt $attempt_count/$max_attempts: Installing libicu" + dnf install -y libicu + if [ $? -eq 0 ]; then + success=true + else + echo "Failed to install libicu" + attempt_count=$(( attempt_count + 1 )) + sleep 5 + fi + done +fi + +# Install dependencies for ubuntu and debian +if [[ "$os_id" =~ ^(ubuntu|debian).* ]]; then + echo "Installing dependencies" + ./bin/installdependencies.sh +fi + +echo "Set file ownership of action runner" +chown -R "$user_name":"$user_name" /opt/actions-runner +chown -R "$user_name":"$user_name" /opt/hostedtoolcache diff --git a/modules/compute-providers/aws/ec2/templates/start-runner-osx.sh b/modules/compute-providers/aws/ec2/templates/start-runner-osx.sh new file mode 100644 index 0000000000..a6da66116d --- /dev/null +++ b/modules/compute-providers/aws/ec2/templates/start-runner-osx.sh @@ -0,0 +1,185 @@ +#!/bin/bash + +# macOS variant of start-runner.sh + +tag_instance_with_runner_id() { + echo "Checking for .runner file to extract agent ID" + + if [[ ! -f "/opt/actions-runner/.runner" ]]; then + echo "Warning: .runner file not found" + return 0 + fi + + echo "Found .runner file, extracting agent ID" + local agent_id + agent_id=$(jq -r '.agentId' /opt/actions-runner/.runner 2>/dev/null || echo "") + + if [[ -z "$agent_id" || "$agent_id" == "null" ]]; then + echo "Warning: Could not extract agent ID from .runner file" + return 0 + fi + + echo "Tagging instance with GitHub runner agent ID: $agent_id" + if aws ec2 create-tags \ + --region "$region" \ + --resources "$instance_id" \ + --tags Key=ghr:github_runner_id,Value="$agent_id"; then + echo "Successfully tagged instance with agent ID: $agent_id" + return 0 + else + echo "Warning: Failed to tag instance with agent ID" + return 0 + fi +} + +cleanup() { + local exit_code="$1" + + if [ "$exit_code" -ne 0 ]; then + echo "ERROR: runner-start-failed with exit code $exit_code" + fi + + if [ "$agent_mode" = "ephemeral" ] || [ "$exit_code" -ne 0 ]; then + echo "Terminating instance" + aws ec2 terminate-instances \ + --instance-ids "$instance_id" \ + --region "$region" || true + fi +} + +trap 'cleanup $?' EXIT + +echo "Retrieving TOKEN from AWS API" +token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 180" || true) +if [ -z "$token" ]; then + retrycount=0 + until [ -n "$token" ]; do + echo "Failed to retrieve token. Retrying in 5 seconds." + sleep 5 + token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 180" || true) + retrycount=$((retrycount + 1)) + if [ $retrycount -gt 40 ]; then + break + fi + done +fi + +region=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region) +echo "Retrieved REGION from AWS API ($region)" + +instance_id=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/meta-data/instance-id) +echo "Retrieved INSTANCE_ID from AWS API ($instance_id)" + +availability_zone=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/meta-data/placement/availability-zone) + +environment=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/meta-data/tags/instance/ghr:environment || echo "") +ssm_config_path=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/meta-data/tags/instance/ghr:ssm_config_path || echo "") +runner_name_prefix=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/meta-data/tags/instance/ghr:runner_name_prefix || echo "") + +echo "Retrieved ghr:environment tag - ($environment)" +echo "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" +echo "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" + +parameters=$(aws ssm get-parameters-by-path \ + --path "$ssm_config_path" \ + --region "$region" \ + --query "Parameters[*].{Name:Name,Value:Value}") +echo "Retrieved parameters from AWS SSM ($parameters)" + +run_as=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/run_as") | .Value') +echo "Retrieved /$ssm_config_path/run_as parameter - ($run_as)" + +agent_mode=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/agent_mode") | .Value') +echo "Retrieved /$ssm_config_path/agent_mode parameter - ($agent_mode)" + +disable_default_labels=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/disable_default_labels") | .Value') +echo "Retrieved /$ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" + +enable_jit_config=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/enable_jit_config") | .Value') +echo "Retrieved /$ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" + +token_path=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/token_path") | .Value') +echo "Retrieved /$ssm_config_path/token_path parameter - ($token_path)" + +echo "Get GH Runner config from AWS SSM" +config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") +while [[ -z "$config" ]]; do + echo "Waiting for GH Runner config to become available in AWS SSM" + sleep 1 + config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") +done + +echo "Delete GH Runner token from AWS SSM" +aws ssm delete-parameter --name "$token_path"/"$instance_id" --region "$region" + +if [ -z "$run_as" ]; then + echo "No user specified, using default ec2-user account" + run_as="ec2-user" +fi + +if [[ "$run_as" == "root" ]]; then + echo "run_as is set to root - export RUNNER_ALLOW_RUNASROOT=1" + export RUNNER_ALLOW_RUNASROOT=1 +fi + +sudo chown -R "$run_as" /opt/actions-runner + +info_arch=$(uname -m) +info_os=$(sw_vers -productName 2>/dev/null || echo "macOS") +info_ver=$(sw_vers -productVersion 2>/dev/null || echo "unknown") + +tee /opt/actions-runner/.setup_info <&1 + + if ($LASTEXITCODE -eq 0) { + Write-Host "Successfully tagged instance with agent ID: $agentId" + return $true + } else { + Write-Host "Warning: Failed to tag instance with agent ID - $tagResult" + return $true + } + } + catch { + Write-Host "Warning: Error processing .runner file - $($_.Exception.Message)" + return $true + } +} + +## Retrieve instance metadata + +Write-Host "Retrieving TOKEN from AWS API" +$token=Invoke-RestMethod -Method PUT -Uri "http://169.254.169.254/latest/api/token" -Headers @{"X-aws-ec2-metadata-token-ttl-seconds" = "180"} +if ( ! $token ) { + $retrycount=0 + do { + echo "Failed to retrieve token. Retrying in 5 seconds." + Start-Sleep 5 + $token=Invoke-RestMethod -Method PUT -Uri "http://169.254.169.254/latest/api/token" -Headers @{"X-aws-ec2-metadata-token-ttl-seconds" = "180"} + $retrycount=$retrycount + 1 + if ( $retrycount -gt 40 ) + { + break + } + } until ($token) +} + +$ami_id=Invoke-RestMethod -Uri "http://169.254.169.254/latest/meta-data/ami-id" -Headers @{"X-aws-ec2-metadata-token" = $token} + +$metadata=Invoke-RestMethod -Uri "http://169.254.169.254/latest/dynamic/instance-identity/document" -Headers @{"X-aws-ec2-metadata-token" = $token} + +$Region = $metadata.region +Write-Host "Retrieved REGION from AWS API ($Region)" + +$InstanceId = $metadata.instanceId +Write-Host "Retrieved InstanceId from AWS API ($InstanceId)" + +$tags=aws ec2 describe-tags --region "$Region" --filters "Name=resource-id,Values=$InstanceId" | ConvertFrom-Json +Write-Host "Retrieved tags from AWS API" + +$environment=$tags.Tags.where( {$_.Key -eq 'ghr:environment'}).value +Write-Host "Retrieved ghr:environment tag - ($environment)" + +$runner_name_prefix=$tags.Tags.where( {$_.Key -eq 'ghr:runner_name_prefix'}).value +Write-Host "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" + +$ssm_config_path=$tags.Tags.where( {$_.Key -eq 'ghr:ssm_config_path'}).value +Write-Host "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" + +$parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$Region" --query "Parameters[*].{Name:Name,Value:Value}") | ConvertFrom-Json +Write-Host "Retrieved parameters from AWS SSM" + +$run_as=$parameters.where( {$_.Name -eq "$ssm_config_path/run_as"}).value +Write-Host "Retrieved $ssm_config_path/run_as parameter - ($run_as)" + +$enable_cloudwatch_agent=$parameters.where( {$_.Name -eq "$ssm_config_path/enable_cloudwatch"}).value +Write-Host "Retrieved $ssm_config_path/enable_cloudwatch parameter - ($enable_cloudwatch_agent)" + +$agent_mode=$parameters.where( {$_.Name -eq "$ssm_config_path/agent_mode"}).value +Write-Host "Retrieved $ssm_config_path/agent_mode parameter - ($agent_mode)" + +$disable_default_labels=$parameters.where( {$_.Name -eq "$ssm_config_path/disable_default_labels"}).value +Write-Host "Retrieved $ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" + +$enable_jit_config=$parameters.where( {$_.Name -eq "$ssm_config_path/enable_jit_config"}).value +Write-Host "Retrieved $ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" + +$token_path=$parameters.where( {$_.Name -eq "$ssm_config_path/token_path"}).value +Write-Host "Retrieved $ssm_config_path/token_path parameter - ($token_path)" + + +if ($enable_cloudwatch_agent -eq "true") +{ + Write-Host "Enabling CloudWatch Agent" + & 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a fetch-config -m ec2 -s -c "ssm:$ssm_config_path/cloudwatch_agent_config_runner" +} + +## Configure the runner + +Write-Host "Get GH Runner config from AWS SSM" +$config = $null +$i = 0 +do { + $config = (aws ssm get-parameters --names "$token_path/$InstanceId" --with-decryption --region $Region --query "Parameters[*].{Name:Name,Value:Value}" | ConvertFrom-Json)[0].value + Write-Host "Waiting for GH Runner config to become available in AWS SSM ($i/30)" + Start-Sleep 1 + $i++ +} while (($null -eq $config) -and ($i -lt 30)) + +Write-Host "Delete GH Runner token from AWS SSM" +aws ssm delete-parameter --name "$token_path/$InstanceId" --region $Region + +# Create or update user +if (-not($run_as)) { + Write-Host "No user specified, using default ec2-user account" + $run_as="ec2-user" +} +Add-Type -AssemblyName "System.Web" +$password = [System.Web.Security.Membership]::GeneratePassword(24, 4) +$securePassword = ConvertTo-SecureString $password -AsPlainText -Force +$username = $run_as +if (!(Get-LocalUser -Name $username -ErrorAction Ignore)) { + New-LocalUser -Name $username -Password $securePassword + Write-Host "Created new user ($username)" +} +else { + Set-LocalUser -Name $username -Password $securePassword + Write-Host "Changed password for user ($username)" +} +# Add user to groups +foreach ($group in @("Administrators", "docker-users")) { + if ((Get-LocalGroup -Name "$group" -ErrorAction Ignore) -and + !(Get-LocalGroupMember -Group "$group" -Member $username -ErrorAction Ignore)) { + Add-LocalGroupMember -Group "$group" -Member $username + Write-Host "Added $username to $group group" + } +} + +# Disable User Access Control (UAC) +# TODO investigate if this is needed or if its overkill - https://github.com/github-aws-runners/terraform-aws-github-runner/issues/1505 +Set-ItemProperty HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System -Name ConsentPromptBehaviorAdmin -Value 0 -Force +Write-Host "Disabled User Access Control (UAC)" + +$runnerExtraOptions = "" +if ($disable_default_labels -eq "true") { + $runnerExtraOptions += "--no-default-labels" +} + +if ($enable_jit_config -eq "false" -or $agent_mode -ne "ephemeral") { + $configCmd = ".\config.cmd --unattended --name $runner_name_prefix$InstanceId --work `"_work`" $runnerExtraOptions $config" + Write-Host "Configure GH Runner (non ephmeral / no JIT) as user $run_as" + Invoke-Expression $configCmd + + # Tag instance with GitHub runner agent ID for non-JIT runners + Tag-InstanceWithRunnerId +} + +$jsonBody = @( + @{ + group='Runner Image' + detail="AMI id: $ami_id" + } +) +ConvertTo-Json -InputObject $jsonBody | Set-Content -Path "$pwd\.setup_info" + + +Write-Host "Starting the runner in $agent_mode mode" +Write-Host "Starting runner after $(((get-date) - (gcim Win32_OperatingSystem).LastBootUpTime).tostring("hh':'mm':'ss''"))" + +if ($agent_mode -eq "ephemeral") { + if ($enable_jit_config -eq "true") { + Write-Host "Starting with jit config" + Invoke-Expression ".\run.cmd --jitconfig $${config}" + } + else { + Write-Host "Starting without jit config" + Invoke-Expression ".\run.cmd" + } + Write-Host "Runner has finished" + + if ($enable_cloudwatch_agent) + { + Write-Host "Stopping CloudWatch Agent" + & 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a stop + } + + Write-Host "Terminating instance" + aws ec2 terminate-instances --instance-ids "$InstanceId" --region "$Region" +} else { + Write-Host "Installing the runner as a service" + + $action = New-ScheduledTaskAction -WorkingDirectory "$pwd" -Execute "run.cmd" + $trigger = Get-CimClass "MSFT_TaskRegistrationTrigger" -Namespace "Root/Microsoft/Windows/TaskScheduler" + Register-ScheduledTask -TaskName "runnertask" -Action $action -Trigger $trigger -User $username -Password $password -RunLevel Highest -Force + Write-Host "Starting runner after $(((get-date) - (gcim Win32_OperatingSystem).LastBootUpTime).tostring("hh':'mm':'ss''"))" +} diff --git a/modules/compute-providers/aws/ec2/templates/start-runner.sh b/modules/compute-providers/aws/ec2/templates/start-runner.sh new file mode 100644 index 0000000000..7f2c0f82c5 --- /dev/null +++ b/modules/compute-providers/aws/ec2/templates/start-runner.sh @@ -0,0 +1,280 @@ +#!/bin/bash + +# https://docs.aws.amazon.com/xray/latest/devguide/xray-api-sendingdata.html +# https://docs.aws.amazon.com/xray/latest/devguide/scorekeep-scripts.html +create_xray_start_segment() { + START_TIME=$(date -d "$(uptime -s)" +%s) + TRACE_ID=$1 + INSTANCE_ID=$2 + SEGMENT_ID=$(dd if=/dev/random bs=8 count=1 2>/dev/null | od -An -tx1 | tr -d ' \t\n') + SEGMENT_DOC="{\"trace_id\": \"$TRACE_ID\", \"id\": \"$SEGMENT_ID\", \"start_time\": $START_TIME, \"in_progress\": true, \"name\": \"Runner\",\"origin\": \"AWS::EC2::Instance\", \"aws\": {\"ec2\":{\"instance_id\":\"$INSTANCE_ID\"}}}" + HEADER='{"format": "json", "version": 1}' + TRACE_DATA="$HEADER\n$SEGMENT_DOC" + echo "$HEADER" > document.txt + echo "$SEGMENT_DOC" >> document.txt + UDP_IP="127.0.0.1" + UDP_PORT=2000 + cat document.txt > /dev/udp/$UDP_IP/$UDP_PORT + echo "$SEGMENT_DOC" +} + +create_xray_success_segment() { + local SEGMENT_DOC=$1 + if [ -z "$SEGMENT_DOC" ]; then + echo "No segment doc provided" + return + fi + SEGMENT_DOC=$(echo "$SEGMENT_DOC" | jq '. | del(.in_progress)') + END_TIME=$(date +%s) + SEGMENT_DOC=$(echo "$SEGMENT_DOC" | jq -c ". + {\"end_time\": $END_TIME}") + HEADER="{\"format\": \"json\", \"version\": 1}" + TRACE_DATA="$HEADER\n$SEGMENT_DOC" + echo "$HEADER" > document.txt + echo "$SEGMENT_DOC" >> document.txt + UDP_IP="127.0.0.1" + UDP_PORT=2000 + cat document.txt > /dev/udp/$UDP_IP/$UDP_PORT + echo "$SEGMENT_DOC" +} + +create_xray_error_segment() { + local SEGMENT_DOC="$1" + if [ -z "$SEGMENT_DOC" ]; then + echo "No segment doc provided" + return + fi + MESSAGE="$2" + ERROR="{\"exceptions\": [{\"message\": \"$MESSAGE\"}]}" + SEGMENT_DOC=$(echo "$SEGMENT_DOC" | jq '. | del(.in_progress)') + END_TIME=$(date +%s) + SEGMENT_DOC=$(echo "$SEGMENT_DOC" | jq -c ". + {\"end_time\": $END_TIME, \"error\": true, \"cause\": $ERROR }") + HEADER="{\"format\": \"json\", \"version\": 1}" + TRACE_DATA="$HEADER\n$SEGMENT_DOC" + echo "$HEADER" > document.txt + echo "$SEGMENT_DOC" >> document.txt + UDP_IP="127.0.0.1" + UDP_PORT=2000 + cat document.txt > /dev/udp/$UDP_IP/$UDP_PORT + echo "$SEGMENT_DOC" +} + +tag_instance_with_runner_id() { + echo "Checking for .runner file to extract agent ID" + + if [[ ! -f "/opt/actions-runner/.runner" ]]; then + echo "Warning: .runner file not found" + return 0 + fi + + echo "Found .runner file, extracting agent ID" + local agent_id + agent_id=$(jq -r '.agentId' /opt/actions-runner/.runner 2>/dev/null || echo "") + + if [[ -z "$agent_id" || "$agent_id" == "null" ]]; then + echo "Warning: Could not extract agent ID from .runner file" + return 0 + fi + + echo "Tagging instance with GitHub runner agent ID: $agent_id" + if aws ec2 create-tags \ + --region "$region" \ + --resources "$instance_id" \ + --tags Key=ghr:github_runner_id,Value="$agent_id"; then + echo "Successfully tagged instance with agent ID: $agent_id" + return 0 + else + echo "Warning: Failed to tag instance with agent ID" + return 0 + fi +} + +cleanup() { + local exit_code="$1" + local error_location="$2" + local error_lineno="$3" + + if [ "$exit_code" -ne 0 ]; then + echo "ERROR: runner-start-failed with exit code $exit_code occurred on $error_location" + create_xray_error_segment "$SEGMENT" "runner-start-failed with exit code $exit_code occurred on $error_location - $error_lineno" + fi + # allows to flush the cloud watch logs and traces + sleep 10 + if [ "$agent_mode" = "ephemeral" ] || [ "$exit_code" -ne 0 ]; then + echo "Stopping CloudWatch service" + systemctl stop amazon-cloudwatch-agent.service || true + echo "Terminating instance" + aws ec2 terminate-instances \ + --instance-ids "$instance_id" \ + --region "$region" \ + || true + fi +} + +trap 'cleanup $? $LINENO $BASH_LINENO' EXIT + +echo "Retrieving TOKEN from AWS API" +token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180" || true) +if [ -z "$token" ]; then + retrycount=0 + until [ -n "$token" ]; do + echo "Failed to retrieve token. Retrying in 5 seconds." + sleep 5 + token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180" || true) + retrycount=$((retrycount + 1)) + if [ $retrycount -gt 40 ]; then + break + fi + done +fi + +ami_id=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/ami-id) + +region=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region) +echo "Retrieved REGION from AWS API ($region)" + +instance_id=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/instance-id) +echo "Retrieved INSTANCE_ID from AWS API ($instance_id)" + +instance_type=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/instance-type) +availability_zone=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/placement/availability-zone) + +%{ if metadata_tags == "enabled" } +environment=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:environment) +ssm_config_path=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:ssm_config_path) +runner_name_prefix=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:runner_name_prefix || echo "") +xray_trace_id=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:trace_id || echo "") + +%{ else } +tags=$(aws ec2 describe-tags --region "$region" --filters "Name=resource-id,Values=$instance_id") +echo "Retrieved tags from AWS API ($tags)" + +environment=$(echo "$tags" | jq -r '.Tags[] | select(.Key == "ghr:environment") | .Value') +ssm_config_path=$(echo "$tags" | jq -r '.Tags[] | select(.Key == "ghr:ssm_config_path") | .Value') +runner_name_prefix=$(echo "$tags" | jq -r '.Tags[] | select(.Key == "ghr:runner_name_prefix") | .Value' || echo "") +xray_trace_id=$(echo "$tags" | jq -r '.Tags[] | select(.Key == "ghr:trace_id") | .Value' || echo "") + +%{ endif } + +echo "Retrieved ghr:environment tag - ($environment)" +echo "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" +echo "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" + +parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$region" --query "Parameters[*].{Name:Name,Value:Value}") +echo "Retrieved parameters from AWS SSM ($parameters)" + +run_as=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/run_as") | .Value') +echo "Retrieved /$ssm_config_path/run_as parameter - ($run_as)" + +enable_cloudwatch_agent=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/enable_cloudwatch") | .Value') +echo "Retrieved /$ssm_config_path/enable_cloudwatch parameter - ($enable_cloudwatch_agent)" + +agent_mode=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/agent_mode") | .Value') +echo "Retrieved /$ssm_config_path/agent_mode parameter - ($agent_mode)" + +disable_default_labels=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/disable_default_labels") | .Value') +echo "Retrieved /$ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" + +enable_jit_config=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/enable_jit_config") | .Value') +echo "Retrieved /$ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" + +token_path=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/token_path") | .Value') +echo "Retrieved /$ssm_config_path/token_path parameter - ($token_path)" + +if [[ "$xray_trace_id" != "" ]]; then + # run xray service + curl https://s3.us-east-2.amazonaws.com/aws-xray-assets.us-east-2/xray-daemon/aws-xray-daemon-linux-3.x.zip -o aws-xray-daemon-linux-3.x.zip + unzip aws-xray-daemon-linux-3.x.zip -d aws-xray-daemon-linux-3.x + chmod +x ./aws-xray-daemon-linux-3.x/xray + ./aws-xray-daemon-linux-3.x/xray -o -n "$region" & + + + SEGMENT=$(create_xray_start_segment "$xray_trace_id" "$instance_id") + echo "$SEGMENT" +fi + +if [[ "$enable_cloudwatch_agent" == "true" ]]; then + echo "Cloudwatch is enabled" + amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:$ssm_config_path/cloudwatch_agent_config_runner" +fi + +## Configure the runner + +echo "Get GH Runner config from AWS SSM" +config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") +while [[ -z "$config" ]]; do + echo "Waiting for GH Runner config to become available in AWS SSM" + sleep 1 + config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") +done + +echo "Delete GH Runner token from AWS SSM" +aws ssm delete-parameter --name "$token_path"/"$instance_id" --region "$region" + +if [ -z "$run_as" ]; then + echo "No user specified, using default ec2-user account" + run_as="ec2-user" +fi + +if [[ "$run_as" == "root" ]]; then + echo "run_as is set to root - export RUNNER_ALLOW_RUNASROOT=1" + export RUNNER_ALLOW_RUNASROOT=1 +fi + +chown -R $run_as /opt/actions-runner + +info_arch=$(uname -p) +info_os=$( ( lsb_release -ds || cat /etc/*release || uname -om ) 2>/dev/null | head -n1 | cut -d "=" -f2- | tr -d '"') + +tee /opt/actions-runner/.setup_info </dev/null 2>&1; then + echo "Homebrew detected; you can install extra dependencies via brew if needed" +fi + +user_name=ec2-user + +${install_runner} + +${post_install} + +# Register runner job hooks +# Ref: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/running-scripts-before-or-after-a-job +%{ if hook_job_started != "" } +cat > /opt/actions-runner/hook_job_started.sh <<'EOF' +${hook_job_started} +EOF +echo ACTIONS_RUNNER_HOOK_JOB_STARTED=/opt/actions-runner/hook_job_started.sh | tee -a /opt/actions-runner/.env +%{ endif } + +%{ if hook_job_completed != "" } +cat > /opt/actions-runner/hook_job_completed.sh <<'EOF' +${hook_job_completed} +EOF +echo ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/opt/actions-runner/hook_job_completed.sh | tee -a /opt/actions-runner/.env +%{ endif } + +${start_runner} diff --git a/modules/compute-providers/aws/ec2/templates/user-data.ps1 b/modules/compute-providers/aws/ec2/templates/user-data.ps1 new file mode 100644 index 0000000000..a1e3a4da66 --- /dev/null +++ b/modules/compute-providers/aws/ec2/templates/user-data.ps1 @@ -0,0 +1,47 @@ + +$ErrorActionPreference = "Continue" +$VerbosePreference = "Continue" +Start-Transcript -Path "C:\UserData.log" -Append + +${pre_install} + +# Install Chocolatey +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12 +$env:chocolateyUseWindowsCompression = 'true' +Invoke-WebRequest https://chocolatey.org/install.ps1 -UseBasicParsing | Invoke-Expression + +# Add Chocolatey to powershell profile +$ChocoProfileValue = @' +$ChocolateyProfile = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" +if (Test-Path($ChocolateyProfile)) { + Import-Module "$ChocolateyProfile" +} + +refreshenv +'@ +# Write it to the $profile location +Set-Content -Path "$PsHome\Microsoft.PowerShell_profile.ps1" -Value $ChocoProfileValue -Force +# Source it +. "$PsHome\Microsoft.PowerShell_profile.ps1" + + +refreshenv + +Write-Host "Installing cloudwatch agent..." +Invoke-WebRequest -Uri https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi -OutFile C:\amazon-cloudwatch-agent.msi +$cloudwatchParams = '/i', 'C:\amazon-cloudwatch-agent.msi', '/qn', '/L*v', 'C:\CloudwatchInstall.log' +Start-Process "msiexec.exe" $cloudwatchParams -Wait -NoNewWindow +Remove-Item C:\amazon-cloudwatch-agent.msi + + +# Install dependent tools +Write-Host "Installing additional development tools" +choco install git awscli -y +refreshenv + +${install_runner} +${post_install} +${start_runner} + +Stop-Transcript + diff --git a/modules/compute-providers/aws/ec2/templates/user-data.sh b/modules/compute-providers/aws/ec2/templates/user-data.sh new file mode 100644 index 0000000000..ca69f26d34 --- /dev/null +++ b/modules/compute-providers/aws/ec2/templates/user-data.sh @@ -0,0 +1,81 @@ +#!/bin/bash -e + +install_with_retry() { + max_attempts=5 + attempt_count=0 + success=false + while [ $success = false ] && [ $attempt_count -le $max_attempts ]; do + echo "Attempting $attempt_count/$max_attempts: Installing $*" + dnf install -y $* + if [ $? -eq 0 ]; then + success=true + else + echo "Failed to install $1 - retrying" + attempt_count=$(( attempt_count + 1 )) + sleep 5 + fi + done +} + +exec > >(tee /var/log/user-data.log | logger -t user-data -s 2>/dev/console) 2>&1 + +# AWS suggest to create a log for debug purpose based on https://aws.amazon.com/premiumsupport/knowledge-center/ec2-linux-log-user-data/ +# As side effect all command, set +x disable debugging explicitly. +# +# An alternative for masking tokens could be: exec > >(sed 's/--token\ [^ ]* /--token\ *** /g' > /var/log/user-data.log) 2>&1 + +set +x + +%{ if enable_debug_logging } +set -x +%{ endif } + +${pre_install} + +max_attempts=5 +attempt_count=0 +success=false +while [ $success = false ] && [ $attempt_count -le $max_attempts ]; do + echo "Attempting $attempt_count/$max_attempts: upgrade-minimal" + dnf upgrade-minimal -y +if [ $? -eq 0 ]; then + success=true + else + echo "Failed to run `dnf upgrad-minimal -y` - retrying" + attempt_count=$(( attempt_count + 1 )) + sleep 5 + fi +done + +# Install docker +install_with_retry docker + +service docker start +usermod -a -G docker ec2-user + +install_with_retry amazon-cloudwatch-agent jq git +install_with_retry --allowerasing curl + +user_name=ec2-user + +${install_runner} + +${post_install} + +# Register runner job hooks +# Ref: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/running-scripts-before-or-after-a-job +%{ if hook_job_started != "" } +cat > /opt/actions-runner/hook_job_started.sh <<'EOF' +${hook_job_started} +EOF +echo ACTIONS_RUNNER_HOOK_JOB_STARTED=/opt/actions-runner/hook_job_started.sh | tee -a /opt/actions-runner/.env +%{ endif } + +%{ if hook_job_completed != "" } +cat > /opt/actions-runner/hook_job_completed.sh <<'EOF' +${hook_job_completed} +EOF +echo ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/opt/actions-runner/hook_job_completed.sh | tee -a /opt/actions-runner/.env +%{ endif } + +${start_runner} diff --git a/modules/compute-providers/aws/ec2/tests/provider.tftest.hcl b/modules/compute-providers/aws/ec2/tests/provider.tftest.hcl new file mode 100644 index 0000000000..bc92537279 --- /dev/null +++ b/modules/compute-providers/aws/ec2/tests/provider.tftest.hcl @@ -0,0 +1,451 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{}" + } + } + + mock_data "aws_ami" { + defaults = { + id = "ami-1234567890abcdef0" + name = "runner-test" + creation_date = "2026-01-01T00:00:00.000Z" + deprecation_time = "" + } + } + + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } +} + +override_data { + target = data.aws_iam_policy_document.scale_up + values = { + json = "{\"Action\":\"ec2:RunInstances\",\"PassRole\":\"arn:aws:iam::123456789012:role/provider-test-runner\"}" + } +} + +override_data { + target = data.aws_iam_policy_document.pool + values = { + json = "{\"Action\":\"iam:PassRole\"}" + } +} + +variables { + aws_region = "eu-west-1" + prefix = "provider-test" + + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/ami-id" + } + kms_key = null + } + binaries_syncer = { + enabled = true + s3 = { + arn = "arn:aws:s3:::runner-distribution" + id = "runner-distribution" + key = "runner.zip" + } + } + cloudwatch_agent = { + enabled = true + } + ssm_enabled = true + managed_security_group_enabled = true + } + + runner = { + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/provider-test-runner" + name = "provider-test-runner" + } + managed_policy_arns = { + readonly = "arn:aws:iam::aws:policy/ReadOnlyAccess" + } + } + } + + ssm = { + paths = { + root = "/github-runner/provider-test" + tokens = "tokens" + config = "config" + } + } +} + +run "separates_control_plane_contract_from_ec2_resources" { + command = plan + + assert { + condition = toset(keys(output.provider)) == toset(["environment_variables", "policies", "resources"]) + error_message = "The EC2 provider contract must expose only integration and resource data; its module identity must not be repeated in the output." + } + + assert { + condition = output.provider.environment_variables.scale_up["INSTANCE_TYPES"] == "m5.large" + error_message = "The provider contract must expose EC2 scale-up environment variables." + } + + assert { + condition = ( + length(output.provider.environment_variables.scale_down) == 0 + && !contains(keys(output.provider.environment_variables.pool), "RUNNER_BOOT_TIME_IN_MINUTES") + ) + error_message = "The EC2 provider must not expose webhook-owned runner boot-time configuration." + } + + assert { + condition = strcontains(output.provider.policies.scale_up.iam_policy_json, "ec2:RunInstances") + error_message = "The EC2 provider must own EC2 scale-up permissions." + } + + assert { + condition = !strcontains(output.provider.policies.scale_up.iam_policy_json, "sqs:ReceiveMessage") + error_message = "The EC2 provider must not own common build-queue permissions." + } + + assert { + condition = strcontains(output.provider.policies.pool.iam_policy_json, "iam:PassRole") + error_message = "The EC2 provider must expose pool permissions for its runner role." + } + + assert { + condition = strcontains(output.provider.policies.scale_up.iam_policy_json, "arn:aws:iam::123456789012:role/provider-test-runner") + error_message = "The EC2 provider must use the common runner role ARN for PassRole." + } + + assert { + condition = output.provider.policies.scale_up.managed_policy_enabled + error_message = "An external AMI SSM parameter must enable the scale-up managed policy attachment at plan time." + } + + assert { + condition = output.provider.policies.pool.managed_policy_enabled + error_message = "An external AMI SSM parameter must enable the pool managed policy attachment at plan time." + } + + assert { + condition = ( + contains(flatten([ + for statement in data.aws_iam_policy_document.scale_up.statement : [ + for condition in statement.condition : condition.variable + ] + ]), "ec2:ResourceTag/ghr:environment") + && contains(flatten([ + for statement in data.aws_iam_policy_document.scale_down.statement : [ + for condition in statement.condition : condition.variable + ] + ]), "ec2:ResourceTag/ghr:environment") + && !contains(flatten([ + for statement in data.aws_iam_policy_document.scale_up.statement : [ + for condition in statement.condition : condition.variable + ] + ]), "ec2:ResourceTag/gh:environment") + && !contains(flatten([ + for statement in data.aws_iam_policy_document.scale_down.statement : [ + for condition in statement.condition : condition.variable + ] + ]), "ec2:ResourceTag/gh:environment") + ) + error_message = "EC2 scale policies must authorize resources by the protected ghr:environment tag." + } + + assert { + condition = toset(keys(output.provider.policies)) == toset(["runner", "scale_up", "scale_down", "pool"]) + error_message = "The EC2 provider must expose policies grouped by their owning common component." + } + + assert { + condition = toset(keys(output.provider.policies.runner.inline_policies)) == toset([ + "ssm_parameters", + "describe_tags", + "create_tags", + "terminate_self", + "session_manager", + "distribution_bucket", + "cloudwatch", + ]) + error_message = "The EC2 provider must return the enabled runner permission documents." + } + + assert { + condition = output.provider.policies.runner.managed_policy_arns["readonly"] == "arn:aws:iam::aws:policy/ReadOnlyAccess" + error_message = "The EC2 provider must return common managed runner policy inputs with its provider policies." + } + + assert { + condition = toset(keys(output.provider.resources)) == toset(["launch_template", "runners_log_groups", "logfiles"]) + error_message = "EC2-specific artifacts must remain nested under provider resources." + } + + assert { + condition = aws_iam_instance_profile.runner[0].role == "provider-test-runner" + error_message = "The EC2 instance profile must use the common runner role name." + } + +} + +run "accepts_partial_typed_compute_options" { + command = plan + + variables { + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/ami-id" + } + kms_key = null + } + binaries_syncer = { + enabled = false + s3 = null + } + cloudwatch_agent = { + enabled = false + } + managed_security_group_enabled = true + overrides = { + name_runner = "custom-runner" + } + metadata_options = { + http_tokens = "optional" + } + } + } + + assert { + condition = local.name_runner == "custom-runner" && local.name_sg == "provider-test-action-runner" + error_message = "Partial name overrides must retain defaults for omitted attributes." + } + + assert { + condition = ( + aws_launch_template.runner.metadata_options[0].http_tokens == "optional" + && aws_launch_template.runner.metadata_options[0].http_endpoint == "enabled" + && aws_launch_template.runner.metadata_options[0].http_put_response_hop_limit == 1 + && aws_launch_template.runner.metadata_options[0].instance_metadata_tags == "enabled" + ) + error_message = "Partial metadata options must retain typed defaults for omitted attributes." + } + + assert { + condition = toset(keys(output.provider.policies.runner.inline_policies)) == toset([ + "ssm_parameters", + "describe_tags", + "create_tags", + "terminate_self", + ]) + error_message = "Disabled optional EC2 features must remove only their corresponding runner policies." + } +} + +run "separates_provider_runner_and_ssm_tags" { + command = plan + + variables { + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = null + kms_key = null + } + binaries_syncer = { + enabled = false + s3 = null + } + cloudwatch_agent = { + enabled = true + } + managed_security_group_enabled = true + tags = { + Name = "runner-name" + Scope = "runner" + RunnerOnly = "runner" + "ghr:environment" = "runner-override" + "ghr:ssm_config_path" = "/runner/override" + "ghr:runner_name_prefix" = "runner-override" + } + } + tags = { + Name = "provider-name" + Scope = "provider" + } + runner = { + name_prefix = "required-prefix" + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/provider-test-runner" + name = "provider-test-runner" + } + } + } + ssm = { + paths = { + root = "/github-runner/provider-test" + tokens = "tokens" + config = "config" + } + parameters = { + tags = { + Name = "ssm-name" + Scope = "ssm" + SsmOnly = "ssm" + "ghr:ami_name" = "ssm-override" + "ghr:ami_creation_date" = "ssm-override" + "ghr:ami_deprecation_time" = "ssm-override" + } + } + } + observability = { + logs = { + tags = { + Name = "log-name" + Scope = "log" + LogOnly = "log" + } + } + } + } + + assert { + condition = ( + aws_launch_template.runner.tags["Name"] == "provider-name" + && aws_launch_template.runner.tags["Scope"] == "provider" + && !contains(keys(aws_launch_template.runner.tags), "RunnerOnly") + && !contains(keys(aws_launch_template.runner.tags), "SsmOnly") + && !contains(keys(aws_launch_template.runner.tags), "ghr:environment") + && !contains(keys(aws_launch_template.runner.tags), "ghr:ssm_config_path") + && !contains(keys(aws_launch_template.runner.tags), "ghr:runner_name_prefix") + ) + error_message = "Non-runner EC2 resources must use provider tags without runner or SSM component tags." + } + + assert { + condition = toset([ + for tag_specification in aws_launch_template.runner.tag_specifications : tag_specification.resource_type + ]) == toset(["instance", "volume", "network-interface", "spot-instances-request"]) + error_message = "The launch template must define runner tags for every supported runner resource type." + } + + assert { + condition = alltrue([ + for tag_specification in aws_launch_template.runner.tag_specifications : ( + tag_specification.tags["Name"] == "runner-name" + && tag_specification.tags["Scope"] == "runner" + && tag_specification.tags["RunnerOnly"] == "runner" + && !contains(keys(tag_specification.tags), "SsmOnly") + && tag_specification.tags["ghr:environment"] == "provider-test" + && tag_specification.tags["ghr:ssm_config_path"] == "/github-runner/provider-test/config" + && tag_specification.tags["ghr:runner_name_prefix"] == "required-prefix" + ) + ]) + error_message = "Runner resource tags must apply runner overrides while protecting mandatory bootstrap tags." + } + + assert { + condition = ( + aws_ssm_parameter.runner_config_run_as.tags["Name"] == "ssm-name" + && aws_ssm_parameter.runner_config_run_as.tags["Scope"] == "ssm" + && aws_ssm_parameter.runner_config_run_as.tags["SsmOnly"] == "ssm" + && !contains(keys(aws_ssm_parameter.runner_config_run_as.tags), "RunnerOnly") + && !contains(keys(aws_ssm_parameter.runner_config_run_as.tags), "ghr:environment") + ) + error_message = "EC2 SSM parameters must merge SSM component tags over provider tags." + } + + assert { + condition = alltrue([ + for log_group in aws_cloudwatch_log_group.gh_runners : ( + log_group.tags["Name"] == "log-name" + && log_group.tags["Scope"] == "log" + && log_group.tags["LogOnly"] == "log" + && !contains(keys(log_group.tags), "RunnerOnly") + && !contains(keys(log_group.tags), "SsmOnly") + ) + ]) + error_message = "EC2 log groups must merge shared log tags over provider tags without runner or SSM tags." + } + + assert { + condition = ( + aws_ssm_parameter.runner_ami_id[0].tags["Name"] == "ssm-name" + && aws_ssm_parameter.runner_ami_id[0].tags["Scope"] == "ssm" + && aws_ssm_parameter.runner_ami_id[0].tags["SsmOnly"] == "ssm" + && aws_ssm_parameter.runner_ami_id[0].tags["ghr:ami_name"] == "runner-test" + && aws_ssm_parameter.runner_ami_id[0].tags["ghr:ami_creation_date"] == "2026-01-01T00:00:00.000Z" + && aws_ssm_parameter.runner_ami_id[0].tags["ghr:ami_deprecation_time"] == "" + ) + error_message = "The managed AMI parameter must preserve authoritative AMI metadata over SSM component tags." + } +} + +run "rejects_external_instance_profile_with_managed_role" { + command = plan + + variables { + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + instance_profile = { + name = "external-runner-profile" + } + binaries_syncer = { + enabled = false + } + } + + runner = { + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/provider-test-runner" + name = "provider-test-runner" + managed = true + } + } + } + } + + expect_failures = [terraform_data.validate_config] +} + +run "requires_distribution_object_when_sync_is_enabled" { + command = plan + + variables { + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + binaries_syncer = { + enabled = true + s3 = null + } + } + } + + expect_failures = [terraform_data.validate_config] +} diff --git a/modules/compute-providers/aws/ec2/trust-policy/README.md b/modules/compute-providers/aws/ec2/trust-policy/README.md new file mode 100644 index 0000000000..f73dc49b9b --- /dev/null +++ b/modules/compute-providers/aws/ec2/trust-policy/README.md @@ -0,0 +1,43 @@ +# EC2 runner trust policy + +This internal submodule builds the EC2 runner-role trust policy independently from EC2 resources that consume the runner role. It preserves the default EC2 service trust and optionally merges an additional IAM trust policy document supplied by the common runner configuration. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.validate_config](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_trust\_policy\_json](#input\_additional\_trust\_policy\_json) | Optional IAM policy document merged with the default EC2 runner-role trust policy. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [assume\_role\_policy](#output\_assume\_role\_policy) | EC2 runner-role trust policy with the optional additional trust policy merged into it. | + diff --git a/modules/compute-providers/aws/ec2/trust-policy/assume-role.tf b/modules/compute-providers/aws/ec2/trust-policy/assume-role.tf new file mode 100644 index 0000000000..bea81c1b9e --- /dev/null +++ b/modules/compute-providers/aws/ec2/trust-policy/assume-role.tf @@ -0,0 +1,18 @@ +data "aws_iam_policy_document" "default" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "assume_role" { + source_policy_documents = compact([ + data.aws_iam_policy_document.default.json, + var.additional_trust_policy_json, + ]) +} diff --git a/modules/compute-providers/aws/ec2/trust-policy/outputs.tf b/modules/compute-providers/aws/ec2/trust-policy/outputs.tf new file mode 100644 index 0000000000..0c28bf1661 --- /dev/null +++ b/modules/compute-providers/aws/ec2/trust-policy/outputs.tf @@ -0,0 +1,4 @@ +output "assume_role_policy" { + description = "EC2 runner-role trust policy with the optional additional trust policy merged into it." + value = data.aws_iam_policy_document.assume_role.json +} diff --git a/modules/compute-providers/aws/ec2/trust-policy/tests/trust-policy.tftest.hcl b/modules/compute-providers/aws/ec2/trust-policy/tests/trust-policy.tftest.hcl new file mode 100644 index 0000000000..be297e0f10 --- /dev/null +++ b/modules/compute-providers/aws/ec2/trust-policy/tests/trust-policy.tftest.hcl @@ -0,0 +1,71 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{}" + } + } +} + +run "returns_default_ec2_trust_policy" { + command = plan + + assert { + condition = toset(data.aws_iam_policy_document.default.statement[0].actions) == toset(["sts:AssumeRole"]) + error_message = "The default EC2 runner role trust policy must allow sts:AssumeRole." + } + + assert { + condition = anytrue([ + for principal in data.aws_iam_policy_document.default.statement[0].principals : + principal.type == "Service" && toset(principal.identifiers) == toset(["ec2.amazonaws.com"]) + ]) + error_message = "The default EC2 runner role trust policy must trust the EC2 service principal." + } + + assert { + condition = ( + length(data.aws_iam_policy_document.assume_role.source_policy_documents) == 1 + && output.assume_role_policy == data.aws_iam_policy_document.assume_role.json + ) + error_message = "The submodule must return the final EC2 assume-role policy." + } +} + +run "merges_additional_trust_policy" { + command = plan + + variables { + additional_trust_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "TrustedAccount" + Effect = "Allow" + Action = "sts:AssumeRole" + Principal = { AWS = "arn:aws:iam::123456789012:root" } + }] + }) + } + + assert { + condition = ( + length(data.aws_iam_policy_document.assume_role.source_policy_documents) == 2 + && data.aws_iam_policy_document.assume_role.source_policy_documents[1] == var.additional_trust_policy_json + && output.assume_role_policy == data.aws_iam_policy_document.assume_role.json + ) + error_message = "The submodule must merge the additional trust policy into the final assume-role policy." + } +} + +run "rejects_invalid_additional_trust_policy" { + command = plan + + variables { + additional_trust_policy_json = "not-json" + } + + plan_options { + target = [terraform_data.validate_config] + } + + expect_failures = [terraform_data.validate_config] +} diff --git a/modules/compute-providers/aws/ec2/trust-policy/validations.tf b/modules/compute-providers/aws/ec2/trust-policy/validations.tf new file mode 100644 index 0000000000..351a1b654f --- /dev/null +++ b/modules/compute-providers/aws/ec2/trust-policy/validations.tf @@ -0,0 +1,8 @@ +resource "terraform_data" "validate_config" { + lifecycle { + precondition { + condition = var.additional_trust_policy_json == null ? true : can(jsondecode(var.additional_trust_policy_json)) + error_message = "additional_trust_policy_json must be valid JSON when set." + } + } +} diff --git a/modules/compute-providers/aws/ec2/trust-policy/variables.tf b/modules/compute-providers/aws/ec2/trust-policy/variables.tf new file mode 100644 index 0000000000..8afb35268e --- /dev/null +++ b/modules/compute-providers/aws/ec2/trust-policy/variables.tf @@ -0,0 +1,5 @@ +variable "additional_trust_policy_json" { + description = "Optional IAM policy document merged with the default EC2 runner-role trust policy." + type = string + default = null +} diff --git a/modules/compute-providers/aws/ec2/trust-policy/versions.tf b/modules/compute-providers/aws/ec2/trust-policy/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/compute-providers/aws/ec2/trust-policy/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/compute-providers/aws/ec2/validations.tf b/modules/compute-providers/aws/ec2/validations.tf new file mode 100644 index 0000000000..0836cf0f7a --- /dev/null +++ b/modules/compute-providers/aws/ec2/validations.tf @@ -0,0 +1,53 @@ +resource "terraform_data" "validate_config" { + lifecycle { + precondition { + condition = contains(["spot", "on-demand"], var.config.instance_target_capacity_type) + error_message = "compute_provider.aws.ec2.instance_target_capacity_type must be spot or on-demand." + } + + precondition { + condition = contains( + ["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized", "price-capacity-optimized", "prioritized"], + var.config.instance_allocation_strategy, + ) + error_message = "compute_provider.aws.ec2.instance_allocation_strategy is not supported." + } + + precondition { + condition = var.config.credit_specification == null ? true : contains(["standard", "unlimited"], var.config.credit_specification) + error_message = "compute_provider.aws.ec2.credit_specification must be null, standard, or unlimited." + } + + precondition { + condition = var.config.cpu_options == null ? true : ( + (var.config.cpu_options.amd_sev_snp == null ? true : contains(["enabled", "disabled"], var.config.cpu_options.amd_sev_snp)) && + (var.config.cpu_options.nested_virtualization == null ? true : contains(["enabled", "disabled"], var.config.cpu_options.nested_virtualization)) + ) + error_message = "compute_provider.aws.ec2.cpu_options amd_sev_snp and nested_virtualization must be enabled or disabled when set." + } + + precondition { + condition = !var.config.binaries_syncer.enabled || var.config.binaries_syncer.s3 != null + error_message = "compute_provider.aws.ec2.binaries_syncer.s3 must be set when compute_provider.aws.ec2.binaries_syncer.enabled is true." + } + + precondition { + condition = var.config.instance_profile == null || !var.runner.iam.role.managed + error_message = "runner.iam.role must be set when compute_provider.aws.ec2.instance_profile selects an external instance profile." + } + } +} + +resource "terraform_data" "validate_runner" { + lifecycle { + precondition { + condition = contains(["linux", "osx", "windows"], var.runner.os) + error_message = "runner.os must be linux, osx, or windows." + } + + precondition { + condition = length(var.runner.name_prefix) <= 45 + error_message = "runner.name_prefix must be at most 45 characters." + } + } +} diff --git a/modules/compute-providers/aws/ec2/variables.tf b/modules/compute-providers/aws/ec2/variables.tf new file mode 100644 index 0000000000..da04a3b37c --- /dev/null +++ b/modules/compute-providers/aws/ec2/variables.tf @@ -0,0 +1,365 @@ +variable "aws_partition" { + description = "AWS partition used to construct IAM ARNs." + type = string + default = "aws" +} + +variable "aws_region" { + description = "AWS region used by compute-provider resources and policy documents." + type = string +} + +variable "prefix" { + description = "Prefix used to identify resources created for the runner configuration." + type = string + default = "github-actions" +} + +variable "tags" { + description = "Base tags available to taggable compute-provider resources. Provider-specific tags override this map within their documented scopes." + type = map(string) + default = {} +} + +variable "config" { + description = <<-EOT + EC2 compute-provider configuration. Paths match `compute_provider.aws.ec2` in the runner configuration. + + - `ami`: Optional AMI discovery and encryption configuration. Null selects defaults for `runner.os` and `runner.architecture`. + - `ami.filter`: AMI filter names mapped to accepted values and merged over the provider defaults. + - `ami.owners`: AWS account IDs or aliases allowed to own the selected AMI. + - `ami.id_ssm_parameter`: Optional externally managed SSM parameter containing the AMI ID. Its object presence is the plan-time ownership discriminator. + - `ami.id_ssm_parameter.arn`: ARN of the external AMI-ID parameter. The ARN may remain unknown until apply. + - `ami.kms_key`: Optional customer-managed KMS key required for encrypted AMIs or snapshots. Its object presence is the plan-time policy discriminator. + - `ami.kms_key.arn`: ARN of the AMI KMS key. The ARN may remain unknown until apply. + - `vpc_id`: VPC in which runner networking resources are created. + - `subnet_ids`: Subnets from which the control plane may launch runners. + - `overrides.name_runner`: Optional Name tag override for runner compute resources. + - `overrides.name_sg`: Optional Name tag override for the managed security group. + - `instance_profile`: Optional externally managed instance profile. Its object presence is the plan-time ownership discriminator. + - `instance_profile.name`: Name of the external instance profile. The name may remain unknown until apply. + - `instance_profile_path`: IAM path for the provider-managed instance profile. Null derives the path from `prefix`. + - `binaries_syncer.enabled`: Uses the synchronized runner distribution from S3 during bootstrap. + - `binaries_syncer.s3`: S3 object containing the synchronized runner distribution. Required when synchronization is enabled. + - `binaries_syncer.s3.arn`: Runner-distribution bucket ARN used by IAM policies. + - `binaries_syncer.s3.id`: Runner-distribution bucket name used in the bootstrap URI. + - `binaries_syncer.s3.key`: Runner-distribution object key. + - `block_device_mappings`: EBS mappings added to the launch template. + - `block_device_mappings[].delete_on_termination`: Deletes the volume when its runner terminates. + - `block_device_mappings[].device_name`: Device name exposed to the runner instance. + - `block_device_mappings[].encrypted`: Enables EBS encryption. + - `block_device_mappings[].iops`: Provisioned IOPS for volume types that support configurable IOPS. + - `block_device_mappings[].kms_key_id`: KMS key ID or ARN used to encrypt the volume. + - `block_device_mappings[].snapshot_id`: Snapshot used to initialize the volume. + - `block_device_mappings[].throughput`: Provisioned throughput for volume types that support it. + - `block_device_mappings[].volume_initialization_rate`: Fixed initialization rate for supported snapshot-backed volumes. + - `block_device_mappings[].volume_size`: EBS volume size in GiB. + - `block_device_mappings[].volume_type`: EBS volume type. + - `ebs_optimized`: Requests EBS-optimized instances. + - `instance_target_capacity_type`: Primary capacity type, either `spot` or `on-demand`. + - `instance_allocation_strategy`: EC2 Fleet allocation strategy. + - `instance_type_priorities`: Optional numeric priorities keyed by instance type. + - `instance_max_spot_price`: Optional maximum hourly Spot price. + - `instance_types`: EC2 instance types available to the control plane. + - `user_data`: Runner bootstrap user-data configuration. + - `user_data.enabled`: Enables launch-template user data. + - `user_data.template`: Optional path to a custom user-data template. + - `user_data.content`: Optional complete user-data content used instead of a template. + - `user_data.pre_install`: Script inserted before runner installation. + - `user_data.post_install`: Script inserted after runner installation. + - `user_data.debug_logging_enabled`: Enables verbose user-data tracing, which can expose secrets. + - `ssm_enabled`: Includes Session Manager permissions in the provider's runner policy group. + - `create_service_linked_role_spot`: Allows scale-up to create the EC2 Spot service-linked role. + - `cloudwatch_agent.enabled`: Enables CloudWatch agent configuration for runner instances. + - `cloudwatch_agent.config`: Optional complete CloudWatch agent configuration. + - `managed_security_group_enabled`: Creates and attaches the provider-managed security group. + - `log_files`: Optional files collected by the CloudWatch agent. Null uses provider defaults. + - `log_files[].log_group_name`: CloudWatch log-group name before optional prefixing. + - `log_files[].prefix_log_group`: Prefixes the log-group name with the runner configuration path. + - `log_files[].file_path`: File or glob read by the CloudWatch agent. + - `log_files[].log_stream_name`: CloudWatch log-stream name template. + - `log_files[].log_class`: CloudWatch log-group class for the collected file. + - `key_name`: Optional EC2 key-pair name. + - `additional_security_group_ids`: Existing security groups attached to runners. + - `detailed_monitoring_enabled`: Enables detailed EC2 monitoring. + - `egress_rules`: Rules created on the managed security group. + - `egress_rules[].cidr_blocks`: IPv4 CIDR destinations. + - `egress_rules[].ipv6_cidr_blocks`: IPv6 CIDR destinations. + - `egress_rules[].prefix_list_ids`: AWS prefix-list destinations. + - `egress_rules[].from_port`: First destination port in the permitted range. + - `egress_rules[].protocol`: IP protocol name or number. Use `-1` for all protocols. + - `egress_rules[].security_groups`: Destination security-group IDs. + - `egress_rules[].self`: Allows traffic to the managed security group itself. + - `egress_rules[].to_port`: Last destination port in the permitted range. + - `egress_rules[].description`: Optional rule description. + - `tags`: Runner instance, volume, network-interface, and eligible Spot-request tags. Provider-required bootstrap tags take final precedence. + - `metadata_options`: Instance Metadata Service configuration. + - `metadata_options.instance_metadata_tags`: Exposes instance tags through Instance Metadata Service when enabled. + - `metadata_options.http_endpoint`: Enables or disables the Instance Metadata Service endpoint. + - `metadata_options.http_tokens`: Controls whether IMDSv2 session tokens are optional or required. + - `metadata_options.http_put_response_hop_limit`: Network hop limit for Instance Metadata Service token responses. + - `credit_specification`: CPU credit mode for burstable instance types. + - `cpu_options`: CPU topology and processor-feature configuration. + - `cpu_options.core_count`: Number of CPU cores exposed to the runner instance. + - `cpu_options.threads_per_core`: Number of hardware threads exposed per CPU core. + - `cpu_options.amd_sev_snp`: Enables or disables AMD SEV-SNP on supported instance types. + - `cpu_options.nested_virtualization`: Enables or disables nested virtualization on supported instance types. + - `placement`: EC2 placement configuration. + - `placement.affinity`: Dedicated Host affinity setting. + - `placement.availability_zone`: Availability Zone in which runner instances are placed. + - `placement.group_id`: Placement-group ID. + - `placement.group_name`: Placement-group name. + - `placement.host_id`: Dedicated Host ID. + - `placement.host_resource_group_arn`: ARN of the host resource group used for placement. + - `placement.spread_domain`: Spread-domain placement value. + - `placement.tenancy`: Instance tenancy, such as `default`, `dedicated`, or `host`. + - `placement.partition_number`: Placement-group partition number. + - `license_specifications`: License Manager configurations added to the launch template. + - `license_specifications[].license_configuration_arn`: ARN of an AWS License Manager license configuration. + - `associate_public_ipv4_address`: Associates a public IPv4 address with runner network interfaces. + - `on_demand_failover_for_errors`: EC2 errors that trigger on-demand fallback after a Spot failure. + - `scale_errors`: EC2 errors treated as retryable scale-up failures. + - `use_dedicated_host`: Enables the dedicated-host launch path. + EOT + + type = object({ + ami = optional(object({ + filter = optional(map(list(string)), { state = ["available"] }) + owners = optional(list(string), ["amazon"]) + id_ssm_parameter = optional(object({ + arn = string + }), null) + kms_key = optional(object({ + arn = string + }), null) + }), null) + vpc_id = string + subnet_ids = list(string) + overrides = optional(object({ + name_runner = optional(string, "") + name_sg = optional(string, "") + }), {}) + instance_profile = optional(object({ + name = string + }), null) + instance_profile_path = optional(string, null) + binaries_syncer = optional(object({ + enabled = optional(bool, true) + s3 = optional(object({ + arn = string + id = string + key = string + }), null) + }), {}) + block_device_mappings = optional(list(object({ + delete_on_termination = optional(bool, true) + device_name = optional(string, "/dev/xvda") + encrypted = optional(bool, true) + iops = optional(number) + kms_key_id = optional(string) + snapshot_id = optional(string) + throughput = optional(number) + volume_initialization_rate = optional(number) + volume_size = number + volume_type = optional(string, "gp3") + })), [{ volume_size = 30 }]) + ebs_optimized = optional(bool, false) + instance_target_capacity_type = optional(string, "spot") + instance_allocation_strategy = optional(string, "lowest-price") + instance_type_priorities = optional(map(number), null) + instance_max_spot_price = optional(string, null) + instance_types = list(string) + user_data = optional(object({ + enabled = optional(bool, true) + template = optional(string, null) + content = optional(string, null) + pre_install = optional(string, "") + post_install = optional(string, "") + debug_logging_enabled = optional(bool, false) + }), {}) + ssm_enabled = optional(bool, false) + create_service_linked_role_spot = optional(bool, false) + cloudwatch_agent = optional(object({ + enabled = optional(bool, true) + config = optional(string, null) + }), {}) + managed_security_group_enabled = optional(bool, true) + log_files = optional(list(object({ + log_group_name = string + prefix_log_group = bool + file_path = string + log_stream_name = string + log_class = optional(string, "STANDARD") + })), null) + key_name = optional(string, null) + additional_security_group_ids = optional(list(string), []) + detailed_monitoring_enabled = optional(bool, false) + egress_rules = optional(list(object({ + cidr_blocks = list(string) + ipv6_cidr_blocks = list(string) + prefix_list_ids = list(string) + from_port = number + protocol = string + security_groups = list(string) + self = bool + to_port = number + description = string + })), [{ + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + prefix_list_ids = null + from_port = 0 + protocol = "-1" + security_groups = null + self = null + to_port = 0 + description = null + }]) + tags = optional(map(string), {}) + metadata_options = optional(object({ + instance_metadata_tags = optional(string, "enabled") + http_endpoint = optional(string, "enabled") + http_tokens = optional(string, "required") + http_put_response_hop_limit = optional(number, 1) + }), {}) + credit_specification = optional(string, null) + cpu_options = optional(object({ + core_count = optional(number) + threads_per_core = optional(number) + amd_sev_snp = optional(string) + nested_virtualization = optional(string) + }), null) + placement = optional(object({ + affinity = optional(string) + availability_zone = optional(string) + group_id = optional(string) + group_name = optional(string) + host_id = optional(string) + host_resource_group_arn = optional(string) + spread_domain = optional(string) + tenancy = optional(string) + partition_number = optional(number) + }), null) + license_specifications = optional(list(object({ + license_configuration_arn = string + })), []) + associate_public_ipv4_address = optional(bool, false) + on_demand_failover_for_errors = optional(list(string), []) + scale_errors = optional(list(string), [ + "UnfulfillableCapacity", + "MaxSpotInstanceCountExceeded", + "TargetCapacityLimitExceededException", + "RequestLimitExceeded", + "ResourceLimitExceeded", + "MaxSpotInstanceCountExceeded", + "MaxSpotFleetRequestCountExceeded", + "InsufficientInstanceCapacity", + "InsufficientCapacityOnHost", + ]) + use_dedicated_host = optional(bool, false) + }) + + nullable = false +} + +variable "runner" { + description = <<-EOT + Provider-neutral runner settings consumed by compute providers. + + - `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`. + - `architecture`: Runner distribution architecture. + - `name_prefix`: Prefix added to registered runner names. + - `run_as_root`: Runs the runner service as root. + - `run_as`: Operating-system user used when `run_as_root` is false. + - `hooks.job_started`: Script installed as the runner job-started hook. + - `hooks.job_completed`: Script installed as the runner job-completed hook. + - `iam.role.arn`: Resolved runner-role ARN referenced by provider policies and resources. + - `iam.role.name`: Resolved runner-role name used by provider resources. + - `iam.role.managed`: Whether runner-config manages the resolved runner role. + - `iam.managed_policy_arns`: Common managed-policy ARNs returned with the provider-specific runner policies for attachment by runner-config. + - `iam.path`: IAM path available to provider-managed IAM resources. Null derives the path from `prefix`. + EOT + type = object({ + os = optional(string, "linux") + architecture = optional(string, "x64") + name_prefix = optional(string, "") + run_as_root = optional(bool, false) + run_as = optional(string, "ec2-user") + hooks = optional(object({ + job_started = optional(string, "") + job_completed = optional(string, "") + }), {}) + iam = object({ + role = object({ + arn = string + name = string + managed = optional(bool, true) + }) + managed_policy_arns = optional(map(string), {}) + path = optional(string, null) + }) + }) + + nullable = false +} + +variable "github" { + description = <<-EOT + GitHub Enterprise Server settings available to compute-provider bootstrap data. + + - `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com. + - `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server. + EOT + type = object({ + enterprise_server = optional(object({ + url = optional(string, null) + ssl_verify = optional(bool, true) + }), {}) + }) + default = {} + nullable = false +} + +variable "ssm" { + description = <<-EOT + Parameter Store paths and tag scopes available to compute-provider bootstrap resources. + + - `paths.root`: Root Parameter Store path for the runner configuration. + - `paths.tokens`: Path segment used for registration tokens and just-in-time configuration. + - `paths.config`: Path segment used for persistent runner and provider configuration. + - `tags`: Shared SSM tags that override module-level `tags`. + - `parameters.tags`: Parameter-specific tags that override module-level and shared SSM tags. + EOT + type = object({ + paths = object({ + root = string + tokens = string + config = string + }) + tags = optional(map(string), {}) + parameters = optional(object({ + tags = optional(map(string), {}) + }), {}) + }) + + nullable = false +} + +variable "observability" { + description = <<-EOT + CloudWatch Logs settings available to compute-provider runner log groups. + + - `logs.retention_in_days`: Retention period for provider-owned runner log groups. + - `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt runner log groups. + - `logs.tags`: Shared log-group tags that override module-level `tags`. + EOT + type = object({ + logs = optional(object({ + retention_in_days = optional(number, 180) + kms_key_id = optional(string, null) + tags = optional(map(string), {}) + }), {}) + }) + default = {} + nullable = false +} diff --git a/modules/compute-providers/aws/ec2/versions.tf b/modules/compute-providers/aws/ec2/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/compute-providers/aws/ec2/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/compute-providers/aws/microvm/README.md b/modules/compute-providers/aws/microvm/README.md new file mode 100644 index 0000000000..07ecf5d50f --- /dev/null +++ b/modules/compute-providers/aws/microvm/README.md @@ -0,0 +1,70 @@ +# AWS Lambda MicroVM runner provider + +This internal module implements the AWS Lambda MicroVM compute provider used by `runner-config`. It returns provider-specific Lambda environment variables, control-plane IAM policy fragments, selected image metadata, native runtime and optional CloudWatch-agent log groups, and collected-file definitions through the common provider contract; the parent owns the runner role, Lambda resources, queues, schedules, and Parameter Store lifecycle. + +Select it with the `compute_provider.aws.microvm` leaf. The Terraform dispatch key is `aws_microvm`, while the runtime `COMPUTE_PROVIDER_TYPE` remains `microvm` for compatibility with the control-plane Lambda. MicroVM lanes require Linux on ARM64 and ephemeral webhook orchestration with just-in-time configuration enabled. + +MicroVM runners use the provider's fixed 28,800-second (8-hour) lifetime; this is not a Terraform input. + +The resolved provider-neutral `runner.iam.role` is passed to Lambda as the MicroVM execution role. The provider creates `/github-self-hosted-runners//microvm` with the common observability lifecycle and derives a metadata prefix at `//microvm-metadata`. Scale-up, scale-down, and pool use that non-secret prefix for MicroVM ownership and lifecycle state; the runner role can read only the lane's `*.tags` metadata records, the CloudWatch enablement parameter, and its lane-scoped one-time JIT parameter. It can also delete that JIT parameter and write to the provider-managed log groups. MicroVMs sharing the execution role can read the tag records for that lane, but not the ownership and cleanup records. When the runner role is supplied externally, its Lambda trust and these permissions remain caller-owned. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.gh_runners](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_cloudwatch_log_group.runtime](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_ssm_parameter.cloudwatch_agent_config_runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_enable_cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [terraform_data.validate_config](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.validate_runner](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.runner_cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.runner_metadata](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.runner_runtime_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.runner_ssm_jit](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | AWS partition used to construct IAM ARNs. | `string` | `"aws"` | no | +| [aws\_region](#input\_aws\_region) | AWS region used by compute-provider resources and policy documents. | `string` | n/a | yes | +| [config](#input\_config) | Lambda MicroVM compute-provider configuration. Paths match `compute_provider.aws.microvm` in runner-config.

- `image_arn`: ARN of the MicroVM image used to run GitHub runners.
- `image_version`: Optional MicroVM image version.
- `ingress_network_connectors`: Up to 10 Lambda network-connector ARNs passed to RunMicrovm.
- `egress_network_connectors`: Up to 10 Lambda network-connector ARNs passed to RunMicrovm.
- `cloudwatch_agent.enabled`: Enables the image CloudWatch agent through the shared runner configuration path.
- `cloudwatch_agent.config`: Optional complete CloudWatch agent configuration. Null renders the provider default from `log_files`. Custom log destinations must also be declared in `log_files` so Terraform creates their groups and IAM permissions.
- `log_files`: Optional files collected by the CloudWatch agent. Null uses the MicroVM defaults.
- `log_files[].log_group_name`: CloudWatch log-group name before optional prefixing.
- `log_files[].prefix_log_group`: Prefixes the log-group name with the runner configuration path.
- `log_files[].file_path`: File or glob read by the CloudWatch agent.
- `log_files[].log_stream_name`: Log-stream template. The image replaces `{microvm_id}` with the current MicroVM identifier.
- `log_files[].log_class`: CloudWatch log-group class for the collected file.
- `environment_variables`: Additional provider-specific Lambda environment variables merged into scale-up, scale-down, and pool.
- `iam.resource_arns.images`: Optional MicroVM image ARN allowlist for RunMicrovm and TerminateMicrovm. Null restricts both actions to `image_arn`; set an explicit list when dynamic image overrides are enabled. Provider-required list and connector permissions remain separately scoped to `*`.
- `iam.additional_policy_json.scale_up`: Optional additional provider policy attached separately to the scale-up Lambda role.
- `iam.managed_policies.scale_up`: Optional managed-policy wrapper attached to the scale-up Lambda role. Wrapper presence controls resource creation during planning.
- `iam.managed_policies.scale_up.arn`: ARN of the scale-up managed policy. The ARN may remain unknown until apply.
- `iam.managed_policies.pool`: Optional managed-policy wrapper attached to the pool Lambda role. Wrapper presence controls resource creation during planning.
- `iam.managed_policies.pool.arn`: ARN of the pool managed policy. The ARN may remain unknown until apply. |
object({
image_arn = string
image_version = optional(string, null)
ingress_network_connectors = optional(list(string), [])
egress_network_connectors = optional(list(string), [])
cloudwatch_agent = optional(object({
enabled = optional(bool, true)
config = optional(string, null)
}), {})
log_files = optional(list(object({
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
log_class = optional(string, "STANDARD")
})), null)
environment_variables = optional(map(string), {})
iam = optional(object({
resource_arns = optional(object({
images = optional(list(string), null)
}), {})
additional_policy_json = optional(object({
scale_up = optional(string, null)
}), {})
managed_policies = optional(object({
scale_up = optional(object({
arn = string
}), null)
pool = optional(object({
arn = string
}), null)
}), {})
}), {})
})
| n/a | yes | +| [github](#input\_github) | GitHub Enterprise Server settings available to compute-provider bootstrap data.

- `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com.
- `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server. |
object({
enterprise_server = optional(object({
url = optional(string, null)
ssl_verify = optional(bool, true)
}), {})
})
| `{}` | no | +| [observability](#input\_observability) | Provider-neutral observability settings applied to the provider-managed MicroVM runtime log group.

- `logs.retention_in_days`: CloudWatch Logs retention period.
- `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt the log group.
- `logs.class`: CloudWatch log-group class.
- `logs.tags`: Tags merged after module-level tags on the log group. |
object({
logs = optional(object({
retention_in_days = optional(number, 180)
kms_key_id = optional(string, null)
class = optional(string, "STANDARD")
tags = optional(map(string), {})
}), {})
})
| `{}` | no | +| [prefix](#input\_prefix) | Prefix used to identify resources created for the runner configuration. | `string` | `"github-actions"` | no | +| [runner](#input\_runner) | Resolved runner settings consumed by the Lambda MicroVM compute provider.

- `os`: Runner operating system. Lambda MicroVM requires `linux`.
- `architecture`: Runner distribution architecture. Lambda MicroVM requires `arm64`.
- `name_prefix`: Prefix added to registered runner names.
- `run_as_root`: Runs the runner service as root.
- `run_as`: Operating-system user used when `run_as_root` is false.
- `hooks.job_started`: Script installed as the runner job-started hook.
- `hooks.job_completed`: Script installed as the runner job-completed hook.
- `iam.role.arn`: Resolved runner-role ARN used as the MicroVM execution role and referenced by provider policies.
- `iam.role.name`: Resolved runner-role name used by provider resources.
- `iam.role.managed`: Whether runner-config manages the resolved runner role. Callers own an external role and must grant it `ssm:GetParameter` on the lane's `microvm-metadata/*.tags` and `enable_cloudwatch` parameters, `ssm:GetParameter` and `ssm:DeleteParameter` on the lane token path, plus `logs:CreateLogStream` and `logs:PutLogEvents` on the provider-managed runtime log group. When the CloudWatch agent is enabled, it also needs `ssm:GetParameter` on `cloudwatch_agent_config_runner` and stream access to the configured runner log groups.
- `iam.managed_policy_arns`: Common managed-policy ARNs returned with the provider-specific runner policies for attachment by runner-config.
- `iam.path`: IAM path available to provider-managed IAM resources. Null derives the path from `prefix`. |
object({
os = optional(string, "linux")
architecture = optional(string, "arm64")
name_prefix = optional(string, "")
run_as_root = optional(bool, false)
run_as = optional(string, "ec2-user")
hooks = optional(object({
job_started = optional(string, "")
job_completed = optional(string, "")
}), {})
iam = object({
role = object({
arn = string
name = string
managed = optional(bool, true)
})
managed_policy_arns = optional(map(string), {})
path = optional(string, null)
})
})
| n/a | yes | +| [ssm](#input\_ssm) | Parameter Store paths and tag scopes available to compute-provider bootstrap resources.

- `paths.root`: Root Parameter Store path for the runner configuration.
- `paths.tokens`: Path segment used for registration tokens and just-in-time configuration.
- `paths.config`: Path segment used for persistent runner and provider configuration. MicroVM control-plane metadata is stored under its `microvm-metadata` child prefix.
- `tags`: Shared SSM tags that override module-level `tags`.
- `parameters.tags`: Parameter-specific tags that override module-level and shared SSM tags. |
object({
paths = object({
root = string
tokens = string
config = string
})
tags = optional(map(string), {})
parameters = optional(object({
tags = optional(map(string), {})
}), {})
})
| n/a | yes | +| [tags](#input\_tags) | Base tags available to taggable compute-provider resources. Provider-specific tags override this map within their documented scopes. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [environment\_variables](#output\_environment\_variables) | Provider-specific Lambda environment variable fragments consumed by runner-config. | +| [policies](#output\_policies) | Provider-specific IAM policy fragments consumed by runner-config. | +| [provider](#output\_provider) | Nested Lambda MicroVM compute-provider contract consumed by runner-config. | +| [resources](#output\_resources) | Provider-specific MicroVM resources exposed by runner-config. | + diff --git a/modules/compute-providers/aws/microvm/control-plane.tf b/modules/compute-providers/aws/microvm/control-plane.tf new file mode 100644 index 0000000000..832a90df04 --- /dev/null +++ b/modules/compute-providers/aws/microvm/control-plane.tf @@ -0,0 +1,118 @@ +data "aws_iam_policy_document" "scale_up" { + statement { + effect = "Allow" + actions = [ + "lambda:ListMicrovms", + "lambda:PassNetworkConnector", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "lambda:RunMicrovm", + "lambda:TerminateMicrovm", + ] + resources = local.microvm_image_resource_arns + } + + statement { + effect = "Allow" + actions = [ + "ssm:AddTagsToResource", + "ssm:DeleteParameter", + "ssm:PutParameter", + ] + resources = [local.microvm_metadata_parameter_arn] + } + + statement { + effect = "Allow" + actions = ["ssm:GetParametersByPath"] + resources = [local.microvm_metadata_path_arn, local.microvm_metadata_parameter_arn] + } + + statement { + effect = "Allow" + actions = ["ssm:GetParameters"] + resources = [local.microvm_metadata_parameter_arn] + } + + statement { + effect = "Allow" + actions = ["iam:PassRole"] + resources = [var.runner.iam.role.arn] + } + + statement { + effect = "Allow" + actions = ["ssm:DeleteParameter"] + resources = [local.runner_token_path_arn] + } +} + +data "aws_iam_policy_document" "scale_down" { + statement { + effect = "Allow" + actions = ["lambda:ListMicrovms"] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["lambda:TerminateMicrovm"] + resources = local.microvm_image_resource_arns + } + + statement { + effect = "Allow" + actions = [ + "ssm:DeleteParameter", + "ssm:PutParameter", + ] + resources = [local.microvm_metadata_parameter_arn] + } + + statement { + effect = "Allow" + actions = ["ssm:GetParametersByPath"] + resources = [local.microvm_metadata_path_arn, local.microvm_metadata_parameter_arn] + } + + statement { + effect = "Allow" + actions = ["ssm:DeleteParameter"] + resources = [local.runner_token_path_arn] + } +} + +locals { + microvm_metadata_ssm_path = "${local.ssm_config_ssm_path}/microvm-metadata" + microvm_metadata_path_arn = "${local.ssm_parameter_arn_prefix}${local.microvm_metadata_ssm_path}" + microvm_metadata_parameter_arn = "${local.microvm_metadata_path_arn}/*" + microvm_image_resource_arns = coalesce( + var.config.iam.resource_arns.images, + [var.config.image_arn], + ) + runner_jit_ssm_path = "/${trim(var.ssm.paths.root, "/")}/${trim(var.ssm.paths.tokens, "/")}" + + microvm_custom_environment_variables = { + for key, value in var.config.environment_variables : key => value + if !contains(["MICROVM_METADATA_TAGS", "MICROVM_RUNNER_CONFIG_SSM_ARN"], key) + } + microvm_environment_variables = merge(local.microvm_custom_environment_variables, { + MICROVM_EGRESS_NETWORK_CONNECTORS = length(var.config.egress_network_connectors) == 0 ? "" : jsonencode(var.config.egress_network_connectors) + MICROVM_EXECUTION_ROLE_ARN = var.runner.iam.role.arn + MICROVM_IMAGE_ARN = var.config.image_arn + MICROVM_IMAGE_VERSION = var.config.image_version == null ? "" : var.config.image_version + MICROVM_INGRESS_NETWORK_CONNECTORS = length(var.config.ingress_network_connectors) == 0 ? "" : jsonencode(var.config.ingress_network_connectors) + MICROVM_LOG_GROUP = aws_cloudwatch_log_group.runtime.name + MICROVM_METADATA_SSM_PATH = local.microvm_metadata_ssm_path + SSM_TOKEN_PATH = local.runner_jit_ssm_path + }) + + scale_up_environment_variables = local.microvm_environment_variables + scale_down_environment_variables = local.microvm_environment_variables + pool_environment_variables = local.microvm_environment_variables +} diff --git a/modules/compute-providers/aws/microvm/logging.tf b/modules/compute-providers/aws/microvm/logging.tf new file mode 100644 index 0000000000..18f87ffeed --- /dev/null +++ b/modules/compute-providers/aws/microvm/logging.tf @@ -0,0 +1,86 @@ +locals { + provider_tags = merge( + { + "Name" = format("%s-action-runner", var.prefix) + }, + var.tags, + ) + + log_group_tags = merge( + local.provider_tags, + var.observability.logs.tags, + ) + + ssm_config_ssm_path = "/${trim(var.ssm.paths.root, "/")}/${trim(var.ssm.paths.config, "/")}" + ssm_parameter_tags = merge( + local.provider_tags, + var.ssm.tags, + var.ssm.parameters.tags, + ) + + runner_log_files = var.config.log_files != null ? var.config.log_files : [ + { + log_group_name = "internal_service" + prefix_log_group = true + file_path = "/var/log/microvm/internal-services.log" + log_stream_name = "{microvm_id}" + log_class = "STANDARD" + }, + { + log_group_name = "run" + prefix_log_group = true + file_path = "/var/log/microvm/run.log" + log_stream_name = "{microvm_id}" + log_class = "STANDARD" + }, + { + log_group_name = "runner" + prefix_log_group = true + file_path = "/opt/actions-runner/_diag/Runner_**.log" + log_stream_name = "{microvm_id}" + log_class = "STANDARD" + }, + ] + + logfiles = var.config.cloudwatch_agent.enabled ? [for log_file in local.runner_log_files : { + log_group_name = log_file.prefix_log_group ? "/github-self-hosted-runners/${var.prefix}/${log_file.log_group_name}" : "/${log_file.log_group_name}" + log_stream_name = log_file.log_stream_name + file_path = log_file.file_path + log_group_class = log_file.log_class + }] : [] + runner_log_group_names = distinct([for log_file in local.logfiles : log_file.log_group_name]) + runner_log_group_classes = [for name in local.runner_log_group_names : [ + for log_file in local.logfiles : log_file.log_group_class + if log_file.log_group_name == name + ][0]] +} + +resource "aws_cloudwatch_log_group" "runtime" { + name = "/github-self-hosted-runners/${var.prefix}/microvm" + retention_in_days = var.observability.logs.retention_in_days + kms_key_id = var.observability.logs.kms_key_id + log_group_class = var.observability.logs.class + tags = local.log_group_tags +} + +resource "aws_ssm_parameter" "cloudwatch_agent_config_runner" { + count = var.config.cloudwatch_agent.enabled ? 1 : 0 + + name = "${local.ssm_config_ssm_path}/cloudwatch_agent_config_runner" + type = "String" + value = var.config.cloudwatch_agent.config != null ? var.config.cloudwatch_agent.config : templatefile( + "${path.module}/templates/cloudwatch_config.json", + { logfiles = jsonencode(local.logfiles) }, + ) + tags = local.ssm_parameter_tags +} + +resource "aws_cloudwatch_log_group" "gh_runners" { + count = length(local.runner_log_group_names) + + name = local.runner_log_group_names[count.index] + retention_in_days = var.observability.logs.retention_in_days + kms_key_id = var.observability.logs.kms_key_id + log_group_class = local.runner_log_group_classes[count.index] + tags = local.log_group_tags +} diff --git a/modules/compute-providers/aws/microvm/outputs.tf b/modules/compute-providers/aws/microvm/outputs.tf new file mode 100644 index 0000000000..6200a8f54e --- /dev/null +++ b/modules/compute-providers/aws/microvm/outputs.tf @@ -0,0 +1,23 @@ +output "environment_variables" { + description = "Provider-specific Lambda environment variable fragments consumed by runner-config." + value = local.provider_environment_variables +} + +output "policies" { + description = "Provider-specific IAM policy fragments consumed by runner-config." + value = local.provider_policies +} + +output "resources" { + description = "Provider-specific MicroVM resources exposed by runner-config." + value = local.provider_resources +} + +output "provider" { + description = "Nested Lambda MicroVM compute-provider contract consumed by runner-config." + value = { + environment_variables = local.provider_environment_variables + policies = local.provider_policies + resources = local.provider_resources + } +} diff --git a/modules/compute-providers/aws/microvm/policies-runner.tf b/modules/compute-providers/aws/microvm/policies-runner.tf new file mode 100644 index 0000000000..9b514e396a --- /dev/null +++ b/modules/compute-providers/aws/microvm/policies-runner.tf @@ -0,0 +1,81 @@ +data "aws_caller_identity" "current" {} + +locals { + ssm_parameter_arn_prefix = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter" + runner_token_path_arn = "${local.ssm_parameter_arn_prefix}/${trim(var.ssm.paths.root, "/")}/${trim(var.ssm.paths.tokens, "/")}/*" + runner_metadata_tags_arn = "${local.microvm_metadata_path_arn}/*.tags" + runner_enable_cloudwatch_arn = "${local.ssm_parameter_arn_prefix}${local.ssm_config_ssm_path}/enable_cloudwatch" + runner_cloudwatch_config_arn = "${local.ssm_parameter_arn_prefix}${local.ssm_config_ssm_path}/cloudwatch_agent_config_runner" + runner_cloudwatch_log_group_arns = [for name in local.runner_log_group_names : + "arn:${var.aws_partition}:logs:${var.aws_region}:${data.aws_caller_identity.current.account_id}:log-group:${name}" + ] + runner_inline_policies = merge({ + ssm_jit = { + name = "runner-microvm-ssm-jit" + policy_json = data.aws_iam_policy_document.runner_ssm_jit.json + } + runtime_logs = { + name = "runner-microvm-runtime-logs" + policy_json = data.aws_iam_policy_document.runner_runtime_logs.json + } + runner_metadata = { + name = "runner-microvm-metadata" + policy_json = data.aws_iam_policy_document.runner_metadata.json + } + }, var.config.cloudwatch_agent.enabled ? { + cloudwatch = { + name = "runner-microvm-cloudwatch" + policy_json = data.aws_iam_policy_document.runner_cloudwatch[0].json + } + } : {}) +} + +data "aws_iam_policy_document" "runner_ssm_jit" { + statement { + effect = "Allow" + actions = [ + "ssm:DeleteParameter", + "ssm:GetParameter", + ] + resources = [local.runner_token_path_arn] + } +} + +data "aws_iam_policy_document" "runner_metadata" { + statement { + effect = "Allow" + actions = ["ssm:GetParameter"] + resources = [local.runner_metadata_tags_arn, local.runner_enable_cloudwatch_arn] + } +} + +data "aws_iam_policy_document" "runner_cloudwatch" { + count = var.config.cloudwatch_agent.enabled ? 1 : 0 + + statement { + effect = "Allow" + actions = ["ssm:GetParameter"] + resources = [local.runner_cloudwatch_config_arn] + } + + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + ] + resources = [for arn in local.runner_cloudwatch_log_group_arns : "${arn}:*"] + } +} + +data "aws_iam_policy_document" "runner_runtime_logs" { + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = ["${aws_cloudwatch_log_group.runtime.arn}:*"] + } +} diff --git a/modules/compute-providers/aws/microvm/provider-contract.tf b/modules/compute-providers/aws/microvm/provider-contract.tf new file mode 100644 index 0000000000..2f487a01d7 --- /dev/null +++ b/modules/compute-providers/aws/microvm/provider-contract.tf @@ -0,0 +1,36 @@ +locals { + provider_environment_variables = { + scale_up = local.scale_up_environment_variables + scale_down = local.scale_down_environment_variables + pool = local.pool_environment_variables + } + + provider_policies = { + runner = { + inline_policies = local.runner_inline_policies + managed_policy_arns = var.runner.iam.managed_policy_arns + } + scale_up = { + iam_policy_json = data.aws_iam_policy_document.scale_up.json + additional_iam_policy_json = var.config.iam.additional_policy_json.scale_up + managed_policy_enabled = var.config.iam.managed_policies.scale_up != null + managed_policy_arn = try(var.config.iam.managed_policies.scale_up.arn, null) + } + scale_down = { + iam_policy_json = data.aws_iam_policy_document.scale_down.json + } + pool = { + iam_policy_json = data.aws_iam_policy_document.scale_up.json + managed_policy_enabled = var.config.iam.managed_policies.pool != null + managed_policy_arn = try(var.config.iam.managed_policies.pool.arn, null) + } + } + + provider_resources = { + image_arn = var.config.image_arn + image_version = var.config.image_version + execution_role_arn = var.runner.iam.role.arn + runners_log_groups = concat([aws_cloudwatch_log_group.runtime], aws_cloudwatch_log_group.gh_runners) + logfiles = local.logfiles + } +} diff --git a/modules/compute-providers/aws/microvm/runner-config.tf b/modules/compute-providers/aws/microvm/runner-config.tf new file mode 100644 index 0000000000..de3b775106 --- /dev/null +++ b/modules/compute-providers/aws/microvm/runner-config.tf @@ -0,0 +1,6 @@ +resource "aws_ssm_parameter" "runner_enable_cloudwatch" { + name = "${local.ssm_config_ssm_path}/enable_cloudwatch" + type = "String" + value = var.config.cloudwatch_agent.enabled + tags = local.ssm_parameter_tags +} diff --git a/modules/compute-providers/aws/microvm/templates/cloudwatch_config.json b/modules/compute-providers/aws/microvm/templates/cloudwatch_config.json new file mode 100644 index 0000000000..554de026e5 --- /dev/null +++ b/modules/compute-providers/aws/microvm/templates/cloudwatch_config.json @@ -0,0 +1,12 @@ +{ + "agent": { + "metrics_collection_interval": 5 + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": ${logfiles} + } + } + } +} diff --git a/modules/compute-providers/aws/microvm/tests/provider.tftest.hcl b/modules/compute-providers/aws/microvm/tests/provider.tftest.hcl new file mode 100644 index 0000000000..de167845f2 --- /dev/null +++ b/modules/compute-providers/aws/microvm/tests/provider.tftest.hcl @@ -0,0 +1,633 @@ +mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } + + mock_data "aws_iam_policy_document" { + defaults = { + json = "{}" + } + } + + mock_resource "aws_cloudwatch_log_group" { + defaults = { + arn = "arn:aws:logs:eu-west-1:123456789012:log-group:/github-self-hosted-runners/microvm-test/microvm" + } + } +} + +variables { + aws_region = "eu-west-1" + prefix = "microvm-test" + + tags = { + Module = "runner" + Name = "module" + } + + config = { + image_arn = "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner" + image_version = "3" + ingress_network_connectors = [ + "arn:aws:lambda:eu-west-1:123456789012:network-connector:ingress", + ] + egress_network_connectors = [ + "arn:aws:lambda:eu-west-1:123456789012:network-connector:egress", + ] + environment_variables = { + MICROVM_CLUSTER = "runner-cluster" + MICROVM_IMAGE_ARN = "caller-cannot-override-provider-contract" + MICROVM_METADATA_SSM_PATH = "/caller/cannot/override/provider-contract" + MICROVM_METADATA_TAGS = "retired-provider-contract" + MICROVM_RUNNER_CONFIG_SSM_ARN = "retired-provider-contract" + SSM_TOKEN_PATH = "/caller/cannot/override/token-path" + } + } + + runner = { + name_prefix = "microvm-" + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/microvm-test-runner" + name = "microvm-test-runner" + } + managed_policy_arns = { + readonly = "arn:aws:iam::aws:policy/ReadOnlyAccess" + } + } + } + + ssm = { + paths = { + root = "/github-action-runners" + tokens = "tokens" + config = "config" + } + tags = { + Name = "ssm" + Precedence = "ssm" + Ssm = "shared" + } + parameters = { + tags = { + Name = "parameter" + Parameter = "metadata" + Precedence = "parameter" + "ghr:environment" = "caller-cannot-override" + "ghr:runner_name_prefix" = "caller-cannot-override" + "ghr:ssm_config_path" = "caller-cannot-override" + } + } + } + + observability = { + logs = { + retention_in_days = 30 + kms_key_id = "arn:aws:kms:eu-west-1:123456789012:key/runtime-logs" + class = "INFREQUENT_ACCESS" + tags = { + Name = "microvm-runtime-logs" + LogOnly = "runtime" + } + } + } +} + +run "exposes_microvm_control_plane_contract" { + command = apply + + assert { + condition = toset(keys(output.provider)) == toset(["environment_variables", "policies", "resources"]) + error_message = "The MicroVM provider contract must expose only integration and resource data." + } + + assert { + condition = ( + output.provider.environment_variables.scale_up["MICROVM_CLUSTER"] == "runner-cluster" + && output.provider.environment_variables.scale_up["MICROVM_IMAGE_ARN"] == "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner" + && output.provider.environment_variables.scale_up["MICROVM_IMAGE_VERSION"] == "3" + && output.provider.environment_variables.scale_up["MICROVM_EXECUTION_ROLE_ARN"] == "arn:aws:iam::123456789012:role/microvm-test-runner" + && jsondecode(output.provider.environment_variables.scale_up["MICROVM_INGRESS_NETWORK_CONNECTORS"])[0] == "arn:aws:lambda:eu-west-1:123456789012:network-connector:ingress" + && jsondecode(output.provider.environment_variables.scale_up["MICROVM_EGRESS_NETWORK_CONNECTORS"])[0] == "arn:aws:lambda:eu-west-1:123456789012:network-connector:egress" + && output.provider.environment_variables.scale_up["MICROVM_LOG_GROUP"] == "/github-self-hosted-runners/microvm-test/microvm" + && output.provider.environment_variables.scale_up["MICROVM_METADATA_SSM_PATH"] == "/github-action-runners/config/microvm-metadata" + && output.provider.environment_variables.scale_up["SSM_TOKEN_PATH"] == "/github-action-runners/tokens" + ) + error_message = "The MicroVM provider must map every configured runtime input to the canonical Lambda environment contract." + } + + assert { + condition = ( + toset(keys(output.provider.environment_variables.scale_up)) == toset([ + "MICROVM_CLUSTER", + "MICROVM_EGRESS_NETWORK_CONNECTORS", + "MICROVM_EXECUTION_ROLE_ARN", + "MICROVM_IMAGE_ARN", + "MICROVM_IMAGE_VERSION", + "MICROVM_INGRESS_NETWORK_CONNECTORS", + "MICROVM_LOG_GROUP", + "MICROVM_METADATA_SSM_PATH", + "SSM_TOKEN_PATH", + ]) + && output.provider.environment_variables.scale_up == output.provider.environment_variables.scale_down + && output.provider.environment_variables.scale_up == output.provider.environment_variables.pool + && !contains(keys(output.provider.environment_variables.scale_up), "RUNNER_BOOT_TIME_IN_MINUTES") + && !contains(keys(output.provider.environment_variables.scale_up), "MICROVM_IMAGE_IDENTIFIER") + && !contains(keys(output.provider.environment_variables.scale_up), "MICROVM_MAXIMUM_DURATION_IN_SECONDS") + && !contains(keys(output.provider.environment_variables.scale_up), "MICROVM_RUN_CONFIG") + && !contains(keys(output.provider.environment_variables.scale_up), "MICROVM_TAGS") + && !contains(keys(output.provider.environment_variables.scale_up), "MICROVM_METADATA_TAGS") + && !contains(keys(output.provider.environment_variables.scale_up), "MICROVM_RUNNER_CONFIG_SSM_ARN") + ) + error_message = "All three control-plane fragments must match the runtime key inventory and omit stale or webhook-owned keys." + } + + assert { + condition = ( + data.aws_iam_policy_document.scale_up.statement[0].actions == toset(["lambda:ListMicrovms", "lambda:PassNetworkConnector"]) + && data.aws_iam_policy_document.scale_up.statement[0].resources == toset(["*"]) + && data.aws_iam_policy_document.scale_up.statement[1].actions == toset(["lambda:RunMicrovm", "lambda:TerminateMicrovm"]) + && data.aws_iam_policy_document.scale_up.statement[1].resources == toset(["arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner"]) + ) + error_message = "Scale-up and pool must receive the MicroVM inventory, connector, launch, and cleanup permissions." + } + + assert { + condition = ( + data.aws_iam_policy_document.scale_up.statement[2].actions == toset(["ssm:AddTagsToResource", "ssm:DeleteParameter", "ssm:PutParameter"]) + && data.aws_iam_policy_document.scale_up.statement[2].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*"]) + && data.aws_iam_policy_document.scale_up.statement[3].actions == toset(["ssm:GetParametersByPath"]) + && data.aws_iam_policy_document.scale_up.statement[3].resources == toset([ + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata", + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*", + ]) + && data.aws_iam_policy_document.scale_up.statement[4].actions == toset(["ssm:GetParameters"]) + && data.aws_iam_policy_document.scale_up.statement[4].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*"]) + ) + error_message = "Scale-up and pool must read the metadata hierarchy and exact JIT fence records while keeping metadata writes child-scoped." + } + + assert { + condition = ( + data.aws_iam_policy_document.scale_up.statement[5].actions == toset(["iam:PassRole"]) + && data.aws_iam_policy_document.scale_up.statement[5].resources == toset(["arn:aws:iam::123456789012:role/microvm-test-runner"]) + && length(data.aws_iam_policy_document.scale_up.statement[5].condition) == 0 + && data.aws_iam_policy_document.scale_up.statement[6].actions == toset(["ssm:DeleteParameter"]) + && data.aws_iam_policy_document.scale_up.statement[6].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/tokens/*"]) + ) + error_message = "Scale-up and pool must receive exact runner-role PassRole and lane-token cleanup grants." + } + + assert { + condition = ( + data.aws_iam_policy_document.scale_down.statement[0].actions == toset(["lambda:ListMicrovms"]) + && data.aws_iam_policy_document.scale_down.statement[0].resources == toset(["*"]) + && data.aws_iam_policy_document.scale_down.statement[1].actions == toset(["lambda:TerminateMicrovm"]) + && data.aws_iam_policy_document.scale_down.statement[1].resources == toset(["arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner"]) + && data.aws_iam_policy_document.scale_down.statement[2].actions == toset(["ssm:DeleteParameter", "ssm:PutParameter"]) + && data.aws_iam_policy_document.scale_down.statement[2].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*"]) + && data.aws_iam_policy_document.scale_down.statement[3].actions == toset(["ssm:GetParametersByPath"]) + && data.aws_iam_policy_document.scale_down.statement[3].resources == toset([ + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata", + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*", + ]) + && data.aws_iam_policy_document.scale_down.statement[4].actions == toset(["ssm:DeleteParameter"]) + && data.aws_iam_policy_document.scale_down.statement[4].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/tokens/*"]) + ) + error_message = "Scale-down must receive lifecycle, metadata, and exact lane-token cleanup permissions." + } + + assert { + condition = ( + length(setintersection(toset(flatten(data.aws_iam_policy_document.scale_up.statement[*].actions)), toset(["lambda:ListTags", "lambda:TagResource", "lambda:UntagResource"]))) == 0 + && length(setintersection(toset(flatten(data.aws_iam_policy_document.scale_down.statement[*].actions)), toset(["lambda:ListTags", "lambda:TagResource", "lambda:UntagResource"]))) == 0 + ) + error_message = "The MicroVM provider must not grant unsupported runtime tagging actions." + } + + assert { + condition = ( + toset(keys(output.provider.policies)) == toset(["runner", "scale_up", "scale_down", "pool"]) + && toset(keys(output.provider.policies.runner.inline_policies)) == toset(["cloudwatch", "runner_metadata", "runtime_logs", "ssm_jit"]) + && output.provider.policies.runner.inline_policies.cloudwatch.name == "runner-microvm-cloudwatch" + && output.provider.policies.runner.inline_policies.runner_metadata.name == "runner-microvm-metadata" + && output.provider.policies.runner.inline_policies.ssm_jit.name == "runner-microvm-ssm-jit" + && output.provider.policies.runner.inline_policies.runtime_logs.name == "runner-microvm-runtime-logs" + && output.provider.policies.runner.managed_policy_arns["readonly"] == "arn:aws:iam::aws:policy/ReadOnlyAccess" + && !output.provider.policies.scale_up.managed_policy_enabled + && !output.provider.policies.pool.managed_policy_enabled + ) + error_message = "The MicroVM provider must return policy fragments grouped by common component." + } + + assert { + condition = ( + data.aws_iam_policy_document.runner_ssm_jit.statement[0].actions == toset(["ssm:DeleteParameter", "ssm:GetParameter"]) + && data.aws_iam_policy_document.runner_ssm_jit.statement[0].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/tokens/*"]) + && data.aws_iam_policy_document.runner_metadata.statement[0].actions == toset(["ssm:GetParameter"]) + && data.aws_iam_policy_document.runner_metadata.statement[0].resources == toset([ + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/enable_cloudwatch", + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*.tags", + ]) + && data.aws_iam_policy_document.runner_cloudwatch[0].statement[0].actions == toset(["ssm:GetParameter"]) + && data.aws_iam_policy_document.runner_cloudwatch[0].statement[0].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/cloudwatch_agent_config_runner"]) + && data.aws_iam_policy_document.runner_cloudwatch[0].statement[1].actions == toset(["logs:CreateLogStream", "logs:DescribeLogStreams", "logs:PutLogEvents"]) + && data.aws_iam_policy_document.runner_cloudwatch[0].statement[1].resources == toset([ + "arn:aws:logs:eu-west-1:123456789012:log-group:/github-self-hosted-runners/microvm-test/internal_service:*", + "arn:aws:logs:eu-west-1:123456789012:log-group:/github-self-hosted-runners/microvm-test/run:*", + "arn:aws:logs:eu-west-1:123456789012:log-group:/github-self-hosted-runners/microvm-test/runner:*", + ]) + && length(data.aws_iam_policy_document.runner_runtime_logs.statement) == 1 + && data.aws_iam_policy_document.runner_runtime_logs.statement[0].actions == toset(["logs:CreateLogStream", "logs:PutLogEvents"]) + && data.aws_iam_policy_document.runner_runtime_logs.statement[0].resources == toset(["arn:aws:logs:eu-west-1:123456789012:log-group:/github-self-hosted-runners/microvm-test/microvm:*"]) + ) + error_message = "Managed MicroVM runners must receive exact lane configuration, metadata, JIT, native-runtime, and CloudWatch-agent permissions." + } + + assert { + condition = ( + toset(keys(output.provider.resources)) == toset(["execution_role_arn", "image_arn", "image_version", "logfiles", "runners_log_groups"]) + && output.provider.resources.image_arn == "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner" + && output.provider.resources.image_version == "3" + && output.provider.resources.execution_role_arn == "arn:aws:iam::123456789012:role/microvm-test-runner" + && length(output.provider.resources.runners_log_groups) == 4 + && output.provider.resources.runners_log_groups[0].name == "/github-self-hosted-runners/microvm-test/microvm" + && toset(slice(output.provider.resources.runners_log_groups[*].name, 1, 4)) == toset([ + "/github-self-hosted-runners/microvm-test/internal_service", + "/github-self-hosted-runners/microvm-test/run", + "/github-self-hosted-runners/microvm-test/runner", + ]) + && output.provider.resources.logfiles == local.logfiles + ) + error_message = "The MicroVM provider must expose its selected image, execution role, native runtime group, and CloudWatch-agent resources." + } + + assert { + condition = ( + aws_cloudwatch_log_group.runtime.name == "/github-self-hosted-runners/microvm-test/microvm" + && aws_cloudwatch_log_group.runtime.retention_in_days == 30 + && aws_cloudwatch_log_group.runtime.kms_key_id == "arn:aws:kms:eu-west-1:123456789012:key/runtime-logs" + && aws_cloudwatch_log_group.runtime.log_group_class == "INFREQUENT_ACCESS" + && aws_cloudwatch_log_group.runtime.tags == tomap({ + Name = "microvm-runtime-logs" + Module = "runner" + LogOnly = "runtime" + }) + ) + error_message = "The MicroVM provider must own its lane-scoped log group and apply the common observability lifecycle and tag scopes." + } + + assert { + condition = ( + aws_ssm_parameter.runner_enable_cloudwatch.name == "/github-action-runners/config/enable_cloudwatch" + && aws_ssm_parameter.runner_enable_cloudwatch.value == "true" + && length(aws_ssm_parameter.cloudwatch_agent_config_runner) == 1 + && aws_ssm_parameter.cloudwatch_agent_config_runner[0].name == "/github-action-runners/config/cloudwatch_agent_config_runner" + && aws_ssm_parameter.runner_enable_cloudwatch.tags["Name"] == "parameter" + && aws_ssm_parameter.runner_enable_cloudwatch.tags["Module"] == "runner" + && aws_ssm_parameter.runner_enable_cloudwatch.tags["Ssm"] == "shared" + && aws_ssm_parameter.runner_enable_cloudwatch.tags["Parameter"] == "metadata" + && aws_ssm_parameter.runner_enable_cloudwatch.tags["Precedence"] == "parameter" + && aws_ssm_parameter.cloudwatch_agent_config_runner[0].tags == aws_ssm_parameter.runner_enable_cloudwatch.tags + ) + error_message = "The MicroVM provider must publish the EC2-compatible CloudWatch enablement and agent-config parameters with standard SSM tag precedence." + } + + assert { + condition = ( + length(local.logfiles) == 3 + && local.logfiles[0].file_path == "/var/log/microvm/internal-services.log" + && local.logfiles[0].log_group_name == "/github-self-hosted-runners/microvm-test/internal_service" + && local.logfiles[1].file_path == "/var/log/microvm/run.log" + && local.logfiles[1].log_group_name == "/github-self-hosted-runners/microvm-test/run" + && local.logfiles[2].file_path == "/opt/actions-runner/_diag/Runner_**.log" + && local.logfiles[2].log_group_name == "/github-self-hosted-runners/microvm-test/runner" + && alltrue([for log_file in local.logfiles : ( + log_file.log_group_class == "STANDARD" + && log_file.log_stream_name == "{microvm_id}" + )]) + && length(jsondecode(aws_ssm_parameter.cloudwatch_agent_config_runner[0].value).logs.logs_collected.files.collect_list) == 3 + && toset(aws_cloudwatch_log_group.gh_runners[*].name) == toset(local.runner_log_group_names) + && alltrue([for log_group in aws_cloudwatch_log_group.gh_runners : ( + log_group.retention_in_days == 30 + && log_group.kms_key_id == "arn:aws:kms:eu-west-1:123456789012:key/runtime-logs" + && log_group.tags == aws_cloudwatch_log_group.runtime.tags + )]) + ) + error_message = "The default MicroVM agent configuration must route internal-service, run, and runner files to separately managed log groups." + } +} + +run "normalizes_ssm_paths_and_arns" { + command = plan + + variables { + ssm = { + paths = { + root = "/github-action-runners/" + tokens = "/tokens/" + config = "/config/" + } + } + } + + assert { + condition = ( + output.provider.environment_variables.scale_up["MICROVM_METADATA_SSM_PATH"] == "/github-action-runners/config/microvm-metadata" + && output.provider.environment_variables.scale_up["SSM_TOKEN_PATH"] == "/github-action-runners/tokens" + && data.aws_iam_policy_document.scale_up.statement[6].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/tokens/*"]) + && data.aws_iam_policy_document.scale_down.statement[4].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/tokens/*"]) + && data.aws_iam_policy_document.runner_ssm_jit.statement[0].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/tokens/*"]) + && data.aws_iam_policy_document.runner_metadata.statement[0].resources == toset([ + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/enable_cloudwatch", + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*.tags", + ]) + ) + error_message = "The MicroVM provider must normalize SSM path segments before exposing hook values or IAM resources." + } +} + +run "accepts_external_runner_role_and_policy_overrides" { + command = plan + + variables { + runner = { + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external-microvm-runner" + name = "external-microvm-runner" + managed = false + } + } + } + config = { + image_arn = "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-override" + iam = { + resource_arns = { + images = ["arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-*"] + } + additional_policy_json = { + scale_up = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + managed_policies = { + scale_up = { + arn = "arn:aws:iam::123456789012:policy/microvm-scale-up" + } + pool = { + arn = "arn:aws:iam::123456789012:policy/microvm-pool" + } + } + } + } + } + + assert { + condition = ( + output.provider.environment_variables.scale_up["MICROVM_EXECUTION_ROLE_ARN"] == "arn:aws:iam::123456789012:role/external-microvm-runner" + && output.provider.environment_variables.scale_up["MICROVM_INGRESS_NETWORK_CONNECTORS"] == "" + && output.provider.environment_variables.scale_up["MICROVM_EGRESS_NETWORK_CONNECTORS"] == "" + && output.provider.environment_variables.scale_up["MICROVM_LOG_GROUP"] == "/github-self-hosted-runners/microvm-test/microvm" + && data.aws_iam_policy_document.scale_up.statement[0].resources == toset(["*"]) + && data.aws_iam_policy_document.scale_up.statement[1].resources == toset(["arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-*"]) + && data.aws_iam_policy_document.scale_up.statement[2].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*"]) + && data.aws_iam_policy_document.scale_up.statement[3].resources == toset([ + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata", + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*", + ]) + && data.aws_iam_policy_document.scale_up.statement[4].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*"]) + && data.aws_iam_policy_document.scale_up.statement[5].resources == toset(["arn:aws:iam::123456789012:role/external-microvm-runner"]) + && data.aws_iam_policy_document.scale_up.statement[6].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/tokens/*"]) + && data.aws_iam_policy_document.scale_down.statement[0].resources == toset(["*"]) + && data.aws_iam_policy_document.scale_down.statement[1].resources == toset(["arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-*"]) + && data.aws_iam_policy_document.scale_down.statement[2].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*"]) + && data.aws_iam_policy_document.scale_down.statement[3].resources == toset([ + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata", + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*", + ]) + && data.aws_iam_policy_document.scale_down.statement[4].resources == toset(["arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/tokens/*"]) + ) + error_message = "The provider-neutral external runner role and image allowlist must reach their scoped statements without narrowing required list, connector, or metadata permissions." + } + + assert { + condition = ( + output.provider.policies.scale_up.additional_iam_policy_json == "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + && output.provider.policies.scale_up.managed_policy_enabled + && output.provider.policies.scale_up.managed_policy_arn == "arn:aws:iam::123456789012:policy/microvm-scale-up" + && output.provider.policies.pool.managed_policy_enabled + && output.provider.policies.pool.managed_policy_arn == "arn:aws:iam::123456789012:policy/microvm-pool" + ) + error_message = "Optional MicroVM policy attachments must stay controlled by wrapper presence." + } + + + assert { + condition = ( + toset(keys(output.provider.policies.runner.inline_policies)) == toset(["cloudwatch", "runner_metadata", "runtime_logs", "ssm_jit"]) + && data.aws_iam_policy_document.runner_metadata.statement[0].resources == toset([ + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/enable_cloudwatch", + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/config/microvm-metadata/*.tags", + ]) + && length(data.aws_iam_policy_document.runner_runtime_logs.statement) == 1 + && data.aws_iam_policy_document.runner_runtime_logs.statement[0].resources == toset(["arn:aws:logs:eu-west-1:123456789012:log-group:/github-self-hosted-runners/microvm-test/microvm:*"]) + ) + error_message = "The provider contract must keep plan-known runner-policy keys and scope runtime logging to its provider-managed group." + } +} + +run "disables_cloudwatch_agent_without_disabling_native_runtime_logging" { + command = apply + + variables { + config = { + image_arn = "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner" + cloudwatch_agent = { + enabled = false + } + } + } + + assert { + condition = ( + tostring(aws_ssm_parameter.runner_enable_cloudwatch.value) == "false" + && length(aws_ssm_parameter.cloudwatch_agent_config_runner) == 0 + && length(aws_cloudwatch_log_group.gh_runners) == 0 + && length(local.logfiles) == 0 + && !contains(keys(output.provider.policies.runner.inline_policies), "cloudwatch") + && length(output.provider.resources.runners_log_groups) == 1 + && output.provider.resources.runners_log_groups[0].name == aws_cloudwatch_log_group.runtime.name + && output.provider.environment_variables.scale_up["MICROVM_LOG_GROUP"] == aws_cloudwatch_log_group.runtime.name + ) + error_message = "Disabling the image CloudWatch agent must retain the explicit false flag and native RunMicrovm logging while removing only agent-owned resources and permissions." + } +} + +run "accepts_custom_cloudwatch_agent_config_and_log_group" { + command = apply + + variables { + config = { + image_arn = "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner" + cloudwatch_agent = { + enabled = true + config = "{\"agent\":{\"region\":\"eu-west-1\"}}" + } + log_files = [{ + log_group_name = "custom-microvm" + prefix_log_group = false + file_path = "/var/log/custom.log" + log_stream_name = "{microvm_id}/custom" + log_class = "INFREQUENT_ACCESS" + }] + } + } + + assert { + condition = ( + aws_ssm_parameter.cloudwatch_agent_config_runner[0].value == "{\"agent\":{\"region\":\"eu-west-1\"}}" + && length(local.logfiles) == 1 + && local.logfiles[0].file_path == "/var/log/custom.log" + && local.logfiles[0].log_group_class == "INFREQUENT_ACCESS" + && local.logfiles[0].log_group_name == "/custom-microvm" + && local.logfiles[0].log_stream_name == "{microvm_id}/custom" + && aws_cloudwatch_log_group.gh_runners[0].name == "/custom-microvm" + && aws_cloudwatch_log_group.gh_runners[0].log_group_class == "INFREQUENT_ACCESS" + && data.aws_iam_policy_document.runner_cloudwatch[0].statement[1].resources == toset([ + "arn:aws:logs:eu-west-1:123456789012:log-group:/custom-microvm:*", + ]) + ) + error_message = "Custom MicroVM agent configuration and log-file routing must replace the generated defaults without widening IAM." + } +} + +run "rejects_invalid_image_arn" { + command = plan + + variables { + config = { + image_arn = "not-a-microvm-image-arn" + } + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_metadata_path_overlapping_jit_path" { + command = plan + + variables { + ssm = { + paths = { + root = "/github-action-runners" + tokens = "config/microvm-metadata" + config = "config" + } + } + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_invalid_metadata_path" { + command = plan + + variables { + ssm = { + paths = { + root = "/github-action-runners" + tokens = "tokens" + config = "invalid config" + } + } + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_invalid_image_resource_allowlist" { + command = plan + + variables { + config = { + image_arn = "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner" + iam = { + resource_arns = { + images = [] + } + } + } + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_invalid_network_connector" { + command = plan + + variables { + config = { + image_arn = "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner" + ingress_network_connectors = [ + " ", + ] + } + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_more_than_ten_network_connectors" { + command = plan + + variables { + config = { + image_arn = "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner" + egress_network_connectors = [ + for index in range(11) : + "arn:aws:lambda:eu-west-1:123456789012:network-connector:egress-${index}" + ] + } + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_unsupported_runner_architecture" { + command = plan + + variables { + runner = { + os = "linux" + architecture = "x64" + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/microvm-test-runner" + name = "microvm-test-runner" + } + } + } + } + + expect_failures = [terraform_data.validate_runner] +} + +run "rejects_unsupported_runner_os" { + command = plan + + variables { + runner = { + os = "windows" + architecture = "arm64" + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/microvm-test-runner" + name = "microvm-test-runner" + } + } + } + } + + expect_failures = [terraform_data.validate_runner] +} diff --git a/modules/compute-providers/aws/microvm/trust-policy/README.md b/modules/compute-providers/aws/microvm/trust-policy/README.md new file mode 100644 index 0000000000..43302d1055 --- /dev/null +++ b/modules/compute-providers/aws/microvm/trust-policy/README.md @@ -0,0 +1,41 @@ +# AWS Lambda MicroVM runner trust policy + +This internal submodule builds the MicroVM runner-role trust policy independently from the runtime module that consumes the role. It preserves the default Lambda service trust and optionally merges an additional IAM trust policy document supplied by the common runner configuration. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_trust\_policy\_json](#input\_additional\_trust\_policy\_json) | Optional IAM policy document merged with the MicroVM provider's default runner-role trust policy. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [assume\_role\_policy](#output\_assume\_role\_policy) | MicroVM runner-role trust policy including any additional trust statements. | + diff --git a/modules/compute-providers/aws/microvm/trust-policy/assume-role.tf b/modules/compute-providers/aws/microvm/trust-policy/assume-role.tf new file mode 100644 index 0000000000..3654bce8bf --- /dev/null +++ b/modules/compute-providers/aws/microvm/trust-policy/assume-role.tf @@ -0,0 +1,21 @@ +data "aws_iam_policy_document" "default" { + statement { + effect = "Allow" + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "assume_role" { + source_policy_documents = compact([ + data.aws_iam_policy_document.default.json, + var.additional_trust_policy_json, + ]) +} diff --git a/modules/compute-providers/aws/microvm/trust-policy/outputs.tf b/modules/compute-providers/aws/microvm/trust-policy/outputs.tf new file mode 100644 index 0000000000..8564675873 --- /dev/null +++ b/modules/compute-providers/aws/microvm/trust-policy/outputs.tf @@ -0,0 +1,4 @@ +output "assume_role_policy" { + description = "MicroVM runner-role trust policy including any additional trust statements." + value = data.aws_iam_policy_document.assume_role.json +} diff --git a/modules/compute-providers/aws/microvm/trust-policy/tests/trust-policy.tftest.hcl b/modules/compute-providers/aws/microvm/trust-policy/tests/trust-policy.tftest.hcl new file mode 100644 index 0000000000..5f0173ebc1 --- /dev/null +++ b/modules/compute-providers/aws/microvm/trust-policy/tests/trust-policy.tftest.hcl @@ -0,0 +1,59 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{}" + } + } +} + +run "returns_default_microvm_trust_policy" { + command = plan + + assert { + condition = toset(data.aws_iam_policy_document.default.statement[0].actions) == toset(["sts:AssumeRole", "sts:TagSession"]) + error_message = "The MicroVM runner role must allow assume-role and tagged sessions." + } + + assert { + condition = anytrue([ + for principal in data.aws_iam_policy_document.default.statement[0].principals : + principal.type == "Service" && toset(principal.identifiers) == toset(["lambda.amazonaws.com"]) + ]) + error_message = "The MicroVM runner role must trust the Lambda service principal required by the provider." + } + + assert { + condition = ( + length(data.aws_iam_policy_document.assume_role.source_policy_documents) == 1 + && output.assume_role_policy == data.aws_iam_policy_document.assume_role.json + ) + error_message = "The MicroVM trust-policy module must return the default trust document as assume_role_policy." + } +} + +run "merges_additional_trust_policy" { + command = plan + + variables { + additional_trust_policy_json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"TrustDeploymentRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":\"arn:aws:iam::123456789012:role/deployer\"}}]}" + } + + assert { + condition = ( + length(data.aws_iam_policy_document.assume_role.source_policy_documents) == 2 + && contains(data.aws_iam_policy_document.assume_role.source_policy_documents, var.additional_trust_policy_json) + && output.assume_role_policy == data.aws_iam_policy_document.assume_role.json + ) + error_message = "The MicroVM trust-policy module must merge and return the additional trust policy document." + } +} + +run "rejects_invalid_additional_trust_policy" { + command = plan + + variables { + additional_trust_policy_json = "{" + } + + expect_failures = [var.additional_trust_policy_json] +} diff --git a/modules/compute-providers/aws/microvm/trust-policy/variables.tf b/modules/compute-providers/aws/microvm/trust-policy/variables.tf new file mode 100644 index 0000000000..1af67309d2 --- /dev/null +++ b/modules/compute-providers/aws/microvm/trust-policy/variables.tf @@ -0,0 +1,10 @@ +variable "additional_trust_policy_json" { + description = "Optional IAM policy document merged with the MicroVM provider's default runner-role trust policy." + type = string + default = null + + validation { + condition = var.additional_trust_policy_json == null ? true : can(jsondecode(var.additional_trust_policy_json)) + error_message = "additional_trust_policy_json must be valid JSON when set." + } +} diff --git a/modules/compute-providers/aws/microvm/trust-policy/versions.tf b/modules/compute-providers/aws/microvm/trust-policy/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/compute-providers/aws/microvm/trust-policy/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/compute-providers/aws/microvm/validations.tf b/modules/compute-providers/aws/microvm/validations.tf new file mode 100644 index 0000000000..75c2aeb33b --- /dev/null +++ b/modules/compute-providers/aws/microvm/validations.tf @@ -0,0 +1,80 @@ +resource "terraform_data" "validate_config" { + lifecycle { + precondition { + condition = can(regex("^arn:[^:]+:lambda:[^:]+:[0-9]{12}:microvm-image:.+$", var.config.image_arn)) + error_message = "compute_provider.aws.microvm.image_arn must be a Lambda MicroVM image ARN." + } + + precondition { + condition = var.config.iam.resource_arns.images == null ? true : ( + length(var.config.iam.resource_arns.images) > 0 && + alltrue([ + for image_arn in var.config.iam.resource_arns.images : + image_arn == "*" || can(regex("^arn:[^:]+:lambda:[^:]+:[0-9]{12}:microvm-image:.+$", image_arn)) + ]) + ) + error_message = "compute_provider.aws.microvm.iam.resource_arns.images must be null or a non-empty list containing only * or Lambda MicroVM image ARN patterns." + } + + precondition { + condition = ( + length(var.config.ingress_network_connectors) <= 10 && + alltrue([ + for connector in var.config.ingress_network_connectors : + can(regex("^arn:[^:]+:lambda:[^:]+:([0-9]{12}|aws):network-connector:[^[:space:]]+$", connector)) + ]) + ) + error_message = "compute_provider.aws.microvm.ingress_network_connectors must contain at most 10 Lambda network-connector ARNs." + } + + precondition { + condition = ( + length(var.config.egress_network_connectors) <= 10 && + alltrue([ + for connector in var.config.egress_network_connectors : + can(regex("^arn:[^:]+:lambda:[^:]+:([0-9]{12}|aws):network-connector:[^[:space:]]+$", connector)) + ]) + ) + error_message = "compute_provider.aws.microvm.egress_network_connectors must contain at most 10 Lambda network-connector ARNs." + } + + precondition { + condition = try(var.config.iam.additional_policy_json.scale_up, null) == null ? true : can(jsondecode(var.config.iam.additional_policy_json.scale_up)) + error_message = "compute_provider.aws.microvm.iam.additional_policy_json.scale_up must be valid JSON when set." + } + + precondition { + condition = !( + local.microvm_metadata_ssm_path == local.runner_jit_ssm_path || + startswith(local.microvm_metadata_ssm_path, "${local.runner_jit_ssm_path}/") || + startswith(local.runner_jit_ssm_path, "${local.microvm_metadata_ssm_path}/") + ) + error_message = "The MicroVM metadata Parameter Store path must be separate from the runner JIT configuration path." + } + + precondition { + condition = ( + startswith(var.ssm.paths.root, "/") && + trim(var.ssm.paths.root, "/") != "" && + trim(var.ssm.paths.config, "/") != "" && + can(regex("^/[A-Za-z0-9_./-]+$", local.microvm_metadata_ssm_path)) && + !strcontains(local.microvm_metadata_ssm_path, "//") + ) + error_message = "The derived MicroVM metadata Parameter Store path must be an absolute path containing only letters, numbers, dot, underscore, hyphen, and slash." + } + } +} + +resource "terraform_data" "validate_runner" { + lifecycle { + precondition { + condition = var.runner.os == "linux" && var.runner.architecture == "arm64" + error_message = "Lambda MicroVM runners require runner.os = linux and runner.architecture = arm64." + } + + precondition { + condition = length(var.runner.name_prefix) <= 45 + error_message = "runner.name_prefix must be at most 45 characters." + } + } +} diff --git a/modules/compute-providers/aws/microvm/variables.tf b/modules/compute-providers/aws/microvm/variables.tf new file mode 100644 index 0000000000..388b06135a --- /dev/null +++ b/modules/compute-providers/aws/microvm/variables.tf @@ -0,0 +1,195 @@ +# tflint-ignore: terraform_unused_declarations +variable "aws_partition" { + description = "AWS partition used to construct IAM ARNs." + type = string + default = "aws" +} + +# tflint-ignore: terraform_unused_declarations +variable "aws_region" { + description = "AWS region used by compute-provider resources and policy documents." + type = string +} + +# tflint-ignore: terraform_unused_declarations +variable "prefix" { + description = "Prefix used to identify resources created for the runner configuration." + type = string + default = "github-actions" +} + +# tflint-ignore: terraform_unused_declarations +variable "tags" { + description = "Base tags available to taggable compute-provider resources. Provider-specific tags override this map within their documented scopes." + type = map(string) + default = {} +} + +variable "config" { + description = <<-EOT + Lambda MicroVM compute-provider configuration. Paths match `compute_provider.aws.microvm` in runner-config. + + - `image_arn`: ARN of the MicroVM image used to run GitHub runners. + - `image_version`: Optional MicroVM image version. + - `ingress_network_connectors`: Up to 10 Lambda network-connector ARNs passed to RunMicrovm. + - `egress_network_connectors`: Up to 10 Lambda network-connector ARNs passed to RunMicrovm. + - `cloudwatch_agent.enabled`: Enables the image CloudWatch agent through the shared runner configuration path. + - `cloudwatch_agent.config`: Optional complete CloudWatch agent configuration. Null renders the provider default from `log_files`. Custom log destinations must also be declared in `log_files` so Terraform creates their groups and IAM permissions. + - `log_files`: Optional files collected by the CloudWatch agent. Null uses the MicroVM defaults. + - `log_files[].log_group_name`: CloudWatch log-group name before optional prefixing. + - `log_files[].prefix_log_group`: Prefixes the log-group name with the runner configuration path. + - `log_files[].file_path`: File or glob read by the CloudWatch agent. + - `log_files[].log_stream_name`: Log-stream template. The image replaces `{microvm_id}` with the current MicroVM identifier. + - `log_files[].log_class`: CloudWatch log-group class for the collected file. + - `environment_variables`: Additional provider-specific Lambda environment variables merged into scale-up, scale-down, and pool. + - `iam.resource_arns.images`: Optional MicroVM image ARN allowlist for RunMicrovm and TerminateMicrovm. Null restricts both actions to `image_arn`; set an explicit list when dynamic image overrides are enabled. Provider-required list and connector permissions remain separately scoped to `*`. + - `iam.additional_policy_json.scale_up`: Optional additional provider policy attached separately to the scale-up Lambda role. + - `iam.managed_policies.scale_up`: Optional managed-policy wrapper attached to the scale-up Lambda role. Wrapper presence controls resource creation during planning. + - `iam.managed_policies.scale_up.arn`: ARN of the scale-up managed policy. The ARN may remain unknown until apply. + - `iam.managed_policies.pool`: Optional managed-policy wrapper attached to the pool Lambda role. Wrapper presence controls resource creation during planning. + - `iam.managed_policies.pool.arn`: ARN of the pool managed policy. The ARN may remain unknown until apply. + EOT + + type = object({ + image_arn = string + image_version = optional(string, null) + ingress_network_connectors = optional(list(string), []) + egress_network_connectors = optional(list(string), []) + cloudwatch_agent = optional(object({ + enabled = optional(bool, true) + config = optional(string, null) + }), {}) + log_files = optional(list(object({ + log_group_name = string + prefix_log_group = bool + file_path = string + log_stream_name = string + log_class = optional(string, "STANDARD") + })), null) + environment_variables = optional(map(string), {}) + iam = optional(object({ + resource_arns = optional(object({ + images = optional(list(string), null) + }), {}) + additional_policy_json = optional(object({ + scale_up = optional(string, null) + }), {}) + managed_policies = optional(object({ + scale_up = optional(object({ + arn = string + }), null) + pool = optional(object({ + arn = string + }), null) + }), {}) + }), {}) + }) + + nullable = false +} + +variable "runner" { + description = <<-EOT + Resolved runner settings consumed by the Lambda MicroVM compute provider. + + - `os`: Runner operating system. Lambda MicroVM requires `linux`. + - `architecture`: Runner distribution architecture. Lambda MicroVM requires `arm64`. + - `name_prefix`: Prefix added to registered runner names. + - `run_as_root`: Runs the runner service as root. + - `run_as`: Operating-system user used when `run_as_root` is false. + - `hooks.job_started`: Script installed as the runner job-started hook. + - `hooks.job_completed`: Script installed as the runner job-completed hook. + - `iam.role.arn`: Resolved runner-role ARN used as the MicroVM execution role and referenced by provider policies. + - `iam.role.name`: Resolved runner-role name used by provider resources. + - `iam.role.managed`: Whether runner-config manages the resolved runner role. Callers own an external role and must grant it `ssm:GetParameter` on the lane's `microvm-metadata/*.tags` and `enable_cloudwatch` parameters, `ssm:GetParameter` and `ssm:DeleteParameter` on the lane token path, plus `logs:CreateLogStream` and `logs:PutLogEvents` on the provider-managed runtime log group. When the CloudWatch agent is enabled, it also needs `ssm:GetParameter` on `cloudwatch_agent_config_runner` and stream access to the configured runner log groups. + - `iam.managed_policy_arns`: Common managed-policy ARNs returned with the provider-specific runner policies for attachment by runner-config. + - `iam.path`: IAM path available to provider-managed IAM resources. Null derives the path from `prefix`. + EOT + type = object({ + os = optional(string, "linux") + architecture = optional(string, "arm64") + name_prefix = optional(string, "") + run_as_root = optional(bool, false) + run_as = optional(string, "ec2-user") + hooks = optional(object({ + job_started = optional(string, "") + job_completed = optional(string, "") + }), {}) + iam = object({ + role = object({ + arn = string + name = string + managed = optional(bool, true) + }) + managed_policy_arns = optional(map(string), {}) + path = optional(string, null) + }) + }) + + nullable = false +} + +# tflint-ignore: terraform_unused_declarations +variable "github" { + description = <<-EOT + GitHub Enterprise Server settings available to compute-provider bootstrap data. + + - `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com. + - `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server. + EOT + type = object({ + enterprise_server = optional(object({ + url = optional(string, null) + ssl_verify = optional(bool, true) + }), {}) + }) + default = {} + nullable = false +} + +# tflint-ignore: terraform_unused_declarations +variable "ssm" { + description = <<-EOT + Parameter Store paths and tag scopes available to compute-provider bootstrap resources. + + - `paths.root`: Root Parameter Store path for the runner configuration. + - `paths.tokens`: Path segment used for registration tokens and just-in-time configuration. + - `paths.config`: Path segment used for persistent runner and provider configuration. MicroVM control-plane metadata is stored under its `microvm-metadata` child prefix. + - `tags`: Shared SSM tags that override module-level `tags`. + - `parameters.tags`: Parameter-specific tags that override module-level and shared SSM tags. + EOT + type = object({ + paths = object({ + root = string + tokens = string + config = string + }) + tags = optional(map(string), {}) + parameters = optional(object({ + tags = optional(map(string), {}) + }), {}) + }) + + nullable = false +} + +variable "observability" { + description = <<-EOT + Provider-neutral observability settings applied to the provider-managed MicroVM runtime log group. + + - `logs.retention_in_days`: CloudWatch Logs retention period. + - `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt the log group. + - `logs.class`: CloudWatch log-group class. + - `logs.tags`: Tags merged after module-level tags on the log group. + EOT + type = object({ + logs = optional(object({ + retention_in_days = optional(number, 180) + kms_key_id = optional(string, null) + class = optional(string, "STANDARD") + tags = optional(map(string), {}) + }), {}) + }) + default = {} + nullable = false +} diff --git a/modules/compute-providers/aws/microvm/versions.tf b/modules/compute-providers/aws/microvm/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/compute-providers/aws/microvm/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md index 21fc8f441b..007003094b 100644 --- a/modules/multi-runner/README.md +++ b/modules/multi-runner/README.md @@ -4,6 +4,8 @@ This module creates many runners with one or more GitHub Apps. The module utilizes the internal modules and deploys parts of the stack for each runner defined. +Terraform 1.4 or later is required. Terraform 1.3 and earlier are no longer supported by this module. + ### GitHub App round-robin To distribute GitHub API rate limit usage, this module supports configuring multiple GitHub Apps via the `additional_github_apps` variable. The control-plane lambdas (scale-up, scale-down, pool, job-retry) randomly select an app for each API call, spreading the load across all configured apps. @@ -99,7 +101,7 @@ module "multi-runner" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | +| [terraform](#requirement\_terraform) | >= 1.4 | | [aws](#requirement\_aws) | >= 6.33 | | [random](#requirement\_random) | ~> 3.0 | @@ -109,6 +111,7 @@ module "multi-runner" { |------|---------| | [aws](#provider\_aws) | >= 6.33 | | [random](#provider\_random) | ~> 3.0 | +| [terraform](#provider\_terraform) | n/a | ## Modules @@ -117,6 +120,7 @@ module "multi-runner" { | [ami\_housekeeper](#module\_ami\_housekeeper) | ../ami-housekeeper | n/a | | [instance\_termination\_watcher](#module\_instance\_termination\_watcher) | ../termination-watcher | n/a | | [runner\_binaries](#module\_runner\_binaries) | ../runner-binaries-syncer | n/a | +| [runner\_configs](#module\_runner\_configs) | ../runner-config | n/a | | [runners](#module\_runners) | ../runners | n/a | | [ssm](#module\_ssm) | ../ssm | n/a | | [webhook](#module\_webhook) | ../webhook | n/a | @@ -130,6 +134,8 @@ module "multi-runner" { | [aws_sqs_queue_policy.build_queue_dlq_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [aws_sqs_queue_policy.build_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [random_string.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [terraform_data.validate_v1](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.validate_v2](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | | [aws_iam_policy_document.deny_insecure_transport](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | ## Inputs @@ -151,9 +157,17 @@ module "multi-runner" { | [enable\_ami\_housekeeper](#input\_enable\_ami\_housekeeper) | Option to disable the lambda to clean up old AMIs. | `bool` | `false` | no | | [enable\_managed\_runner\_security\_group](#input\_enable\_managed\_runner\_security\_group) | Enabling the default managed security group creation. Unmanaged security groups can be specified via `runner_additional_security_group_ids`. | `bool` | `true` | no | | [eventbridge](#input\_eventbridge) | Enable the use of EventBridge by the module. By enabling this feature events will be put on the EventBridge by the webhook instead of directly dispatching to queues for scaling. |
object({
enable = optional(bool, true)
accept_events = optional(list(string), [])
})
| `{}` | no | +| [experimental\_global\_config](#input\_experimental\_global\_config) | Experimental global defaults shared by all runner lanes. |
object({
tags = optional(map(string), {})

roles = optional(object({
path = optional(string, null)
permissions_boundary = optional(string, null)
}), {})

runner = optional(object({
os = optional(string, null)
architecture = optional(string, null)
disable_default_labels = optional(bool, false)
extra_labels = optional(list(string), [])
group_name = optional(string, "Default")
name_prefix = optional(string, "")
run_as_root = optional(bool, false)
run_as = optional(string, "ec2-user")
auto_update_disabled = optional(bool, false)
tags = optional(map(string), {})
hooks = optional(object({
job_started = optional(string, "")
job_completed = optional(string, "")
}), {})
iam = optional(object({
role = optional(object({
arn = string
}), null)
managed_policy_arns = optional(map(string), {})
additional_trust_policy_json = optional(string, null)
path = optional(string, null)
permissions_boundary = optional(string, null)
}), {})
}), {})
})
| `{}` | no | +| [experimental\_global\_config\_compute\_provider](#input\_experimental\_global\_config\_compute\_provider) | Experimental global compute-provider configuration. |
object({
selections = optional(map(object({
namespace = string
type = string
})), null)
aws = optional(object({
ec2 = optional(object({
vpc_id = optional(string, null)
subnet_ids = optional(list(string), null)
managed_security_group_enabled = optional(bool, true)
egress_rules = optional(list(object({
cidr_blocks = list(string)
ipv6_cidr_blocks = list(string)
prefix_list_ids = list(string)
from_port = number
protocol = string
security_groups = list(string)
self = bool
to_port = number
description = string
})), [{
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
prefix_list_ids = null
from_port = 0
protocol = "-1"
security_groups = null
self = null
to_port = 0
description = null
}])
additional_security_group_ids = optional(list(string), [])
cloudwatch_agent = optional(object({
config = optional(string, null)
}), {})
instance_profile_path = optional(string, null)
key_name = optional(string, null)
associate_public_ipv4_address = optional(bool, false)
tags = optional(map(string), {})
ami = optional(object({
housekeeper = optional(object({
enabled = optional(bool, false)
cleanup_config = optional(object({
maxItems = optional(number)
minimumDaysOld = optional(number)
amiFilters = optional(list(object({
Name = string
Values = list(string)
})))
launchTemplateNames = optional(list(string))
ssmParameterNames = optional(list(string))
dryRun = optional(bool)
}), {})
artifact = optional(object({
zip = optional(string, null)
s3 = optional(object({
key = string
object_version = optional(string, null)
}), null)
}), {})
lambda = optional(object({
memory_size = optional(number, 256)
timeout = optional(number, 300)
}), {})
schedule = optional(object({
expression = optional(string, "cron(11 7 * * ? *)")
}), {})
}), {})
}), {})
instance_termination_watcher = optional(object({
enabled = optional(bool, false)
features = optional(object({
runner_deregistration = optional(object({
enabled = optional(bool, true)
}), {})
spot_termination_handler = optional(object({
enabled = optional(bool, true)
}), {})
spot_termination_notification_watcher = optional(object({
enabled = optional(bool, true)
}), {})
}), {})
environment_variables = optional(map(string), {})
artifact = optional(object({
zip = optional(string, null)
s3 = optional(object({
key = string
object_version = optional(string, null)
}), null)
}), {})
lambda = optional(object({
memory_size = optional(number, null)
timeout = optional(number, null)
}), {})
}), {})
runner_binaries = optional(object({
enabled = optional(bool, true)
s3 = optional(object({
encryption = optional(object({
enabled = optional(bool, true)
bucket_key_enabled = optional(bool, null)
sse_algorithm = optional(string, "AES256")
kms_master_key_id = optional(string, null)
}), {})
tags = optional(map(string), {})
versioning = optional(string, "Disabled")
logging = optional(object({
bucket = optional(string, null)
prefix = optional(string, null)
}), {})
}), {})
syncer = optional(object({
artifact = optional(object({
zip = optional(string, null)
s3 = optional(object({
key = string
object_version = optional(string, null)
}), null)
}), {})
lambda = optional(object({
memory_size = optional(number, 256)
timeout = optional(number, 300)
}), {})
schedule = optional(object({
expression = optional(string, "cron(27 * * * ? *)")
state = optional(string, "ENABLED")
}), {})
}), {})
}), {})
}), {})
}), {})
})
| `{}` | no | +| [experimental\_global\_config\_github](#input\_experimental\_global\_config\_github) | Experimental global GitHub configuration. |
object({
app = optional(object({
key_base64 = optional(string)
key_base64_ssm = optional(object({
arn = string
name = string
}))
id = optional(string)
id_ssm = optional(object({
arn = string
name = string
}))
webhook_secret = optional(string)
webhook_secret_ssm = optional(object({
arn = string
name = string
}))
}), null)
additional_apps = optional(list(object({
key_base64 = optional(string)
key_base64_ssm = optional(object({ arn = string, name = string }))
id = optional(string)
id_ssm = optional(object({ arn = string, name = string }))
installation_id = optional(string)
installation_id_ssm = optional(object({ arn = string, name = string }))
})), [])
enterprise_server = optional(object({
url = optional(string, null)
ssl_verify = optional(bool, true)
}), {})
user_agent = optional(string, "github-aws-runners")
})
| `{}` | no | +| [experimental\_global\_config\_lambda](#input\_experimental\_global\_config\_lambda) | Experimental global Lambda configuration. |
object({
artifact = optional(object({
s3 = optional(object({
bucket = optional(string, null)
}), {})
}), {})
runtime = optional(string, "nodejs24.x")
architecture = optional(string, "arm64")
principals = optional(list(object({
type = string
identifiers = list(string)
})), [])
subnet_ids = optional(list(string), [])
security_group_ids = optional(list(string), [])
tags = optional(map(string), {})
role = optional(object({
path = optional(string, null)
permissions_boundary = optional(string, null)
}), {})
})
| `{}` | no | +| [experimental\_global\_config\_observability](#input\_experimental\_global\_config\_observability) | Experimental global observability configuration. |
object({
logs = optional(object({
level = optional(string, "info")
retention_in_days = optional(number, 180)
kms_key_id = optional(string, null)
class = optional(string, "STANDARD")
tags = optional(map(string), {})
}), {})
tracing = optional(object({
mode = optional(string, null)
capture_http_requests = optional(bool, false)
capture_error = optional(bool, false)
}), {})
metrics = optional(object({
enabled = optional(bool, false)
namespace = optional(string, "GitHub Runners")
metric = optional(object({
github_app_rate_limit = optional(object({
enabled = optional(bool, true)
}), {})
job_retry = optional(object({
enabled = optional(bool, true)
}), {})
spot_termination_warning = optional(object({
enabled = optional(bool, true)
}), {})
}), {})
}), {})
})
| `{}` | no | +| [experimental\_global\_config\_orchestration\_provider](#input\_experimental\_global\_config\_orchestration\_provider) | Experimental global orchestration-provider configuration. |
object({
webhook = optional(object({
queue_selection_strategy = optional(string, "first")
eventbridge = optional(object({
enabled = optional(bool, true)
accept_events = optional(list(string), [])
}), {})
matcher_config_parameter_store_tier = optional(string, "Standard")
runner = optional(object({
boot_time_in_minutes = optional(number, 5)
ephemeral = optional(bool, false)
jit_config_enabled = optional(bool, null)
maximum_count = optional(number, null)
}), {})

github = optional(object({
repository_white_list = optional(list(string), [])
}), {})

lambda = optional(object({
artifact = optional(object({
zip = optional(string, null)
s3 = optional(object({
key = string
object_version = optional(string, null)
}), null)
}), {})
scale = optional(object({
up = optional(object({
memory_size = optional(number, 512)
timeout = optional(number, 30)
reserved_concurrent_executions = optional(number, 1)
job_queued_check_enabled = optional(bool, null)
event_source_mapping = optional(object({
batch_size = optional(number, 10)
maximum_batching_window_in_seconds = optional(number, 0)
}), {})
tags = optional(map(string), {})
}), {})
down = optional(object({
memory_size = optional(number, 512)
timeout = optional(number, 60)
schedule_expression = optional(string, "cron(*/5 * * * ? *)")
minimum_running_time_in_minutes = optional(number, null)
idle_config = optional(list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = optional(string, "oldest_first")
})), [])
tags = optional(map(string), {})
}), {})
}), {})
webhook = optional(object({
artifact = optional(object({
zip = optional(string, null)
s3 = optional(object({
key = string
object_version = optional(string, null)
}), null)
}), {})
api_gateway_access_log_settings = optional(object({
destination_arn = string
format = string
}), null)
memory_size = optional(number, 256)
timeout = optional(number, 10)
tags = optional(map(string), {})
}), {})
pool = optional(object({
memory_size = optional(number, 512)
timeout = optional(number, 60)
reserved_concurrent_executions = optional(number, 1)
config = optional(list(object({
schedule_expression = string
schedule_expression_timezone = optional(string)
size = number
})), [])
include_busy_runners = optional(bool, false)
runner_owner = optional(string, null)
tags = optional(map(string), {})
}), {})
}), {})

queue = optional(object({
delay_webhook_event = optional(number, 30)
job_queue_retention_in_seconds = optional(number, 86400)
visibility_timeout_seconds = optional(number, 180)
redrive_build_queue = optional(object({
enabled = optional(bool, false)
maxReceiveCount = optional(number, null)
}), {
enabled = false
maxReceiveCount = null
})
tags = optional(map(string), {})
encryption = optional(object({
kms_data_key_reuse_period_seconds = number
kms_master_key_id = string
sqs_managed_sse_enabled = bool
}), {
kms_data_key_reuse_period_seconds = null
kms_master_key_id = null
sqs_managed_sse_enabled = true
})
}), {})
}), {})
})
| `{}` | no | +| [experimental\_global\_config\_ssm](#input\_experimental\_global\_config\_ssm) | Experimental global SSM configuration. |
object({
paths = optional(object({
root = optional(string, null)
app = optional(string, "app")
webhook = optional(string, "webhook")
tokens = optional(string, "runners/tokens")
config = optional(string, "runners/config")
}), {})
kms_key_id = optional(string, null)
tags = optional(map(string), {})
parameters = optional(object({
tags = optional(map(string), {})
}), {})
housekeeper = optional(object({
schedule_expression = optional(string, "rate(1 day)")
state = optional(string, "ENABLED")
tags = optional(map(string), {})
lambda = optional(object({
artifact = optional(object({
zip = optional(string, null)
s3 = optional(object({
key = string
object_version = optional(string, null)
}), null)
}), {})
memory_size = optional(number, 512)
timeout = optional(number, 60)
}), {})
config = optional(object({
tokenPath = optional(string, null)
minimumDaysOld = optional(number, 1)
dryRun = optional(bool, false)
}), {})
}), {})
})
| `{}` | no | +| [experimental\_multi\_runner\_config](#input\_experimental\_multi\_runner\_config) | Experimental per-runner and per-lane overrides. |
map(object({
tags = optional(map(string), {})

runner = optional(object({
os = optional(string, null)
architecture = optional(string, null)
disable_default_labels = optional(bool, null)
extra_labels = optional(list(string), null)
group_name = optional(string, null)
name_prefix = optional(string, null)
run_as_root = optional(bool, null)
run_as = optional(string, null)
auto_update_disabled = optional(bool, null)
tags = optional(map(string), {})
hooks = optional(object({
job_started = optional(string, null)
job_completed = optional(string, null)
}), {})
iam = optional(object({
role = optional(object({
arn = string
}), null)
managed_policy_arns = optional(map(string), null)
additional_trust_policy_json = optional(string, null)
path = optional(string, null)
permissions_boundary = optional(string, null)
}), {})
}), {})

lambda = optional(object({
runtime = optional(string, null)
architecture = optional(string, null)
subnet_ids = optional(list(string), null)
security_group_ids = optional(list(string), null)
tags = optional(map(string), {})
role = optional(object({
path = optional(string, null)
permissions_boundary = optional(string, null)
}), {})
}), {})

orchestration_provider = object({
webhook = optional(object({
runner = optional(object({
boot_time_in_minutes = optional(number, null)
ephemeral = optional(bool, null)
jit_config_enabled = optional(bool, null)
maximum_count = optional(number, null)
}), {})

github = optional(object({
organization_runners = optional(bool, false)
}), {})

matcherConfig = object({
labelMatchers = list(list(string))
exactMatch = optional(bool, false)
bidirectionalLabelMatch = optional(bool, false)
priority = optional(number, 999)
dynamic_labels_enabled = optional(bool, false)
awsDynamicLabelsPolicy = optional(object({
blocked_keys = optional(list(string), [])
restricted_keys = optional(map(object({
allowed = optional(list(string), [])
denied = optional(list(string), [])
max = optional(string, null)
})), {})
}), null)
})

queue = optional(object({
delay_webhook_event = optional(number, null)
job_queue_retention_in_seconds = optional(number, null)
visibility_timeout_seconds = optional(number, null)
redrive_build_queue = optional(object({
enabled = optional(bool, null)
maxReceiveCount = optional(number, null)
}), null)
tags = optional(map(string), {})
}), {})

lambda = optional(object({
scale = optional(object({
up = optional(object({
memory_size = optional(number, null)
timeout = optional(number, null)
reserved_concurrent_executions = optional(number, null)
job_queued_check_enabled = optional(bool, null)
event_source_mapping = optional(object({
batch_size = optional(number, null)
maximum_batching_window_in_seconds = optional(number, null)
}), {})
tags = optional(map(string), {})
}), {})
down = optional(object({
memory_size = optional(number, null)
timeout = optional(number, null)
schedule_expression = optional(string, null)
minimum_running_time_in_minutes = optional(number, null)
idle_config = optional(list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = optional(string, "oldest_first")
})), null)
tags = optional(map(string), {})
}), {})
}), {})
pool = optional(object({
memory_size = optional(number, null)
timeout = optional(number, null)
reserved_concurrent_executions = optional(number, null)
config = optional(list(object({
schedule_expression = string
schedule_expression_timezone = optional(string)
size = number
})), null)
include_busy_runners = optional(bool, null)
runner_owner = optional(string, null)
tags = optional(map(string), {})
}), {})
}), {})

job_retry = optional(object({
enabled = optional(bool, false)
delay_in_seconds = optional(number, 300)
delay_backoff = optional(number, 2)
max_attempts = optional(number, 1)
tags = optional(map(string), {})
lambda = optional(object({
memory_size = optional(number, 256)
reserved_concurrent_executions = optional(number, 1)
timeout = optional(number, 30)
}), {})
}), {})

}), null)

})

ssm = optional(object({
paths = optional(object({
root = optional(string, null)
tokens = optional(string, null)
config = optional(string, null)
}), {})
tags = optional(map(string), {})
parameters = optional(object({
tags = optional(map(string), {})
}), {})
housekeeper = optional(object({
schedule_expression = optional(string, null)
state = optional(string, null)
tags = optional(map(string), {})
lambda = optional(object({
artifact = optional(object({
zip = optional(string, null)
s3 = optional(object({
key = string
object_version = optional(string, null)
}), null)
}), {})
memory_size = optional(number, null)
timeout = optional(number, null)
}), {})
config = optional(object({
tokenPath = optional(string, null)
minimumDaysOld = optional(number, null)
dryRun = optional(bool, null)
}), {})
}), {})
}), {})

observability = optional(object({
logs = optional(object({
level = optional(string, null)
retention_in_days = optional(number, null)
kms_key_id = optional(string, null)
class = optional(string, null)
tags = optional(map(string), {})
}), {})
tracing = optional(object({
mode = optional(string, null)
capture_http_requests = optional(bool, null)
capture_error = optional(bool, null)
}), {})
metrics = optional(object({
enabled = optional(bool, null)
namespace = optional(string, null)
metric = optional(object({
github_app_rate_limit = optional(object({
enabled = optional(bool, null)
}), {})
job_retry = optional(object({
enabled = optional(bool, null)
}), {})
spot_termination_warning = optional(object({
enabled = optional(bool, null)
}), {})
}), {})
}), {})
}), {})

compute_provider = object({
aws = optional(object({
ec2 = optional(object({
metadata_options = optional(object({
instance_metadata_tags = optional(string, "enabled")
http_endpoint = optional(string, "enabled")
http_tokens = optional(string, "required")
http_put_response_hop_limit = optional(number, 1)
}), {})
ami = optional(object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter = optional(object({
arn = string
}), null)
kms_key = optional(object({
arn = string
}), null)
}), null)
block_device_mappings = optional(list(object({
delete_on_termination = optional(bool, true)
device_name = optional(string, "/dev/xvda")
encrypted = optional(bool, true)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = number
volume_type = optional(string, "gp3")
})), [{
volume_size = 30
}])
create_service_linked_role_spot = optional(bool, false)
credit_specification = optional(string, null)
ebs_optimized = optional(bool, false)
cloudwatch_agent = optional(object({
enabled = optional(bool, true)
config = optional(string, null)
}), {})
binaries_syncer = optional(object({
enabled = optional(bool, null)
}), {})
detailed_monitoring_enabled = optional(bool, false)
ssm_enabled = optional(bool, false)
user_data = optional(object({
enabled = optional(bool, true)
template = optional(string, null)
content = optional(string, null)
pre_install = optional(string, "")
post_install = optional(string, "")
debug_logging_enabled = optional(bool, false)
}), {})
instance_allocation_strategy = optional(string, "lowest-price")
instance_max_spot_price = optional(string, null)
instance_target_capacity_type = optional(string, "spot")
instance_type_priorities = optional(map(number), null)
instance_types = list(string)
additional_security_group_ids = optional(list(string), null)
managed_security_group_enabled = optional(bool, null)
egress_rules = optional(list(object({
cidr_blocks = list(string)
ipv6_cidr_blocks = list(string)
prefix_list_ids = list(string)
from_port = number
protocol = string
security_groups = list(string)
self = bool
to_port = number
description = string
})), null)
instance_profile_path = optional(string, null)
key_name = optional(string, null)
associate_public_ipv4_address = optional(bool, null)
instance_profile = optional(object({
name = string
}), null)
on_demand_failover_for_errors = optional(list(string), [])
scale_errors = optional(list(string), [
"UnfulfillableCapacity",
"MaxSpotInstanceCountExceeded",
"TargetCapacityLimitExceededException",
"RequestLimitExceeded",
"ResourceLimitExceeded",
"MaxSpotInstanceCountExceeded",
"MaxSpotFleetRequestCountExceeded",
"InsufficientInstanceCapacity",
"InsufficientCapacityOnHost",
])
subnet_ids = optional(list(string), null)
vpc_id = optional(string, null)
cpu_options = optional(object({
core_count = optional(number)
threads_per_core = optional(number)
amd_sev_snp = optional(string)
nested_virtualization = optional(string)
}), null)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_id = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
spread_domain = optional(string)
tenancy = optional(string)
partition_number = optional(number)
}), null)
license_specifications = optional(list(object({
license_configuration_arn = string
})), [])
use_dedicated_host = optional(bool, false)
log_files = optional(list(object({
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
log_class = optional(string, "STANDARD")
})), null)
tags = optional(map(string), {})
}), null)
}), {})
})

}))
| `{}` | no | | [ghes\_ssl\_verify](#input\_ghes\_ssl\_verify) | GitHub Enterprise SSL verification. Set to 'false' when custom certificate (chains) is used for GitHub Enterprise Server (insecure). | `bool` | `true` | no | | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. Example: https://github.internal.co - DO NOT SET IF USING PUBLIC GITHUB. .However if you are using GitHub Enterprise Cloud with data-residency (ghe.com), set the endpoint here. Example - https://companyname.ghe.com\| | `string` | `null` | no | -| [github\_app](#input\_github\_app) | GitHub app parameters, see your github app.
You can optionally create the SSM parameters yourself and provide the ARN and name here, through the `*_ssm` attributes.
If you chose to provide the configuration values directly here,
please ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`).
Note: the provided SSM parameters arn and name have a precedence over the actual value (i.e `key_base64_ssm` has a precedence over `key_base64` etc). |
object({
key_base64 = optional(string)
key_base64_ssm = optional(object({
arn = string
name = string
}))
id = optional(string)
id_ssm = optional(object({
arn = string
name = string
}))
webhook_secret = optional(string)
webhook_secret_ssm = optional(object({
arn = string
name = string
}))
})
| n/a | yes | +| [github\_app](#input\_github\_app) | GitHub app parameters for the stable v1 interface, see your github app.
Omit this value when using the experimental v2 interface and provide the
app through `experimental_global_config_github` instead.
You can optionally create the SSM parameters yourself and provide the ARN and name here, through the `*_ssm` attributes.
If you chose to provide the configuration values directly here,
please ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`).
Note: the provided SSM parameters arn and name have a precedence over the actual value (i.e `key_base64_ssm` has a precedence over `key_base64` etc). |
object({
key_base64 = optional(string)
key_base64_ssm = optional(object({
arn = string
name = string
}))
id = optional(string)
id_ssm = optional(object({
arn = string
name = string
}))
webhook_secret = optional(string)
webhook_secret_ssm = optional(object({
arn = string
name = string
}))
})
| `{}` | no | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. The following attributes are supported: `instance_profile_name` overrides the instance profile name used in the launch template. `runner_role_arn` overrides the IAM role ARN used for the runner instances. |
object({
override_instance_profile = optional(bool, null)
instance_profile_name = optional(string, null)
override_runner_role = optional(bool, null)
runner_role_arn = optional(string, null)
})
|
{
"instance_profile_name": null,
"override_instance_profile": false,
"override_runner_role": false,
"runner_role_arn": null
}
| no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | | [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the spot termination watcher lambda function. This feature is Beta, changes will not trigger a major release as long in beta.

`enable`: Enable or disable the spot termination watcher.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`environment_variables`: Additional environment variables to merge into the Lambda configuration.
`memory_size`: Memory size limit in MB of the lambda.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`timeout`: Time out of the lambda in seconds.
`zip`: File location of the lambda zip file. |
object({
enable = optional(bool, false)
features = optional(object({
enable_spot_termination_handler = optional(bool, true)
enable_spot_termination_notification_watcher = optional(bool, true)
}), {})
enable_runner_deregistration = optional(bool, true)
environment_variables = optional(map(string), {})
memory_size = optional(number, null)
s3_key = optional(string, null)
s3_object_version = optional(string, null)
timeout = optional(number, null)
zip = optional(string, null)
})
| `{}` | no | @@ -174,7 +188,7 @@ module "multi-runner" { | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | | [matcher\_config\_parameter\_store\_tier](#input\_matcher\_config\_parameter\_store\_tier) | The tier of the parameter store for the matcher configuration. Valid values are `Standard`, and `Advanced`. | `string` | `"Standard"` | no | | [metrics](#input\_metrics) | Configuration for metrics created by the module, by default metrics are disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. |
object({
enable = optional(bool, false)
namespace = optional(string, "GitHub Runners")
metric = optional(object({
enable_github_app_rate_limit = optional(bool, true)
enable_job_retry = optional(bool, true)
enable_spot_termination_warning = optional(bool, true)
}), {})
})
| `{}` | no | -| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
instance\_type\_priorities: "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
awsDynamicLabelsPolicy: "Optional AWS dynamic label policy evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose provider dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} |
map(object({
runner_config = object({
runner_os = string
runner_architecture = string
runner_metadata_options = optional(map(any), {
instance_metadata_tags = "enabled"
http_endpoint = "enabled"
http_tokens = "required"
http_put_response_hop_limit = 1
})
ami = optional(object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter_arn = optional(string, null)
kms_key_arn = optional(string, null)
}), null)
create_service_linked_role_spot = optional(bool, false)
credit_specification = optional(string, null)
delay_webhook_event = optional(number, 30)
disable_runner_autoupdate = optional(bool, false)
ebs_optimized = optional(bool, false)
enable_ephemeral_runners = optional(bool, false)
enable_job_queued_check = optional(bool, null)
enable_on_demand_failover_for_errors = optional(list(string), [])
scale_errors = optional(list(string), [
"UnfulfillableCapacity",
"MaxSpotInstanceCountExceeded",
"TargetCapacityLimitExceededException",
"RequestLimitExceeded",
"ResourceLimitExceeded",
"MaxSpotInstanceCountExceeded",
"MaxSpotFleetRequestCountExceeded",
"InsufficientInstanceCapacity",
"InsufficientCapacityOnHost",
])
enable_organization_runners = optional(bool, false)
enable_runner_binaries_syncer = optional(bool, true)
enable_ssm_on_runners = optional(bool, false)
enable_userdata = optional(bool, true)
instance_allocation_strategy = optional(string, "lowest-price")
instance_type_priorities = optional(map(number), null)
instance_max_spot_price = optional(string, null)
instance_target_capacity_type = optional(string, "spot")
instance_types = list(string)
job_queue_retention_in_seconds = optional(number, 86400)
minimum_running_time_in_minutes = optional(number, null)
pool_runner_owner = optional(string, null)
runner_as_root = optional(bool, false)
runner_boot_time_in_minutes = optional(number, 5)
runner_disable_default_labels = optional(bool, false)
runner_extra_labels = optional(list(string), [])
runner_group_name = optional(string, "Default")
runner_name_prefix = optional(string, "")
runner_run_as = optional(string, "ec2-user")
runners_maximum_count = number
runner_additional_security_group_ids = optional(list(string), [])
scale_down_schedule_expression = optional(string, "cron(*/5 * * * ? *)")
scale_up_reserved_concurrent_executions = optional(number, 1)
lambda_event_source_mapping_batch_size = optional(number, null)
lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
userdata_template = optional(string, null)
userdata_content = optional(string, null)
enable_jit_config = optional(bool, null)
enable_runner_detailed_monitoring = optional(bool, false)
enable_cloudwatch_agent = optional(bool, true)
cloudwatch_config = optional(string, null)
userdata_pre_install = optional(string, "")
userdata_post_install = optional(string, "")
runner_hook_job_started = optional(string, "")
runner_hook_job_completed = optional(string, "")
runner_ec2_tags = optional(map(string), {})
runner_iam_role_managed_policy_arns = optional(list(string), [])
vpc_id = optional(string, null)
subnet_ids = optional(list(string), null)
idle_config = optional(list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = optional(string, "oldest_first")
})), [])
cpu_options = optional(object({
core_count = optional(number)
threads_per_core = optional(number)
amd_sev_snp = optional(string)
nested_virtualization = optional(string)
}), null)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_id = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
spread_domain = optional(string)
tenancy = optional(string)
partition_number = optional(number)
}), null)
license_specifications = optional(list(object({
license_configuration_arn = string
})), [])
use_dedicated_host = optional(bool, false)
runner_log_files = optional(list(object({
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
log_class = optional(string, "STANDARD")
})), null)
block_device_mappings = optional(list(object({
delete_on_termination = optional(bool, true)
device_name = optional(string, "/dev/xvda")
encrypted = optional(bool, true)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = number
volume_type = optional(string, "gp3")
})), [{
volume_size = 30
}])
pool_config = optional(list(object({
schedule_expression = string
schedule_expression_timezone = optional(string)
size = number
})), [])
job_retry = optional(object({
enable = optional(bool, false)
delay_in_seconds = optional(number, 300)
delay_backoff = optional(number, 2)
lambda_memory_size = optional(number, 256)
lambda_timeout = optional(number, 30)
max_attempts = optional(number, 1)
}), {})
iam_overrides = optional(object({
override_instance_profile = optional(bool, null)
instance_profile_name = optional(string, null)
override_runner_role = optional(bool, null)
runner_role_arn = optional(string, null)
}), {
override_instance_profile = false
instance_profile_name = null
override_runner_role = false
runner_role_arn = null
})
})
matcherConfig = object({
labelMatchers = list(list(string))
exactMatch = optional(bool, false)
bidirectionalLabelMatch = optional(bool, false)
priority = optional(number, 999)
enableDynamicLabels = optional(bool, false)
awsDynamicLabelsPolicy = optional(any, null)
})
redrive_build_queue = optional(object({
enabled = bool
maxReceiveCount = number
}), {
enabled = false
maxReceiveCount = null
})
}))
| n/a | yes | +| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
instance\_type\_priorities: "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
awsDynamicLabelsPolicy: "Optional AWS dynamic label policy evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose provider dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} |
map(object({
runner_config = object({
runner_os = string
runner_architecture = string
runner_metadata_options = optional(map(any), {
instance_metadata_tags = "enabled"
http_endpoint = "enabled"
http_tokens = "required"
http_put_response_hop_limit = 1
})
ami = optional(object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter_arn = optional(string, null)
kms_key_arn = optional(string, null)
}), null)
create_service_linked_role_spot = optional(bool, false)
credit_specification = optional(string, null)
delay_webhook_event = optional(number, 30)
disable_runner_autoupdate = optional(bool, false)
ebs_optimized = optional(bool, false)
enable_ephemeral_runners = optional(bool, false)
enable_job_queued_check = optional(bool, null)
enable_on_demand_failover_for_errors = optional(list(string), [])
scale_errors = optional(list(string), [
"UnfulfillableCapacity",
"MaxSpotInstanceCountExceeded",
"TargetCapacityLimitExceededException",
"RequestLimitExceeded",
"ResourceLimitExceeded",
"MaxSpotInstanceCountExceeded",
"MaxSpotFleetRequestCountExceeded",
"InsufficientInstanceCapacity",
"InsufficientCapacityOnHost",
])
enable_organization_runners = optional(bool, false)
enable_runner_binaries_syncer = optional(bool, true)
enable_ssm_on_runners = optional(bool, false)
enable_userdata = optional(bool, true)
instance_allocation_strategy = optional(string, "lowest-price")
instance_type_priorities = optional(map(number), null)
instance_max_spot_price = optional(string, null)
instance_target_capacity_type = optional(string, "spot")
instance_types = list(string)
job_queue_retention_in_seconds = optional(number, 86400)
minimum_running_time_in_minutes = optional(number, null)
pool_runner_owner = optional(string, null)
runner_as_root = optional(bool, false)
runner_boot_time_in_minutes = optional(number, 5)
runner_disable_default_labels = optional(bool, false)
runner_extra_labels = optional(list(string), [])
runner_group_name = optional(string, "Default")
runner_name_prefix = optional(string, "")
runner_run_as = optional(string, "ec2-user")
runners_maximum_count = number
runner_additional_security_group_ids = optional(list(string), [])
scale_down_schedule_expression = optional(string, "cron(*/5 * * * ? *)")
scale_up_reserved_concurrent_executions = optional(number, 1)
lambda_event_source_mapping_batch_size = optional(number, null)
lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
userdata_template = optional(string, null)
userdata_content = optional(string, null)
enable_jit_config = optional(bool, null)
enable_runner_detailed_monitoring = optional(bool, false)
enable_cloudwatch_agent = optional(bool, true)
cloudwatch_config = optional(string, null)
userdata_pre_install = optional(string, "")
userdata_post_install = optional(string, "")
runner_hook_job_started = optional(string, "")
runner_hook_job_completed = optional(string, "")
runner_ec2_tags = optional(map(string), {})
runner_iam_role_managed_policy_arns = optional(list(string), [])
vpc_id = optional(string, null)
subnet_ids = optional(list(string), null)
idle_config = optional(list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = optional(string, "oldest_first")
})), [])
cpu_options = optional(object({
core_count = optional(number)
threads_per_core = optional(number)
amd_sev_snp = optional(string)
nested_virtualization = optional(string)
}), null)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_id = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
spread_domain = optional(string)
tenancy = optional(string)
partition_number = optional(number)
}), null)
license_specifications = optional(list(object({
license_configuration_arn = string
})), [])
use_dedicated_host = optional(bool, false)
runner_log_files = optional(list(object({
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
log_class = optional(string, "STANDARD")
})), null)
block_device_mappings = optional(list(object({
delete_on_termination = optional(bool, true)
device_name = optional(string, "/dev/xvda")
encrypted = optional(bool, true)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = number
volume_type = optional(string, "gp3")
})), [{
volume_size = 30
}])
pool_config = optional(list(object({
schedule_expression = string
schedule_expression_timezone = optional(string)
size = number
})), [])
job_retry = optional(object({
enable = optional(bool, false)
delay_in_seconds = optional(number, 300)
delay_backoff = optional(number, 2)
lambda_memory_size = optional(number, 256)
lambda_timeout = optional(number, 30)
max_attempts = optional(number, 1)
}), {})
iam_overrides = optional(object({
override_instance_profile = optional(bool, null)
instance_profile_name = optional(string, null)
override_runner_role = optional(bool, null)
runner_role_arn = optional(string, null)
}), {
override_instance_profile = false
instance_profile_name = null
override_runner_role = false
runner_role_arn = null
})
})
matcherConfig = object({
labelMatchers = list(list(string))
exactMatch = optional(bool, false)
bidirectionalLabelMatch = optional(bool, false)
priority = optional(number, 999)
enableDynamicLabels = optional(bool, false)
awsDynamicLabelsPolicy = optional(any, null)
})
redrive_build_queue = optional(object({
enabled = bool
maxReceiveCount = number
}), {
enabled = false
maxReceiveCount = null
})
}))
| `{}` | no | | [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no | | [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no | | [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no | @@ -202,13 +216,13 @@ module "multi-runner" { | [scale\_up\_lambda\_memory\_size](#input\_scale\_up\_lambda\_memory\_size) | Memory size limit in MB for scale\_up lambda. | `number` | `512` | no | | [ssm\_paths](#input\_ssm\_paths) | The root path used in SSM to store configuration and secrets. |
object({
root = optional(string, "github-action-runners")
app = optional(string, "app")
runners = optional(string, "runners")
webhook = optional(string, "webhook")
})
| `{}` | no | | [state\_event\_rule\_binaries\_syncer](#input\_state\_event\_rule\_binaries\_syncer) | Option to disable EventBridge Lambda trigger for the binary syncer, useful to stop automatic updates of binary distribution | `string` | `"ENABLED"` | no | -| [subnet\_ids](#input\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | n/a | yes | +| [subnet\_ids](#input\_subnet\_ids) | List of subnets in which stable v1 action runners will be launched. Omit when using the experimental v2 interface. | `list(string)` | `null` | no | | [syncer\_lambda\_s3\_key](#input\_syncer\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `string` | `null` | no | | [syncer\_lambda\_s3\_object\_version](#input\_syncer\_lambda\_s3\_object\_version) | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `string` | `null` | no | | [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | | [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. |
object({
mode = optional(string, null)
capture_http_requests = optional(bool, false)
capture_error = optional(bool, false)
})
| `{}` | no | | [user\_agent](#input\_user\_agent) | User agent used for API calls by lambda functions. | `string` | `"github-aws-runners"` | no | -| [vpc\_id](#input\_vpc\_id) | The VPC for security groups of the action runners. | `string` | n/a | yes | +| [vpc\_id](#input\_vpc\_id) | The VPC for security groups of stable v1 action runners. Omit when using the experimental v2 interface. | `string` | `null` | no | | [webhook\_lambda\_apigateway\_access\_log\_settings](#input\_webhook\_lambda\_apigateway\_access\_log\_settings) | Access log settings for webhook API gateway. |
object({
destination_arn = string
format = string
})
| `null` | no | | [webhook\_lambda\_memory\_size](#input\_webhook\_lambda\_memory\_size) | Memory size limit in MB for webhook lambda. | `number` | `256` | no | | [webhook\_lambda\_s3\_key](#input\_webhook\_lambda\_s3\_key) | S3 key for webhook lambda function. Required if using S3 bucket to specify lambdas. | `string` | `null` | no | @@ -224,6 +238,7 @@ module "multi-runner" { | [instance\_termination\_handler](#output\_instance\_termination\_handler) | n/a | | [instance\_termination\_watcher](#output\_instance\_termination\_watcher) | n/a | | [runners\_map](#output\_runners\_map) | n/a | +| [runners\_map\_v2](#output\_runners\_map\_v2) | n/a | | [ssm\_parameters](#output\_ssm\_parameters) | n/a | | [webhook](#output\_webhook) | n/a | diff --git a/modules/multi-runner/ami-housekeeper.tf b/modules/multi-runner/ami-housekeeper.tf index 385e6010c9..88d82b7f0b 100644 --- a/modules/multi-runner/ami-housekeeper.tf +++ b/modules/multi-runner/ami-housekeeper.tf @@ -1,35 +1,35 @@ module "ami_housekeeper" { - count = var.enable_ami_housekeeper ? 1 : 0 + count = try(local.effective_config.compute_provider.aws.ec2.ami.housekeeper.enabled, false) ? 1 : 0 source = "../ami-housekeeper" prefix = var.prefix tags = local.tags aws_partition = var.aws_partition - lambda_zip = var.ami_housekeeper_lambda_zip - lambda_s3_bucket = var.lambda_s3_bucket - lambda_s3_key = var.ami_housekeeper_lambda_s3_key - lambda_s3_object_version = var.ami_housekeeper_lambda_s3_object_version + lambda_zip = try(local.effective_config.compute_provider.aws.ec2.ami.housekeeper.artifact.zip, null) + lambda_s3_bucket = try(local.effective_config.lambda.artifact.s3.bucket, null) + lambda_s3_key = try(local.effective_config.compute_provider.aws.ec2.ami.housekeeper.artifact.s3.key, null) + lambda_s3_object_version = try(local.effective_config.compute_provider.aws.ec2.ami.housekeeper.artifact.s3.object_version, null) - lambda_architecture = var.lambda_architecture - lambda_principals = var.lambda_principals - lambda_runtime = var.lambda_runtime - lambda_security_group_ids = var.lambda_security_group_ids - lambda_subnet_ids = var.lambda_subnet_ids - lambda_memory_size = var.ami_housekeeper_lambda_memory_size - lambda_timeout = var.ami_housekeeper_lambda_timeout - lambda_tags = var.lambda_tags - tracing_config = var.tracing_config + lambda_architecture = local.effective_config.lambda.architecture + lambda_principals = local.effective_config.lambda.principals + lambda_runtime = local.effective_config.lambda.runtime + lambda_security_group_ids = local.effective_config.lambda.security_group_ids + lambda_subnet_ids = local.effective_config.lambda.subnet_ids + lambda_memory_size = local.effective_config.compute_provider.aws.ec2.ami.housekeeper.lambda.memory_size + lambda_timeout = local.effective_config.compute_provider.aws.ec2.ami.housekeeper.lambda.timeout + lambda_tags = local.effective_config.lambda.tags + tracing_config = local.effective_config.observability.tracing - logging_retention_in_days = var.logging_retention_in_days - logging_kms_key_id = var.logging_kms_key_id - log_class = var.log_class - log_level = var.log_level + logging_retention_in_days = local.effective_config.observability.logs.retention_in_days + logging_kms_key_id = local.effective_config.observability.logs.kms_key_id + log_class = local.effective_config.observability.logs.class + log_level = local.effective_config.observability.logs.level - role_path = var.role_path - role_permissions_boundary = var.role_permissions_boundary + role_path = local.effective_config.roles.path + role_permissions_boundary = local.effective_config.roles.permissions_boundary - cleanup_config = var.ami_housekeeper_cleanup_config - lambda_schedule_expression = var.ami_housekeeper_lambda_schedule_expression + cleanup_config = local.effective_config.compute_provider.aws.ec2.ami.housekeeper.cleanup_config + lambda_schedule_expression = local.effective_config.compute_provider.aws.ec2.ami.housekeeper.schedule.expression } diff --git a/modules/multi-runner/config.experimental.effective.tf b/modules/multi-runner/config.experimental.effective.tf new file mode 100644 index 0000000000..28831a0033 --- /dev/null +++ b/modules/multi-runner/config.experimental.effective.tf @@ -0,0 +1,59 @@ +# Assemble the resource-ready configuration after runner-binary discovery. +locals { + effective_config = merge(local.resolved_config, { + multi_runner_config = { + for k, v in local.resolved_config.multi_runner_config : k => merge(v, { + runner = merge(v.runner, { + labels = sort(setunion( + v.runner.disable_default_labels ? [] : compact([ + "self-hosted", + v.runner.os, + v.runner.architecture, + ]), + v.orchestration_provider.webhook == null ? [] : flatten(v.orchestration_provider.webhook.matcherConfig.labelMatchers), + compact(v.runner.extra_labels), + )) + }) + + github = { + enterprise_server = local.normalized_config.github.enterprise_server + user_agent = local.normalized_config.github.user_agent + } + + lambda = merge(v.lambda, { + artifact = local.normalized_config.lambda.artifact + principals = local.normalized_config.lambda.principals + }) + + orchestration_provider = { + webhook = v.orchestration_provider.webhook == null ? null : merge(v.orchestration_provider.webhook, { + queue = merge(v.orchestration_provider.webhook.queue, { + kms_key_id = local.normalized_config.orchestration_provider.webhook.queue.encryption.kms_master_key_id + }) + + lambda = merge(v.orchestration_provider.webhook.lambda, { + artifact = local.normalized_config.orchestration_provider.webhook.lambda.artifact + }) + }) + } + + ssm = merge(v.ssm, { + kms_key_id = local.normalized_config.ssm.kms_key_id + }) + + compute_provider = merge(v.compute_provider, { + aws = merge(v.compute_provider.aws, { + ec2 = v.compute_provider.aws.ec2 == null ? null : merge(v.compute_provider.aws.ec2, { + binaries_syncer = merge(v.compute_provider.aws.ec2.binaries_syncer, { + s3 = v.compute_provider.aws.ec2.binaries_syncer.enabled ? try( + local.runner_binaries_by_os_and_arch_map["${v.runner.os}_${v.runner.architecture}"], + null, + ) : null + }) + }) + }) + }) + }) + } + }) +} diff --git a/modules/multi-runner/config.experimental.resolved.tf b/modules/multi-runner/config.experimental.resolved.tf new file mode 100644 index 0000000000..0b55f271ca --- /dev/null +++ b/modules/multi-runner/config.experimental.resolved.tf @@ -0,0 +1,475 @@ +# Project stable v1 inputs into the experimental schema, resolve every runner +# configuration against the experimental global defaults, and assemble the +# resource-ready configuration consumed by the multi-runner resources. +locals { + # Reassemble the split experimental inputs into the canonical shape consumed + # by the translation and precedence logic below. + experimental = { + tags = var.experimental_global_config.tags + roles = var.experimental_global_config.roles + runner = var.experimental_global_config.runner + github = var.experimental_global_config_github + lambda = var.experimental_global_config_lambda + orchestration_provider = var.experimental_global_config_orchestration_provider + ssm = var.experimental_global_config_ssm + observability = var.experimental_global_config_observability + compute_provider = var.experimental_global_config_compute_provider + multi_runner_config = var.experimental_multi_runner_config + } + + stable_to_experimental = { + tags = local.stable_to_experimental_tags + roles = local.stable_to_experimental_roles + runner = local.stable_to_experimental_runner + github = local.stable_to_experimental_github + lambda = local.stable_to_experimental_lambda + orchestration_provider = local.stable_to_experimental_orchestration_provider + ssm = local.stable_to_experimental_ssm + observability = local.stable_to_experimental_observability + compute_provider = local.stable_to_experimental_compute_provider + multi_runner_config = local.stable_to_experimental_multi_runner_config + } + + # A non-empty experimental map selects v2 normalization. + use_v2_config = length(var.experimental_multi_runner_config) > 0 + + normalized_config = local.use_v2_config ? local.experimental : local.stable_to_experimental +} + +locals { + # Resolve each lane against the translated global configuration. This stage + # is used by runner-binary discovery and must not depend on its outputs. + resolved_config = merge(local.normalized_config, { + multi_runner_config = { + for k, v in local.normalized_config.multi_runner_config : k => merge(v, { + tags = merge(local.normalized_config.tags, v.tags) + + runner = merge(v.runner, { + os = try(coalesce( + v.runner.os, + local.normalized_config.runner.os, + ), null) + architecture = try(coalesce( + v.runner.architecture, + local.normalized_config.runner.architecture, + ), null) + disable_default_labels = coalesce( + v.runner.disable_default_labels, + local.normalized_config.runner.disable_default_labels, + ) + extra_labels = sort(distinct(concat( + try(flatten(v.orchestration_provider.webhook.matcherConfig.labelMatchers), []), + coalesce( + v.runner.extra_labels, + local.normalized_config.runner.extra_labels, + ), + ))) + group_name = coalesce( + v.runner.group_name, + local.normalized_config.runner.group_name, + ) + name_prefix = v.runner.name_prefix != null ? v.runner.name_prefix : local.normalized_config.runner.name_prefix + run_as_root = coalesce( + v.runner.run_as_root, + local.normalized_config.runner.run_as_root, + ) + run_as = coalesce( + v.runner.run_as, + local.normalized_config.runner.run_as, + ) + auto_update_disabled = coalesce( + v.runner.auto_update_disabled, + local.normalized_config.runner.auto_update_disabled, + ) + tags = merge(local.normalized_config.runner.tags, v.runner.tags) + hooks = { + job_started = v.runner.hooks.job_started != null ? v.runner.hooks.job_started : local.normalized_config.runner.hooks.job_started + job_completed = v.runner.hooks.job_completed != null ? v.runner.hooks.job_completed : local.normalized_config.runner.hooks.job_completed + } + iam = { + role = try(coalesce( + v.runner.iam.role, + local.normalized_config.runner.iam.role, + ), null) + managed_policy_arns = try(coalesce( + v.runner.iam.managed_policy_arns, + (v.runner.iam.role != null || local.normalized_config.runner.iam.role != null) ? {} : local.normalized_config.runner.iam.managed_policy_arns, + ), {}) + additional_trust_policy_json = try(coalesce( + v.runner.iam.additional_trust_policy_json, + (v.runner.iam.role != null || local.normalized_config.runner.iam.role != null) ? null : local.normalized_config.runner.iam.additional_trust_policy_json, + ), null) + path = try(coalesce( + v.runner.iam.path, + local.normalized_config.runner.iam.path, + local.normalized_config.roles.path, + ), null) + permissions_boundary = try(coalesce( + v.runner.iam.permissions_boundary, + local.normalized_config.runner.iam.permissions_boundary, + local.normalized_config.roles.permissions_boundary, + ), null) + } + }) + + lambda = merge(v.lambda, { + runtime = coalesce( + v.lambda.runtime, + local.normalized_config.lambda.runtime, + ) + architecture = coalesce( + v.lambda.architecture, + local.normalized_config.lambda.architecture, + ) + subnet_ids = coalesce( + v.lambda.subnet_ids, + local.normalized_config.lambda.subnet_ids, + ) + security_group_ids = coalesce( + v.lambda.security_group_ids, + local.normalized_config.lambda.security_group_ids, + ) + tags = merge(local.normalized_config.lambda.tags, v.lambda.tags) + role = { + path = try(coalesce( + v.lambda.role.path, + local.normalized_config.lambda.role.path, + local.normalized_config.roles.path, + ), null) + permissions_boundary = try(coalesce( + v.lambda.role.permissions_boundary, + local.normalized_config.lambda.role.permissions_boundary, + local.normalized_config.roles.permissions_boundary, + ), null) + } + }) + + orchestration_provider = { + webhook = v.orchestration_provider.webhook == null ? null : merge(v.orchestration_provider.webhook, { + runner = { + boot_time_in_minutes = coalesce( + v.orchestration_provider.webhook.runner.boot_time_in_minutes, + local.normalized_config.orchestration_provider.webhook.runner.boot_time_in_minutes, + ) + ephemeral = coalesce( + v.orchestration_provider.webhook.runner.ephemeral, + local.normalized_config.orchestration_provider.webhook.runner.ephemeral, + ) + jit_config_enabled = try(coalesce( + v.orchestration_provider.webhook.runner.jit_config_enabled, + local.normalized_config.orchestration_provider.webhook.runner.jit_config_enabled, + ), null) + maximum_count = try(coalesce( + v.orchestration_provider.webhook.runner.maximum_count, + local.normalized_config.orchestration_provider.webhook.runner.maximum_count, + ), null) + } + + lambda = merge(v.orchestration_provider.webhook.lambda, { + scale = merge(v.orchestration_provider.webhook.lambda.scale, { + up = merge(v.orchestration_provider.webhook.lambda.scale.up, { + memory_size = coalesce( + v.orchestration_provider.webhook.lambda.scale.up.memory_size, + local.normalized_config.orchestration_provider.webhook.lambda.scale.up.memory_size, + ) + timeout = coalesce( + v.orchestration_provider.webhook.lambda.scale.up.timeout, + local.normalized_config.orchestration_provider.webhook.lambda.scale.up.timeout, + ) + reserved_concurrent_executions = coalesce( + v.orchestration_provider.webhook.lambda.scale.up.reserved_concurrent_executions, + local.normalized_config.orchestration_provider.webhook.lambda.scale.up.reserved_concurrent_executions, + ) + job_queued_check_enabled = try(coalesce( + v.orchestration_provider.webhook.lambda.scale.up.job_queued_check_enabled, + local.normalized_config.orchestration_provider.webhook.lambda.scale.up.job_queued_check_enabled, + ), null) + event_source_mapping = { + batch_size = coalesce( + v.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.batch_size, + local.normalized_config.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.batch_size, + ) + maximum_batching_window_in_seconds = coalesce( + v.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.maximum_batching_window_in_seconds, + local.normalized_config.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.maximum_batching_window_in_seconds, + ) + } + tags = merge(local.normalized_config.orchestration_provider.webhook.lambda.scale.up.tags, v.orchestration_provider.webhook.lambda.scale.up.tags) + }) + down = merge(v.orchestration_provider.webhook.lambda.scale.down, { + memory_size = coalesce( + v.orchestration_provider.webhook.lambda.scale.down.memory_size, + local.normalized_config.orchestration_provider.webhook.lambda.scale.down.memory_size, + ) + timeout = coalesce( + v.orchestration_provider.webhook.lambda.scale.down.timeout, + local.normalized_config.orchestration_provider.webhook.lambda.scale.down.timeout, + ) + schedule_expression = coalesce( + v.orchestration_provider.webhook.lambda.scale.down.schedule_expression, + local.normalized_config.orchestration_provider.webhook.lambda.scale.down.schedule_expression, + ) + minimum_running_time_in_minutes = try(coalesce( + v.orchestration_provider.webhook.lambda.scale.down.minimum_running_time_in_minutes, + local.normalized_config.orchestration_provider.webhook.lambda.scale.down.minimum_running_time_in_minutes, + ), null) + idle_config = coalesce( + v.orchestration_provider.webhook.lambda.scale.down.idle_config, + local.normalized_config.orchestration_provider.webhook.lambda.scale.down.idle_config, + ) + tags = merge(local.normalized_config.orchestration_provider.webhook.lambda.scale.down.tags, v.orchestration_provider.webhook.lambda.scale.down.tags) + }) + }) + pool = merge(v.orchestration_provider.webhook.lambda.pool, { + memory_size = coalesce( + v.orchestration_provider.webhook.lambda.pool.memory_size, + local.normalized_config.orchestration_provider.webhook.lambda.pool.memory_size, + ) + timeout = coalesce( + v.orchestration_provider.webhook.lambda.pool.timeout, + local.normalized_config.orchestration_provider.webhook.lambda.pool.timeout, + ) + reserved_concurrent_executions = coalesce( + v.orchestration_provider.webhook.lambda.pool.reserved_concurrent_executions, + local.normalized_config.orchestration_provider.webhook.lambda.pool.reserved_concurrent_executions, + ) + config = coalesce( + v.orchestration_provider.webhook.lambda.pool.config, + local.normalized_config.orchestration_provider.webhook.lambda.pool.config, + ) + include_busy_runners = coalesce( + v.orchestration_provider.webhook.lambda.pool.include_busy_runners, + local.normalized_config.orchestration_provider.webhook.lambda.pool.include_busy_runners, + ) + runner_owner = try(coalesce( + v.orchestration_provider.webhook.lambda.pool.runner_owner, + local.normalized_config.orchestration_provider.webhook.lambda.pool.runner_owner, + ), null) + tags = merge(local.normalized_config.orchestration_provider.webhook.lambda.pool.tags, v.orchestration_provider.webhook.lambda.pool.tags) + }) + }) + + queue = merge(v.orchestration_provider.webhook.queue, { + delay_webhook_event = coalesce( + v.orchestration_provider.webhook.queue.delay_webhook_event, + local.normalized_config.orchestration_provider.webhook.queue.delay_webhook_event, + ) + job_queue_retention_in_seconds = coalesce( + v.orchestration_provider.webhook.queue.job_queue_retention_in_seconds, + local.normalized_config.orchestration_provider.webhook.queue.job_queue_retention_in_seconds, + ) + visibility_timeout_seconds = coalesce( + v.orchestration_provider.webhook.queue.visibility_timeout_seconds, + local.normalized_config.orchestration_provider.webhook.queue.visibility_timeout_seconds, + ) + redrive_build_queue = { + enabled = coalesce( + try(v.orchestration_provider.webhook.queue.redrive_build_queue.enabled, null), + local.normalized_config.orchestration_provider.webhook.queue.redrive_build_queue.enabled, + ) + maxReceiveCount = try( + coalesce( + try(v.orchestration_provider.webhook.queue.redrive_build_queue.maxReceiveCount, null), + local.normalized_config.orchestration_provider.webhook.queue.redrive_build_queue.maxReceiveCount, + ), + null, + ) + } + tags = merge(local.normalized_config.orchestration_provider.webhook.queue.tags, v.orchestration_provider.webhook.queue.tags) + }) + }) + } + + ssm = merge(v.ssm, { + paths = { + root = "${trimsuffix(coalesce( + v.ssm.paths.root, + local.normalized_config.ssm.paths.root, + "/github-action-runners/${var.prefix}", + ), "/")}/${k}" + tokens = coalesce( + v.ssm.paths.tokens, + local.normalized_config.ssm.paths.tokens, + ) + config = coalesce( + v.ssm.paths.config, + local.normalized_config.ssm.paths.config, + ) + } + tags = merge(local.normalized_config.ssm.tags, v.ssm.tags) + parameters = { + tags = merge(local.normalized_config.ssm.parameters.tags, v.ssm.parameters.tags) + } + housekeeper = { + schedule_expression = coalesce( + v.ssm.housekeeper.schedule_expression, + local.normalized_config.ssm.housekeeper.schedule_expression, + ) + state = coalesce( + v.ssm.housekeeper.state, + local.normalized_config.ssm.housekeeper.state, + ) + tags = merge(local.normalized_config.ssm.housekeeper.tags, v.ssm.housekeeper.tags) + lambda = { + # Artifact precedence: lane ZIP, lane S3, global ZIP, then global + # S3. + artifact = v.ssm.housekeeper.lambda.artifact.zip != null ? { + zip = v.ssm.housekeeper.lambda.artifact.zip + s3 = null + } : v.ssm.housekeeper.lambda.artifact.s3 != null ? { + zip = null + s3 = v.ssm.housekeeper.lambda.artifact.s3 + } : local.normalized_config.ssm.housekeeper.lambda.artifact.zip != null ? { + zip = local.normalized_config.ssm.housekeeper.lambda.artifact.zip + s3 = null + } : { + zip = null + s3 = local.normalized_config.ssm.housekeeper.lambda.artifact.s3 + } + memory_size = coalesce( + v.ssm.housekeeper.lambda.memory_size, + local.normalized_config.ssm.housekeeper.lambda.memory_size, + ) + timeout = coalesce( + v.ssm.housekeeper.lambda.timeout, + local.normalized_config.ssm.housekeeper.lambda.timeout, + ) + } + config = { + tokenPath = try(coalesce( + v.ssm.housekeeper.config.tokenPath, + local.normalized_config.ssm.housekeeper.config.tokenPath, + ), null) + minimumDaysOld = coalesce( + v.ssm.housekeeper.config.minimumDaysOld, + local.normalized_config.ssm.housekeeper.config.minimumDaysOld, + ) + dryRun = coalesce( + v.ssm.housekeeper.config.dryRun, + local.normalized_config.ssm.housekeeper.config.dryRun, + ) + } + } + }) + + observability = { + logs = { + level = coalesce( + v.observability.logs.level, + local.normalized_config.observability.logs.level, + ) + retention_in_days = coalesce( + v.observability.logs.retention_in_days, + local.normalized_config.observability.logs.retention_in_days, + ) + kms_key_id = try(coalesce( + v.observability.logs.kms_key_id, + local.normalized_config.observability.logs.kms_key_id, + ), null) + class = coalesce( + v.observability.logs.class, + local.normalized_config.observability.logs.class, + ) + tags = merge(local.normalized_config.observability.logs.tags, v.observability.logs.tags) + } + tracing = { + mode = try(coalesce( + v.observability.tracing.mode, + local.normalized_config.observability.tracing.mode, + ), null) + capture_http_requests = coalesce( + v.observability.tracing.capture_http_requests, + local.normalized_config.observability.tracing.capture_http_requests, + ) + capture_error = coalesce( + v.observability.tracing.capture_error, + local.normalized_config.observability.tracing.capture_error, + ) + } + metrics = { + enabled = coalesce( + v.observability.metrics.enabled, + local.normalized_config.observability.metrics.enabled, + ) + namespace = coalesce( + v.observability.metrics.namespace, + local.normalized_config.observability.metrics.namespace, + ) + metric = { + github_app_rate_limit = { + enabled = coalesce( + v.observability.metrics.metric.github_app_rate_limit.enabled, + local.normalized_config.observability.metrics.metric.github_app_rate_limit.enabled, + ) + } + job_retry = { + enabled = coalesce( + v.observability.metrics.metric.job_retry.enabled, + local.normalized_config.observability.metrics.metric.job_retry.enabled, + ) + } + spot_termination_warning = { + enabled = coalesce( + v.observability.metrics.metric.spot_termination_warning.enabled, + local.normalized_config.observability.metrics.metric.spot_termination_warning.enabled, + ) + } + } + } + } + + compute_provider = { + aws = { + ec2 = v.compute_provider.aws.ec2 == null ? null : merge(v.compute_provider.aws.ec2, { + vpc_id = try(coalesce( + v.compute_provider.aws.ec2.vpc_id, + local.normalized_config.compute_provider.aws.ec2.vpc_id, + ), null) + subnet_ids = try(coalesce( + v.compute_provider.aws.ec2.subnet_ids, + local.normalized_config.compute_provider.aws.ec2.subnet_ids, + ), null) + managed_security_group_enabled = coalesce( + v.compute_provider.aws.ec2.managed_security_group_enabled, + local.normalized_config.compute_provider.aws.ec2.managed_security_group_enabled, + ) + egress_rules = coalesce( + v.compute_provider.aws.ec2.egress_rules, + local.normalized_config.compute_provider.aws.ec2.egress_rules, + ) + additional_security_group_ids = coalesce( + v.compute_provider.aws.ec2.additional_security_group_ids, + local.normalized_config.compute_provider.aws.ec2.additional_security_group_ids, + ) + instance_profile_path = try(coalesce( + v.compute_provider.aws.ec2.instance_profile_path, + local.normalized_config.compute_provider.aws.ec2.instance_profile_path, + ), null) + key_name = try(coalesce( + v.compute_provider.aws.ec2.key_name, + local.normalized_config.compute_provider.aws.ec2.key_name, + ), null) + associate_public_ipv4_address = coalesce( + v.compute_provider.aws.ec2.associate_public_ipv4_address, + local.normalized_config.compute_provider.aws.ec2.associate_public_ipv4_address, + ) + cloudwatch_agent = merge(v.compute_provider.aws.ec2.cloudwatch_agent, { + config = try(coalesce( + v.compute_provider.aws.ec2.cloudwatch_agent.config, + local.normalized_config.compute_provider.aws.ec2.cloudwatch_agent.config, + ), null) + }) + binaries_syncer = { + enabled = coalesce( + v.compute_provider.aws.ec2.binaries_syncer.enabled, + local.normalized_config.compute_provider.aws.ec2.runner_binaries.enabled, + ) + } + tags = merge(local.normalized_config.compute_provider.aws.ec2.tags, v.compute_provider.aws.ec2.tags) + }) + } + } + }) + } + }) +} diff --git a/modules/multi-runner/config.experimental.translation.tf b/modules/multi-runner/config.experimental.translation.tf new file mode 100644 index 0000000000..bcdc14ae4c --- /dev/null +++ b/modules/multi-runner/config.experimental.translation.tf @@ -0,0 +1,560 @@ +# Translate stable v1 inputs into the experimental v2 structure. +locals { + stable_to_experimental_tags = var.tags + + stable_to_experimental_roles = { + path = var.role_path + permissions_boundary = var.role_permissions_boundary + } + + stable_to_experimental_runner = { + os = null + architecture = null + disable_default_labels = false + extra_labels = [] + group_name = "Default" + name_prefix = "" + run_as_root = false + run_as = "ec2-user" + auto_update_disabled = false + tags = {} + hooks = { + job_started = "" + job_completed = "" + } + iam = { + role = null + managed_policy_arns = {} + additional_trust_policy_json = null + path = null + permissions_boundary = null + } + } + + stable_to_experimental_github = { + app = var.github_app + additional_apps = var.additional_github_apps + enterprise_server = { + url = var.ghes_url + ssl_verify = var.ghes_ssl_verify + } + user_agent = var.user_agent + } + + stable_to_experimental_lambda = { + artifact = { + s3 = { + bucket = var.lambda_s3_bucket + } + } + runtime = var.lambda_runtime + architecture = var.lambda_architecture + principals = var.lambda_principals + subnet_ids = var.lambda_subnet_ids + security_group_ids = var.lambda_security_group_ids + tags = var.lambda_tags + role = { + path = null + permissions_boundary = null + } + } + + stable_to_experimental_orchestration_provider = { + webhook = { + queue_selection_strategy = var.queue_selection_strategy + eventbridge = { + enabled = var.eventbridge.enable + accept_events = var.eventbridge.accept_events + } + matcher_config_parameter_store_tier = var.matcher_config_parameter_store_tier + runner = { + boot_time_in_minutes = 5 + ephemeral = false + jit_config_enabled = null + maximum_count = null + } + github = { + repository_white_list = var.repository_white_list + } + lambda = { + artifact = { + zip = var.lambda_s3_bucket == null ? var.runners_lambda_zip : null + s3 = var.lambda_s3_bucket == null ? null : { + key = var.runners_lambda_s3_key + object_version = var.runners_lambda_s3_object_version + } + } + scale = { + up = { + memory_size = var.scale_up_lambda_memory_size + timeout = var.runners_scale_up_lambda_timeout + reserved_concurrent_executions = 1 + job_queued_check_enabled = null + event_source_mapping = { + batch_size = var.lambda_event_source_mapping_batch_size + maximum_batching_window_in_seconds = var.lambda_event_source_mapping_maximum_batching_window_in_seconds + } + tags = {} + } + down = { + memory_size = var.scale_down_lambda_memory_size + timeout = var.runners_scale_down_lambda_timeout + schedule_expression = "cron(*/5 * * * ? *)" + minimum_running_time_in_minutes = null + idle_config = [] + tags = {} + } + } + webhook = { + artifact = { + zip = var.lambda_s3_bucket == null ? var.webhook_lambda_zip : null + s3 = var.lambda_s3_bucket == null ? null : { + key = var.webhook_lambda_s3_key + object_version = var.webhook_lambda_s3_object_version + } + } + api_gateway_access_log_settings = var.webhook_lambda_apigateway_access_log_settings + memory_size = var.webhook_lambda_memory_size + timeout = var.webhook_lambda_timeout + tags = {} + } + pool = { + memory_size = 512 + timeout = var.pool_lambda_timeout + reserved_concurrent_executions = var.pool_lambda_reserved_concurrent_executions + config = [] + include_busy_runners = false + runner_owner = null + tags = {} + } + } + queue = { + delay_webhook_event = 30 + job_queue_retention_in_seconds = 86400 + visibility_timeout_seconds = var.runners_scale_up_lambda_timeout + redrive_build_queue = { + enabled = false + maxReceiveCount = null + } + tags = {} + encryption = var.queue_encryption + } + } + } + + stable_to_experimental_ssm = { + paths = { + root = "/${var.ssm_paths.root}/${var.prefix}" + app = var.ssm_paths.app + webhook = var.ssm_paths.webhook + tokens = "${var.ssm_paths.runners}/tokens" + config = "${var.ssm_paths.runners}/config" + } + kms_key_id = var.kms_key_arn + tags = {} + parameters = { + tags = var.parameter_store_tags + } + housekeeper = { + schedule_expression = var.runners_ssm_housekeeper.schedule_expression + state = var.runners_ssm_housekeeper.enabled ? "ENABLED" : "DISABLED" + tags = {} + lambda = { + artifact = { + zip = var.lambda_s3_bucket == null ? var.runners_lambda_zip : null + s3 = var.lambda_s3_bucket == null ? null : { + key = var.runners_lambda_s3_key + object_version = var.runners_lambda_s3_object_version + } + } + memory_size = var.runners_ssm_housekeeper.lambda_memory_size + timeout = var.runners_ssm_housekeeper.lambda_timeout + } + config = { + tokenPath = var.runners_ssm_housekeeper.config.tokenPath + minimumDaysOld = var.runners_ssm_housekeeper.config.minimumDaysOld + dryRun = var.runners_ssm_housekeeper.config.dryRun + } + } + } + + stable_to_experimental_observability = { + logs = { + level = var.log_level + retention_in_days = var.logging_retention_in_days + kms_key_id = var.logging_kms_key_id + class = var.log_class + tags = {} + } + tracing = var.tracing_config + metrics = { + enabled = var.metrics.enable + namespace = var.metrics.namespace + metric = { + github_app_rate_limit = { + enabled = var.metrics.metric.enable_github_app_rate_limit + } + job_retry = { + enabled = var.metrics.metric.enable_job_retry + } + spot_termination_warning = { + enabled = var.metrics.metric.enable_spot_termination_warning + } + } + } + } + + stable_to_experimental_compute_provider = { + selections = null + aws = { + ec2 = { + vpc_id = var.vpc_id + subnet_ids = var.subnet_ids + managed_security_group_enabled = var.enable_managed_runner_security_group + egress_rules = var.runner_egress_rules + additional_security_group_ids = var.runner_additional_security_group_ids + cloudwatch_agent = { + config = var.cloudwatch_config + } + instance_profile_path = var.instance_profile_path + key_name = var.key_name + associate_public_ipv4_address = var.associate_public_ipv4_address + tags = {} + ami = { + housekeeper = { + enabled = var.enable_ami_housekeeper + cleanup_config = var.ami_housekeeper_cleanup_config + artifact = { + zip = var.lambda_s3_bucket == null ? var.ami_housekeeper_lambda_zip : null + s3 = var.lambda_s3_bucket == null ? null : { + key = var.ami_housekeeper_lambda_s3_key + object_version = var.ami_housekeeper_lambda_s3_object_version + } + } + lambda = { + memory_size = var.ami_housekeeper_lambda_memory_size + timeout = var.ami_housekeeper_lambda_timeout + } + schedule = { + expression = var.ami_housekeeper_lambda_schedule_expression + } + } + } + instance_termination_watcher = { + enabled = var.instance_termination_watcher.enable + features = { + runner_deregistration = { + enabled = var.instance_termination_watcher.enable_runner_deregistration + } + spot_termination_handler = { + enabled = var.instance_termination_watcher.features.enable_spot_termination_handler + } + spot_termination_notification_watcher = { + enabled = var.instance_termination_watcher.features.enable_spot_termination_notification_watcher + } + } + environment_variables = var.instance_termination_watcher.environment_variables + artifact = { + zip = var.lambda_s3_bucket == null ? var.instance_termination_watcher.zip : null + s3 = var.lambda_s3_bucket == null ? null : { + key = var.instance_termination_watcher.s3_key + object_version = var.instance_termination_watcher.s3_object_version + } + } + lambda = { + memory_size = var.instance_termination_watcher.memory_size + timeout = var.instance_termination_watcher.timeout + } + } + runner_binaries = { + enabled = true + s3 = { + encryption = { + enabled = var.runner_binaries_s3_sse_configuration != null + bucket_key_enabled = try(var.runner_binaries_s3_sse_configuration.rule.bucket_key_enabled, null) + sse_algorithm = try(var.runner_binaries_s3_sse_configuration.rule.apply_server_side_encryption_by_default.sse_algorithm, "AES256") + kms_master_key_id = try(var.runner_binaries_s3_sse_configuration.rule.apply_server_side_encryption_by_default.kms_master_key_id, null) + } + tags = var.runner_binaries_s3_tags + versioning = var.runner_binaries_s3_versioning + logging = { + bucket = null + prefix = null + } + } + syncer = { + artifact = { + zip = var.lambda_s3_bucket == null ? var.runner_binaries_syncer_lambda_zip : null + s3 = var.lambda_s3_bucket == null ? null : { + key = var.syncer_lambda_s3_key + object_version = var.syncer_lambda_s3_object_version + } + } + lambda = { + memory_size = var.runner_binaries_syncer_memory_size + timeout = var.runner_binaries_syncer_lambda_timeout + } + schedule = { + expression = "cron(27 * * * ? *)" + state = var.state_event_rule_binaries_syncer + } + } + } + } + } + } + + stable_to_experimental_multi_runner_config = { + for k, v in var.multi_runner_config : k => { + tags = {} + + runner = { + os = v.runner_config.runner_os + architecture = v.runner_config.runner_architecture + disable_default_labels = v.runner_config.runner_disable_default_labels + extra_labels = v.runner_config.runner_extra_labels + group_name = v.runner_config.runner_group_name + name_prefix = v.runner_config.runner_name_prefix + run_as_root = v.runner_config.runner_as_root + run_as = v.runner_config.runner_run_as + auto_update_disabled = v.runner_config.disable_runner_autoupdate + tags = {} + hooks = { + job_started = v.runner_config.runner_hook_job_started + job_completed = v.runner_config.runner_hook_job_completed + } + iam = { + role = v.runner_config.iam_overrides.override_runner_role == true ? { + arn = v.runner_config.iam_overrides.runner_role_arn + } : null + managed_policy_arns = { + for policy_index, policy_arn in v.runner_config.runner_iam_role_managed_policy_arns : + "legacy-${policy_index}" => policy_arn + } + additional_trust_policy_json = null + path = null + permissions_boundary = null + } + } + + lambda = { + runtime = null + architecture = null + subnet_ids = null + security_group_ids = null + tags = {} + role = { + path = null + permissions_boundary = null + } + } + + orchestration_provider = { + webhook = { + runner = { + boot_time_in_minutes = v.runner_config.runner_boot_time_in_minutes + ephemeral = v.runner_config.enable_ephemeral_runners + jit_config_enabled = v.runner_config.enable_jit_config + maximum_count = v.runner_config.runners_maximum_count + } + + github = { + organization_runners = v.runner_config.enable_organization_runners + } + + matcherConfig = { + labelMatchers = v.matcherConfig.labelMatchers + exactMatch = v.matcherConfig.exactMatch + bidirectionalLabelMatch = v.matcherConfig.bidirectionalLabelMatch + priority = v.matcherConfig.priority + dynamic_labels_enabled = v.matcherConfig.enableDynamicLabels + awsDynamicLabelsPolicy = v.matcherConfig.awsDynamicLabelsPolicy + } + + lambda = { + scale = { + up = { + memory_size = null + timeout = null + reserved_concurrent_executions = v.runner_config.scale_up_reserved_concurrent_executions + job_queued_check_enabled = v.runner_config.enable_job_queued_check + event_source_mapping = { + batch_size = v.runner_config.lambda_event_source_mapping_batch_size + maximum_batching_window_in_seconds = v.runner_config.lambda_event_source_mapping_maximum_batching_window_in_seconds + } + tags = {} + } + down = { + memory_size = null + timeout = null + schedule_expression = v.runner_config.scale_down_schedule_expression + minimum_running_time_in_minutes = v.runner_config.minimum_running_time_in_minutes + idle_config = v.runner_config.idle_config + tags = {} + } + } + pool = { + memory_size = null + timeout = null + reserved_concurrent_executions = null + config = v.runner_config.pool_config + include_busy_runners = false + runner_owner = v.runner_config.pool_runner_owner + tags = {} + } + } + + queue = { + delay_webhook_event = v.runner_config.delay_webhook_event + job_queue_retention_in_seconds = v.runner_config.job_queue_retention_in_seconds + visibility_timeout_seconds = var.runners_scale_up_lambda_timeout + redrive_build_queue = v.redrive_build_queue + tags = {} + } + + job_retry = { + enabled = v.runner_config.job_retry.enable + delay_in_seconds = v.runner_config.job_retry.delay_in_seconds + delay_backoff = v.runner_config.job_retry.delay_backoff + max_attempts = v.runner_config.job_retry.max_attempts + tags = {} + lambda = { + memory_size = v.runner_config.job_retry.lambda_memory_size + reserved_concurrent_executions = 1 + timeout = v.runner_config.job_retry.lambda_timeout + } + } + } + } + + ssm = { + paths = { + root = null + tokens = null + config = null + } + tags = {} + parameters = { + tags = {} + } + housekeeper = { + schedule_expression = null + state = null + tags = {} + lambda = { + artifact = { + zip = null + s3 = null + } + memory_size = null + timeout = null + } + config = { + tokenPath = null + minimumDaysOld = null + dryRun = null + } + } + } + + observability = { + logs = { + level = null + retention_in_days = null + kms_key_id = null + class = null + tags = {} + } + tracing = { + mode = null + capture_http_requests = null + capture_error = null + } + metrics = { + enabled = null + namespace = null + metric = { + github_app_rate_limit = { + enabled = null + } + job_retry = { + enabled = null + } + spot_termination_warning = { + enabled = null + } + } + } + } + + compute_provider = { + aws = { + ec2 = { + metadata_options = { + instance_metadata_tags = tostring(v.runner_config.runner_metadata_options["instance_metadata_tags"]) + http_endpoint = tostring(v.runner_config.runner_metadata_options["http_endpoint"]) + http_tokens = tostring(v.runner_config.runner_metadata_options["http_tokens"]) + http_put_response_hop_limit = tonumber(v.runner_config.runner_metadata_options["http_put_response_hop_limit"]) + } + ami = v.runner_config.ami == null ? null : { + filter = v.runner_config.ami.filter + owners = v.runner_config.ami.owners + id_ssm_parameter = v.runner_config.ami.id_ssm_parameter_arn == null ? null : { + arn = v.runner_config.ami.id_ssm_parameter_arn + } + kms_key = v.runner_config.ami.kms_key_arn == null ? null : { + arn = v.runner_config.ami.kms_key_arn + } + } + block_device_mappings = v.runner_config.block_device_mappings + create_service_linked_role_spot = v.runner_config.create_service_linked_role_spot + credit_specification = v.runner_config.credit_specification + ebs_optimized = v.runner_config.ebs_optimized + cloudwatch_agent = { + enabled = v.runner_config.enable_cloudwatch_agent + config = v.runner_config.cloudwatch_config + } + binaries_syncer = { + enabled = v.runner_config.enable_runner_binaries_syncer + } + detailed_monitoring_enabled = v.runner_config.enable_runner_detailed_monitoring + ssm_enabled = v.runner_config.enable_ssm_on_runners + user_data = { + enabled = v.runner_config.enable_userdata + template = v.runner_config.userdata_template + content = v.runner_config.userdata_content + pre_install = v.runner_config.userdata_pre_install + post_install = v.runner_config.userdata_post_install + debug_logging_enabled = false + } + instance_allocation_strategy = v.runner_config.instance_allocation_strategy + instance_max_spot_price = v.runner_config.instance_max_spot_price + instance_target_capacity_type = v.runner_config.instance_target_capacity_type + instance_type_priorities = v.runner_config.instance_type_priorities + instance_types = v.runner_config.instance_types + additional_security_group_ids = length(v.runner_config.runner_additional_security_group_ids) == 0 ? null : v.runner_config.runner_additional_security_group_ids + managed_security_group_enabled = null + egress_rules = null + instance_profile_path = null + key_name = null + associate_public_ipv4_address = null + instance_profile = v.runner_config.iam_overrides.override_instance_profile == true ? { + name = v.runner_config.iam_overrides.instance_profile_name + } : null + on_demand_failover_for_errors = v.runner_config.enable_on_demand_failover_for_errors + scale_errors = v.runner_config.scale_errors + subnet_ids = v.runner_config.subnet_ids + vpc_id = v.runner_config.vpc_id + cpu_options = v.runner_config.cpu_options + placement = v.runner_config.placement + license_specifications = v.runner_config.license_specifications + use_dedicated_host = v.runner_config.use_dedicated_host + log_files = v.runner_config.runner_log_files + tags = v.runner_config.runner_ec2_tags + } + } + } + } + } + +} diff --git a/modules/multi-runner/main.tf b/modules/multi-runner/main.tf index bd96e30847..b9dbf243a3 100644 --- a/modules/multi-runner/main.tf +++ b/modules/multi-runner/main.tf @@ -1,10 +1,10 @@ locals { - tags = merge(var.tags, { + tags = merge(local.effective_config.tags, { "ghr:environment" = var.prefix }) - primary_app_id = coalesce(var.github_app.id_ssm, module.ssm.parameters.github_app_id) - primary_app_key_base64 = coalesce(var.github_app.key_base64_ssm, module.ssm.parameters.github_app_key_base64) + primary_app_id = coalesce(local.effective_config.github.app.id_ssm, module.ssm.parameters.github_app_id) + primary_app_key_base64 = coalesce(local.effective_config.github.app.key_base64_ssm, module.ssm.parameters.github_app_key_base64) github_app_parameters = { id = concat( @@ -19,24 +19,13 @@ locals { [null], [for p in module.ssm.additional_app_parameters : p.installation_id] ) - webhook_secret = coalesce(var.github_app.webhook_secret_ssm, module.ssm.parameters.github_app_webhook_secret) + webhook_secret = coalesce(local.effective_config.github.app.webhook_secret_ssm, module.ssm.parameters.github_app_webhook_secret) } - runner_extra_labels = { for k, v in var.multi_runner_config : k => sort(setunion(flatten(v.matcherConfig.labelMatchers), compact(v.runner_config.runner_extra_labels))) } - - runner_config = { for k, v in var.multi_runner_config : k => merge( - { - id = aws_sqs_queue.queued_builds[k].id - arn = aws_sqs_queue.queued_builds[k].arn - url = aws_sqs_queue.queued_builds[k].url - }, - merge(v, { runner_config = merge(v.runner_config, { runner_extra_labels = local.runner_extra_labels[k] }) }), - ) } - - tmp_distinct_list_unique_os_and_arch = distinct([for i, config in local.runner_config : { "os_type" : config.runner_config.runner_os, "architecture" : config.runner_config.runner_architecture } if config.runner_config.enable_runner_binaries_syncer]) - unique_os_and_arch = { for i, v in local.tmp_distinct_list_unique_os_and_arch : "${v.os_type}_${v.architecture}" => v } - - ssm_root_path = "/${var.ssm_paths.root}/${var.prefix}" + ssm_root_path = trimsuffix(coalesce( + local.effective_config.ssm.paths.root, + "/github-action-runners/${var.prefix}", + ), "/") } resource "random_string" "random" { diff --git a/modules/multi-runner/outputs.tf b/modules/multi-runner/outputs.tf index 50adb7fe46..4c7d4a6cd3 100644 --- a/modules/multi-runner/outputs.tf +++ b/modules/multi-runner/outputs.tf @@ -21,6 +21,18 @@ output "runners_map" { } } +output "runners_map_v2" { + value = { for runner_key, runner in module.runner_configs : runner_key => { + runner = runner.runner + orchestration_provider = runner.orchestration_provider + scale_up = runner.scale_up + scale_down = runner.scale_down + pool = runner.pool + provider = runner.provider + } + } +} + output "binaries_syncer_map" { value = { for runner_binary_key, runner_binary in module.runner_binaries : runner_binary_key => { lambda = runner_binary.lambda @@ -39,8 +51,8 @@ output "webhook" { lambda_role = module.webhook.role endpoint = "${module.webhook.gateway.api_endpoint}/${module.webhook.endpoint_relative_path}" webhook = module.webhook.webhook - dispatcher = var.eventbridge.enable ? module.webhook.dispatcher : null - eventbridge = var.eventbridge.enable ? module.webhook.eventbridge : null + dispatcher = local.effective_config.orchestration_provider.webhook.eventbridge.enabled ? module.webhook.dispatcher : null + eventbridge = local.effective_config.orchestration_provider.webhook.eventbridge.enabled ? module.webhook.eventbridge : null } } @@ -67,7 +79,7 @@ output "ssm_parameters" { } output "instance_termination_watcher" { - value = var.instance_termination_watcher.enable && var.instance_termination_watcher.features.enable_spot_termination_notification_watcher ? { + value = try(local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.enabled, false) && local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.features.spot_termination_notification_watcher.enabled ? { lambda = module.instance_termination_watcher[0].spot_termination_notification.lambda lambda_log_group = module.instance_termination_watcher[0].spot_termination_notification.lambda_log_group lambda_role = module.instance_termination_watcher[0].spot_termination_notification.lambda_role @@ -75,7 +87,7 @@ output "instance_termination_watcher" { } output "instance_termination_handler" { - value = var.instance_termination_watcher.enable && var.instance_termination_watcher.features.enable_spot_termination_handler ? { + value = try(local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.enabled, false) && local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.features.spot_termination_handler.enabled ? { lambda = module.instance_termination_watcher[0].spot_termination_handler.lambda lambda_log_group = module.instance_termination_watcher[0].spot_termination_handler.lambda_log_group lambda_role = module.instance_termination_watcher[0].spot_termination_handler.lambda_role diff --git a/modules/multi-runner/queues.tf b/modules/multi-runner/queues.tf index bcc75f99cc..0f57020571 100644 --- a/modules/multi-runner/queues.tf +++ b/modules/multi-runner/queues.tf @@ -27,42 +27,56 @@ data "aws_iam_policy_document" "deny_insecure_transport" { } resource "aws_sqs_queue" "queued_builds" { - for_each = var.multi_runner_config + for_each = local.effective_config.multi_runner_config name = "${var.prefix}-${each.key}-queued-builds" - delay_seconds = each.value.runner_config.delay_webhook_event - visibility_timeout_seconds = var.runners_scale_up_lambda_timeout - message_retention_seconds = each.value.runner_config.job_queue_retention_in_seconds + delay_seconds = each.value.orchestration_provider.webhook.queue.delay_webhook_event + visibility_timeout_seconds = each.value.orchestration_provider.webhook.queue.visibility_timeout_seconds + message_retention_seconds = each.value.orchestration_provider.webhook.queue.job_queue_retention_in_seconds receive_wait_time_seconds = 0 - redrive_policy = each.value.redrive_build_queue.enabled ? jsonencode({ + redrive_policy = each.value.orchestration_provider.webhook.queue.redrive_build_queue.enabled ? jsonencode({ deadLetterTargetArn = aws_sqs_queue.queued_builds_dlq[each.key].arn, - maxReceiveCount = each.value.redrive_build_queue.maxReceiveCount + maxReceiveCount = each.value.orchestration_provider.webhook.queue.redrive_build_queue.maxReceiveCount }) : null - sqs_managed_sse_enabled = var.queue_encryption.sqs_managed_sse_enabled - kms_master_key_id = var.queue_encryption.kms_master_key_id - kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds + sqs_managed_sse_enabled = local.effective_config.orchestration_provider.webhook.queue.encryption.sqs_managed_sse_enabled + kms_master_key_id = local.effective_config.orchestration_provider.webhook.queue.encryption.kms_master_key_id + kms_data_key_reuse_period_seconds = local.effective_config.orchestration_provider.webhook.queue.encryption.kms_data_key_reuse_period_seconds - tags = var.tags + tags = merge( + local.effective_config.tags, + each.value.tags, + each.value.orchestration_provider.webhook.queue.tags, + ) } resource "aws_sqs_queue_policy" "build_queue_policy" { - for_each = var.multi_runner_config + for_each = local.effective_config.multi_runner_config queue_url = aws_sqs_queue.queued_builds[each.key].id policy = data.aws_iam_policy_document.deny_insecure_transport.json } resource "aws_sqs_queue" "queued_builds_dlq" { - for_each = { for config, values in var.multi_runner_config : config => values if values.redrive_build_queue.enabled } - name = "${var.prefix}-${each.key}-queued-builds_dead_letter" + for_each = { + for config, values in local.effective_config.multi_runner_config : config => values + if values.orchestration_provider.webhook.queue.redrive_build_queue.enabled + } + name = "${var.prefix}-${each.key}-queued-builds_dead_letter" - sqs_managed_sse_enabled = var.queue_encryption.sqs_managed_sse_enabled - kms_master_key_id = var.queue_encryption.kms_master_key_id - kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds - tags = var.tags + sqs_managed_sse_enabled = local.effective_config.orchestration_provider.webhook.queue.encryption.sqs_managed_sse_enabled + kms_master_key_id = local.effective_config.orchestration_provider.webhook.queue.encryption.kms_master_key_id + kms_data_key_reuse_period_seconds = local.effective_config.orchestration_provider.webhook.queue.encryption.kms_data_key_reuse_period_seconds + tags = merge( + local.effective_config.tags, + each.value.tags, + each.value.orchestration_provider.webhook.queue.tags, + ) } resource "aws_sqs_queue_policy" "build_queue_dlq_policy" { - for_each = { for config, values in var.multi_runner_config : config => values if values.redrive_build_queue.enabled } + for_each = { + for config, values in local.effective_config.multi_runner_config : config => values + if values.orchestration_provider.webhook.queue.redrive_build_queue.enabled + } queue_url = aws_sqs_queue.queued_builds_dlq[each.key].id policy = data.aws_iam_policy_document.deny_insecure_transport.json } diff --git a/modules/multi-runner/runner-binaries.tf b/modules/multi-runner/runner-binaries.tf index fb511bb3c5..a02af95ad1 100644 --- a/modules/multi-runner/runner-binaries.tf +++ b/modules/multi-runner/runner-binaries.tf @@ -1,8 +1,28 @@ +locals { + # Derive binary targets from the resolved runner lanes before the binary + # module is instantiated, so the effective configuration can consume the + # binary outputs without depending on its own inputs. + resolved_runner_binary_targets = distinct([ + for config in local.resolved_config.multi_runner_config : { + os_type = config.runner.os + architecture = config.runner.architecture + } + if try(config.compute_provider.aws.ec2.binaries_syncer.enabled, false) + ]) + + resolved_runner_binary_targets_by_key = { + for target in local.resolved_runner_binary_targets : + "${target.os_type}_${target.architecture}" => target + } +} + module "runner_binaries" { source = "../runner-binaries-syncer" - for_each = local.unique_os_and_arch + for_each = local.resolved_runner_binary_targets_by_key prefix = "${var.prefix}-${each.value.os_type}-${each.value.architecture}" - tags = local.tags + tags = merge(local.resolved_config.tags, { + "ghr:environment" = var.prefix + }) # force mandatory lower case for s3 bucketname distribution_bucket_name = lower("${var.prefix}-${each.value.os_type}-${each.value.architecture}-dist-${random_string.random.result}") @@ -10,35 +30,44 @@ module "runner_binaries" { runner_os = each.value.os_type runner_architecture = each.value.architecture - lambda_s3_bucket = var.lambda_s3_bucket - syncer_lambda_s3_key = var.syncer_lambda_s3_key - syncer_lambda_s3_object_version = var.syncer_lambda_s3_object_version - lambda_runtime = var.lambda_runtime - lambda_architecture = var.lambda_architecture - lambda_zip = var.runner_binaries_syncer_lambda_zip - lambda_memory_size = var.runner_binaries_syncer_memory_size - lambda_timeout = var.runner_binaries_syncer_lambda_timeout - lambda_tags = var.lambda_tags - tracing_config = var.tracing_config - logging_retention_in_days = var.logging_retention_in_days - logging_kms_key_id = var.logging_kms_key_id - log_class = var.log_class - state_event_rule_binaries_syncer = var.state_event_rule_binaries_syncer - - server_side_encryption_configuration = var.runner_binaries_s3_sse_configuration - s3_tags = var.runner_binaries_s3_tags - s3_versioning = var.runner_binaries_s3_versioning - - role_path = var.role_path - role_permissions_boundary = var.role_permissions_boundary - - log_level = var.log_level - - lambda_subnet_ids = var.lambda_subnet_ids - lambda_security_group_ids = var.lambda_security_group_ids + lambda_s3_bucket = try(local.resolved_config.lambda.artifact.s3.bucket, null) + syncer_lambda_s3_key = try(local.resolved_config.compute_provider.aws.ec2.runner_binaries.syncer.artifact.s3.key, null) + syncer_lambda_s3_object_version = try(local.resolved_config.compute_provider.aws.ec2.runner_binaries.syncer.artifact.s3.object_version, null) + lambda_runtime = local.resolved_config.lambda.runtime + lambda_architecture = local.resolved_config.lambda.architecture + lambda_zip = local.resolved_config.compute_provider.aws.ec2.runner_binaries.syncer.artifact.zip + lambda_memory_size = local.resolved_config.compute_provider.aws.ec2.runner_binaries.syncer.lambda.memory_size + lambda_timeout = local.resolved_config.compute_provider.aws.ec2.runner_binaries.syncer.lambda.timeout + lambda_tags = local.resolved_config.lambda.tags + tracing_config = local.resolved_config.observability.tracing + logging_retention_in_days = local.resolved_config.observability.logs.retention_in_days + logging_kms_key_id = local.resolved_config.observability.logs.kms_key_id + log_class = local.resolved_config.observability.logs.class + lambda_schedule_expression = local.resolved_config.compute_provider.aws.ec2.runner_binaries.syncer.schedule.expression + state_event_rule_binaries_syncer = local.resolved_config.compute_provider.aws.ec2.runner_binaries.syncer.schedule.state + + server_side_encryption_configuration = try(local.resolved_config.compute_provider.aws.ec2.runner_binaries.s3.encryption.enabled, false) ? { + rule = { + bucket_key_enabled = local.resolved_config.compute_provider.aws.ec2.runner_binaries.s3.encryption.bucket_key_enabled + apply_server_side_encryption_by_default = { + sse_algorithm = local.resolved_config.compute_provider.aws.ec2.runner_binaries.s3.encryption.sse_algorithm + kms_master_key_id = local.resolved_config.compute_provider.aws.ec2.runner_binaries.s3.encryption.kms_master_key_id + } + } + } : null + s3_tags = local.resolved_config.compute_provider.aws.ec2.runner_binaries.s3.tags + s3_versioning = local.resolved_config.compute_provider.aws.ec2.runner_binaries.s3.versioning + + role_path = local.resolved_config.roles.path + role_permissions_boundary = local.resolved_config.roles.permissions_boundary + + log_level = local.resolved_config.observability.logs.level + + lambda_subnet_ids = local.resolved_config.lambda.subnet_ids + lambda_security_group_ids = local.resolved_config.lambda.security_group_ids aws_partition = var.aws_partition - lambda_principals = var.lambda_principals + lambda_principals = local.resolved_config.lambda.principals } locals { runner_binaries_by_os_and_arch_map = { diff --git a/modules/multi-runner/runners.experimental.tf b/modules/multi-runner/runners.experimental.tf new file mode 100644 index 0000000000..4e8652aa00 --- /dev/null +++ b/modules/multi-runner/runners.experimental.tf @@ -0,0 +1,38 @@ +module "runner_configs" { + source = "../runner-config" + for_each = { + for runner_key, runner_config in local.effective_config.multi_runner_config : + runner_key => runner_config if local.use_v2_config + } + + aws_region = var.aws_region + aws_partition = var.aws_partition + prefix = "${var.prefix}-${each.key}" + + tags = merge( + each.value.tags, + { "ghr:environment" = var.prefix }, + ) + runner = each.value.runner + github = merge(each.value.github, { + app_parameters = local.github_app_parameters + }) + lambda = each.value.lambda + orchestration_provider = { + webhook = each.value.orchestration_provider.webhook == null ? null : { + runner = each.value.orchestration_provider.webhook.runner + github = each.value.orchestration_provider.webhook.github + queue = merge(each.value.orchestration_provider.webhook.queue, { + build = { + arn = aws_sqs_queue.queued_builds[each.key].arn + url = aws_sqs_queue.queued_builds[each.key].url + } + }) + lambda = each.value.orchestration_provider.webhook.lambda + job_retry = each.value.orchestration_provider.webhook.job_retry + } + } + ssm = each.value.ssm + observability = each.value.observability + compute_provider = each.value.compute_provider +} diff --git a/modules/multi-runner/runners.tf b/modules/multi-runner/runners.tf index 892113dcc7..5e0bc44022 100644 --- a/modules/multi-runner/runners.tf +++ b/modules/multi-runner/runners.tf @@ -1,130 +1,174 @@ module "runners" { - source = "../runners" - for_each = local.runner_config + source = "../runners" + for_each = { + for runner_key, runner_config in local.effective_config.multi_runner_config : + runner_key => runner_config if !local.use_v2_config + } aws_region = var.aws_region aws_partition = var.aws_partition - vpc_id = coalesce(each.value.runner_config.vpc_id, var.vpc_id) - subnet_ids = coalesce(each.value.runner_config.subnet_ids, var.subnet_ids) + vpc_id = each.value.compute_provider.aws.ec2.vpc_id + subnet_ids = each.value.compute_provider.aws.ec2.subnet_ids prefix = "${var.prefix}-${each.key}" - tags = merge(local.tags, { + tags = merge(local.effective_config.tags, each.value.tags, { "ghr:environment" = "${var.prefix}-${each.key}" }) - s3_runner_binaries = each.value.runner_config.enable_runner_binaries_syncer ? local.runner_binaries_by_os_and_arch_map["${each.value.runner_config.runner_os}_${each.value.runner_config.runner_architecture}"] : null + s3_runner_binaries = try(each.value.compute_provider.aws.ec2.binaries_syncer.enabled, false) ? local.runner_binaries_by_os_and_arch_map["${each.value.runner.os}_${each.value.runner.architecture}"] : null ssm_paths = { - root = "${local.ssm_root_path}/${each.key}" - tokens = "${var.ssm_paths.runners}/tokens" - config = "${var.ssm_paths.runners}/config" + root = each.value.ssm.paths.root + tokens = each.value.ssm.paths.tokens + config = each.value.ssm.paths.config + } + + runner_os = each.value.runner.os + instance_types = each.value.compute_provider.aws.ec2.instance_types + instance_target_capacity_type = each.value.compute_provider.aws.ec2.instance_target_capacity_type + instance_allocation_strategy = each.value.compute_provider.aws.ec2.instance_allocation_strategy + instance_type_priorities = each.value.compute_provider.aws.ec2.instance_type_priorities + instance_max_spot_price = each.value.compute_provider.aws.ec2.instance_max_spot_price + block_device_mappings = each.value.compute_provider.aws.ec2.block_device_mappings + + runner_architecture = each.value.runner.architecture + ami = try(each.value.compute_provider.aws.ec2.ami == null ? null : { + filter = each.value.compute_provider.aws.ec2.ami.filter + owners = each.value.compute_provider.aws.ec2.ami.owners + id_ssm_parameter_arn = try(each.value.compute_provider.aws.ec2.ami.id_ssm_parameter.arn, null) + kms_key_arn = try(each.value.compute_provider.aws.ec2.ami.kms_key.arn, null) + }, null) + + sqs_build_queue = { "arn" : aws_sqs_queue.queued_builds[each.key].arn, "url" : aws_sqs_queue.queued_builds[each.key].url } + github_app_parameters = local.github_app_parameters + ebs_optimized = each.value.compute_provider.aws.ec2.ebs_optimized + enable_on_demand_failover_for_errors = each.value.compute_provider.aws.ec2.on_demand_failover_for_errors + scale_errors = each.value.compute_provider.aws.ec2.scale_errors + enable_organization_runners = each.value.orchestration_provider.webhook.github.organization_runners + enable_ephemeral_runners = each.value.orchestration_provider.webhook.runner.ephemeral + enable_jit_config = each.value.orchestration_provider.webhook.runner.jit_config_enabled + enable_job_queued_check = each.value.orchestration_provider.webhook.lambda.scale.up.job_queued_check_enabled + disable_runner_autoupdate = each.value.runner.auto_update_disabled + enable_managed_runner_security_group = each.value.compute_provider.aws.ec2.managed_security_group_enabled + enable_runner_detailed_monitoring = each.value.compute_provider.aws.ec2.detailed_monitoring_enabled + scale_down_schedule_expression = each.value.orchestration_provider.webhook.lambda.scale.down.schedule_expression + minimum_running_time_in_minutes = each.value.orchestration_provider.webhook.lambda.scale.down.minimum_running_time_in_minutes + runner_boot_time_in_minutes = each.value.orchestration_provider.webhook.runner.boot_time_in_minutes + runner_disable_default_labels = each.value.runner.disable_default_labels + runner_labels = each.value.runner.disable_default_labels ? sort(distinct(each.value.runner.extra_labels)) : sort(distinct(concat(["self-hosted", each.value.runner.os, each.value.runner.architecture], each.value.runner.extra_labels))) + runner_as_root = each.value.runner.run_as_root + runner_run_as = each.value.runner.run_as + runners_maximum_count = each.value.orchestration_provider.webhook.runner.maximum_count + idle_config = each.value.orchestration_provider.webhook.lambda.scale.down.idle_config + enable_ssm_on_runners = each.value.compute_provider.aws.ec2.ssm_enabled + egress_rules = each.value.compute_provider.aws.ec2.egress_rules + runner_additional_security_group_ids = each.value.compute_provider.aws.ec2.additional_security_group_ids + metadata_options = each.value.compute_provider.aws.ec2.metadata_options + credit_specification = each.value.compute_provider.aws.ec2.credit_specification + cpu_options = each.value.compute_provider.aws.ec2.cpu_options + placement = each.value.compute_provider.aws.ec2.placement + license_specifications = each.value.compute_provider.aws.ec2.license_specifications + use_dedicated_host = each.value.compute_provider.aws.ec2.use_dedicated_host + + enable_runner_binaries_syncer = each.value.compute_provider.aws.ec2.binaries_syncer.enabled + lambda_s3_bucket = try(local.effective_config.lambda.artifact.s3.bucket, null) + runners_lambda_s3_key = try(local.effective_config.orchestration_provider.webhook.lambda.artifact.s3.key, null) + runners_lambda_s3_object_version = try(local.effective_config.orchestration_provider.webhook.lambda.artifact.s3.object_version, null) + lambda_runtime = each.value.lambda.runtime + lambda_architecture = each.value.lambda.architecture + lambda_zip = local.effective_config.orchestration_provider.webhook.lambda.artifact.zip + lambda_scale_up_memory_size = each.value.orchestration_provider.webhook.lambda.scale.up.memory_size + lambda_event_source_mapping_batch_size = each.value.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.batch_size + lambda_event_source_mapping_maximum_batching_window_in_seconds = each.value.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.maximum_batching_window_in_seconds + lambda_timeout_scale_up = each.value.orchestration_provider.webhook.lambda.scale.up.timeout + lambda_scale_down_memory_size = each.value.orchestration_provider.webhook.lambda.scale.down.memory_size + lambda_timeout_scale_down = each.value.orchestration_provider.webhook.lambda.scale.down.timeout + lambda_subnet_ids = each.value.lambda.subnet_ids + lambda_security_group_ids = each.value.lambda.security_group_ids + lambda_tags = each.value.lambda.tags + tracing_config = each.value.observability.tracing + logging_retention_in_days = each.value.observability.logs.retention_in_days + logging_kms_key_id = each.value.observability.logs.kms_key_id + log_class = each.value.observability.logs.class + enable_cloudwatch_agent = each.value.compute_provider.aws.ec2.cloudwatch_agent.enabled + cloudwatch_config = each.value.compute_provider.aws.ec2.cloudwatch_agent.config + runner_log_files = each.value.compute_provider.aws.ec2.log_files + runner_group_name = each.value.runner.group_name + runner_name_prefix = each.value.runner.name_prefix + parameter_store_tags = each.value.ssm.parameters.tags + + scale_up_reserved_concurrent_executions = each.value.orchestration_provider.webhook.lambda.scale.up.reserved_concurrent_executions + + instance_profile_path = each.value.compute_provider.aws.ec2.instance_profile_path + role_path = each.value.runner.iam.path + role_permissions_boundary = each.value.runner.iam.permissions_boundary + + enable_userdata = each.value.compute_provider.aws.ec2.user_data.enabled + userdata_template = each.value.compute_provider.aws.ec2.user_data.template + userdata_content = each.value.compute_provider.aws.ec2.user_data.content + userdata_pre_install = each.value.compute_provider.aws.ec2.user_data.pre_install + userdata_post_install = each.value.compute_provider.aws.ec2.user_data.post_install + enable_user_data_debug_logging = each.value.compute_provider.aws.ec2.user_data.debug_logging_enabled + runner_hook_job_started = each.value.runner.hooks.job_started + runner_hook_job_completed = each.value.runner.hooks.job_completed + key_name = each.value.compute_provider.aws.ec2.key_name + runner_ec2_tags = each.value.compute_provider.aws.ec2.tags + + create_service_linked_role_spot = each.value.compute_provider.aws.ec2.create_service_linked_role_spot + + runner_iam_role_managed_policy_arns = values(each.value.runner.iam.managed_policy_arns) + iam_overrides = { + override_instance_profile = each.value.compute_provider.aws.ec2.instance_profile != null + instance_profile_name = try(each.value.compute_provider.aws.ec2.instance_profile.name, null) + override_runner_role = each.value.runner.iam.role != null + runner_role_arn = try(each.value.runner.iam.role.arn, null) } - runner_os = each.value.runner_config.runner_os - instance_types = each.value.runner_config.instance_types - instance_target_capacity_type = each.value.runner_config.instance_target_capacity_type - instance_allocation_strategy = each.value.runner_config.instance_allocation_strategy - instance_type_priorities = each.value.runner_config.instance_type_priorities - instance_max_spot_price = each.value.runner_config.instance_max_spot_price - block_device_mappings = each.value.runner_config.block_device_mappings + ghes_url = local.effective_config.github.enterprise_server.url + ghes_ssl_verify = local.effective_config.github.enterprise_server.ssl_verify + user_agent = local.effective_config.github.user_agent + + kms_key_arn = local.effective_config.ssm.kms_key_id + + log_level = each.value.observability.logs.level + + pool_config = each.value.orchestration_provider.webhook.lambda.pool.config + pool_lambda_memory_size = each.value.orchestration_provider.webhook.lambda.pool.memory_size + pool_lambda_timeout = each.value.orchestration_provider.webhook.lambda.pool.timeout + pool_runner_owner = each.value.orchestration_provider.webhook.lambda.pool.runner_owner + pool_lambda_reserved_concurrent_executions = each.value.orchestration_provider.webhook.lambda.pool.reserved_concurrent_executions + pool_include_busy_runners = each.value.orchestration_provider.webhook.lambda.pool.include_busy_runners + associate_public_ipv4_address = each.value.compute_provider.aws.ec2.associate_public_ipv4_address + + ssm_housekeeper = { + schedule_expression = each.value.ssm.housekeeper.schedule_expression + state = each.value.ssm.housekeeper.state + artifact = { + zip = each.value.ssm.housekeeper.lambda.artifact.zip + s3_bucket = try(local.effective_config.lambda.artifact.s3.bucket, null) + s3_key = try(each.value.ssm.housekeeper.lambda.artifact.s3.key, null) + s3_object_version = try(each.value.ssm.housekeeper.lambda.artifact.s3.object_version, null) + } + lambda_memory_size = each.value.ssm.housekeeper.lambda.memory_size + lambda_timeout = each.value.ssm.housekeeper.lambda.timeout + config = each.value.ssm.housekeeper.config + } - runner_architecture = each.value.runner_config.runner_architecture - ami = each.value.runner_config.ami + job_retry = { + enable = each.value.orchestration_provider.webhook.job_retry.enabled + delay_in_seconds = each.value.orchestration_provider.webhook.job_retry.delay_in_seconds + delay_backoff = each.value.orchestration_provider.webhook.job_retry.delay_backoff + lambda_memory_size = each.value.orchestration_provider.webhook.job_retry.lambda.memory_size + lambda_reserved_concurrent_executions = each.value.orchestration_provider.webhook.job_retry.lambda.reserved_concurrent_executions + lambda_timeout = each.value.orchestration_provider.webhook.job_retry.lambda.timeout + max_attempts = each.value.orchestration_provider.webhook.job_retry.max_attempts + } - sqs_build_queue = { "arn" : each.value.arn, "url" : each.value.url } - github_app_parameters = local.github_app_parameters - ebs_optimized = each.value.runner_config.ebs_optimized - enable_on_demand_failover_for_errors = each.value.runner_config.enable_on_demand_failover_for_errors - scale_errors = each.value.runner_config.scale_errors - enable_organization_runners = each.value.runner_config.enable_organization_runners - enable_ephemeral_runners = each.value.runner_config.enable_ephemeral_runners - enable_jit_config = each.value.runner_config.enable_jit_config - enable_job_queued_check = each.value.runner_config.enable_job_queued_check - disable_runner_autoupdate = each.value.runner_config.disable_runner_autoupdate - enable_managed_runner_security_group = var.enable_managed_runner_security_group - enable_runner_detailed_monitoring = each.value.runner_config.enable_runner_detailed_monitoring - scale_down_schedule_expression = each.value.runner_config.scale_down_schedule_expression - minimum_running_time_in_minutes = each.value.runner_config.minimum_running_time_in_minutes - runner_boot_time_in_minutes = each.value.runner_config.runner_boot_time_in_minutes - runner_disable_default_labels = each.value.runner_config.runner_disable_default_labels - runner_labels = each.value.runner_config.runner_disable_default_labels ? sort(distinct(each.value.runner_config.runner_extra_labels)) : sort(distinct(concat(["self-hosted", each.value.runner_config.runner_os, each.value.runner_config.runner_architecture], each.value.runner_config.runner_extra_labels))) - runner_as_root = each.value.runner_config.runner_as_root - runner_run_as = each.value.runner_config.runner_run_as - runners_maximum_count = each.value.runner_config.runners_maximum_count - idle_config = each.value.runner_config.idle_config - enable_ssm_on_runners = each.value.runner_config.enable_ssm_on_runners - egress_rules = var.runner_egress_rules - runner_additional_security_group_ids = try(coalescelist(each.value.runner_config.runner_additional_security_group_ids, var.runner_additional_security_group_ids), []) - metadata_options = each.value.runner_config.runner_metadata_options - credit_specification = each.value.runner_config.credit_specification - cpu_options = each.value.runner_config.cpu_options - placement = each.value.runner_config.placement - license_specifications = each.value.runner_config.license_specifications - use_dedicated_host = each.value.runner_config.use_dedicated_host - - enable_runner_binaries_syncer = each.value.runner_config.enable_runner_binaries_syncer - lambda_s3_bucket = var.lambda_s3_bucket - runners_lambda_s3_key = var.runners_lambda_s3_key - runners_lambda_s3_object_version = var.runners_lambda_s3_object_version - lambda_runtime = var.lambda_runtime - lambda_architecture = var.lambda_architecture - lambda_zip = var.runners_lambda_zip - lambda_scale_up_memory_size = var.scale_up_lambda_memory_size - lambda_event_source_mapping_batch_size = coalesce(each.value.runner_config.lambda_event_source_mapping_batch_size, var.lambda_event_source_mapping_batch_size) - lambda_event_source_mapping_maximum_batching_window_in_seconds = coalesce(each.value.runner_config.lambda_event_source_mapping_maximum_batching_window_in_seconds, var.lambda_event_source_mapping_maximum_batching_window_in_seconds) - lambda_timeout_scale_up = var.runners_scale_up_lambda_timeout - lambda_scale_down_memory_size = var.scale_down_lambda_memory_size - lambda_timeout_scale_down = var.runners_scale_down_lambda_timeout - lambda_subnet_ids = var.lambda_subnet_ids - lambda_security_group_ids = var.lambda_security_group_ids - lambda_tags = var.lambda_tags - tracing_config = var.tracing_config - logging_retention_in_days = var.logging_retention_in_days - logging_kms_key_id = var.logging_kms_key_id - log_class = var.log_class - enable_cloudwatch_agent = each.value.runner_config.enable_cloudwatch_agent - cloudwatch_config = try(coalesce(each.value.runner_config.cloudwatch_config, var.cloudwatch_config), null) - runner_log_files = each.value.runner_config.runner_log_files - runner_group_name = each.value.runner_config.runner_group_name - runner_name_prefix = each.value.runner_config.runner_name_prefix - parameter_store_tags = var.parameter_store_tags - - scale_up_reserved_concurrent_executions = each.value.runner_config.scale_up_reserved_concurrent_executions - - instance_profile_path = var.instance_profile_path - role_path = var.role_path - role_permissions_boundary = var.role_permissions_boundary - - enable_userdata = each.value.runner_config.enable_userdata - userdata_template = each.value.runner_config.userdata_template - userdata_content = each.value.runner_config.userdata_content - userdata_pre_install = each.value.runner_config.userdata_pre_install - userdata_post_install = each.value.runner_config.userdata_post_install - runner_hook_job_started = each.value.runner_config.runner_hook_job_started - runner_hook_job_completed = each.value.runner_config.runner_hook_job_completed - key_name = var.key_name - runner_ec2_tags = each.value.runner_config.runner_ec2_tags - - create_service_linked_role_spot = each.value.runner_config.create_service_linked_role_spot - - runner_iam_role_managed_policy_arns = each.value.runner_config.runner_iam_role_managed_policy_arns - iam_overrides = each.value.runner_config.iam_overrides - - ghes_url = var.ghes_url - ghes_ssl_verify = var.ghes_ssl_verify - user_agent = var.user_agent - - kms_key_arn = var.kms_key_arn - - log_level = var.log_level - - pool_config = each.value.runner_config.pool_config - pool_lambda_timeout = var.pool_lambda_timeout - pool_runner_owner = each.value.runner_config.pool_runner_owner - pool_lambda_reserved_concurrent_executions = var.pool_lambda_reserved_concurrent_executions - associate_public_ipv4_address = var.associate_public_ipv4_address - - ssm_housekeeper = var.runners_ssm_housekeeper - - job_retry = each.value.runner_config.job_retry - - metrics = var.metrics + metrics = { + enable = each.value.observability.metrics.enabled + namespace = each.value.observability.metrics.namespace + metric = { + enable_github_app_rate_limit = each.value.observability.metrics.metric.github_app_rate_limit.enabled + enable_job_retry = each.value.observability.metrics.metric.job_retry.enabled + enable_spot_termination_warning = each.value.observability.metrics.metric.spot_termination_warning.enabled + } + } } diff --git a/modules/multi-runner/ssm.tf b/modules/multi-runner/ssm.tf index 3e4b740fdd..4c27495b8d 100644 --- a/modules/multi-runner/ssm.tf +++ b/modules/multi-runner/ssm.tf @@ -1,8 +1,11 @@ module "ssm" { source = "../ssm" - kms_key_arn = var.kms_key_arn - path_prefix = "${local.ssm_root_path}/${var.ssm_paths.app}" - github_app = var.github_app - additional_github_apps = var.additional_github_apps - tags = local.tags + kms_key_arn = local.effective_config.ssm.kms_key_id + path_prefix = "${local.ssm_root_path}/${local.effective_config.ssm.paths.app}" + github_app = local.effective_config.github.app + additional_github_apps = local.effective_config.github.additional_apps + tags = merge( + local.tags, + local.effective_config.ssm.tags, + ) } diff --git a/modules/multi-runner/termination-watcher.tf b/modules/multi-runner/termination-watcher.tf index 750db361bf..a710ae9620 100644 --- a/modules/multi-runner/termination-watcher.tf +++ b/modules/multi-runner/termination-watcher.tf @@ -1,36 +1,51 @@ locals { lambda_instance_termination_watcher = { - prefix = var.prefix - tags = local.tags - aws_partition = var.aws_partition - architecture = var.lambda_architecture - principals = var.lambda_principals - runtime = var.lambda_runtime - security_group_ids = var.lambda_security_group_ids - subnet_ids = var.lambda_subnet_ids - log_level = var.log_level - log_class = var.log_class - logging_kms_key_id = var.logging_kms_key_id - logging_retention_in_days = var.logging_retention_in_days - role_path = var.role_path - role_permissions_boundary = var.role_permissions_boundary - s3_bucket = var.lambda_s3_bucket - tracing_config = var.tracing_config - lambda_tags = var.lambda_tags - metrics = var.metrics - enable_runner_deregistration = var.instance_termination_watcher.enable_runner_deregistration - github_app_parameters = var.instance_termination_watcher.enable_runner_deregistration ? { + prefix = var.prefix + tags = local.tags + aws_partition = var.aws_partition + architecture = local.effective_config.lambda.architecture + principals = local.effective_config.lambda.principals + runtime = local.effective_config.lambda.runtime + security_group_ids = local.effective_config.lambda.security_group_ids + subnet_ids = local.effective_config.lambda.subnet_ids + log_level = local.effective_config.observability.logs.level + log_class = local.effective_config.observability.logs.class + logging_kms_key_id = local.effective_config.observability.logs.kms_key_id + logging_retention_in_days = local.effective_config.observability.logs.retention_in_days + role_path = local.effective_config.roles.path + role_permissions_boundary = local.effective_config.roles.permissions_boundary + s3_bucket = try(local.effective_config.lambda.artifact.s3.bucket, null) + s3_key = try(local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.artifact.s3.key, null) + s3_object_version = try(local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.artifact.s3.object_version, null) + zip = local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.artifact.zip + tracing_config = local.effective_config.observability.tracing + lambda_tags = local.effective_config.lambda.tags + metrics = { + enable = local.effective_config.observability.metrics.enabled + namespace = local.effective_config.observability.metrics.namespace + metric = { + enable_github_app_rate_limit = local.effective_config.observability.metrics.metric.github_app_rate_limit.enabled + enable_job_retry = local.effective_config.observability.metrics.metric.job_retry.enabled + enable_spot_termination_warning = local.effective_config.observability.metrics.metric.spot_termination_warning.enabled + } + } + features = { + enable_spot_termination_handler = local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.features.spot_termination_handler.enabled + enable_spot_termination_notification_watcher = local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.features.spot_termination_notification_watcher.enabled + } + enable_runner_deregistration = local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.features.runner_deregistration.enabled + github_app_parameters = local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.features.runner_deregistration.enabled ? { id = local.github_app_parameters.id[0] key_base64 = local.github_app_parameters.key_base64[0] } : null - ghes_url = var.ghes_url - environment_variables = var.instance_termination_watcher.environment_variables + ghes_url = local.effective_config.github.enterprise_server.url + environment_variables = local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.environment_variables } } module "instance_termination_watcher" { source = "../termination-watcher" - count = var.instance_termination_watcher.enable ? 1 : 0 + count = try(local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.enabled, false) ? 1 : 0 - config = merge(local.lambda_instance_termination_watcher, var.instance_termination_watcher) + config = local.lambda_instance_termination_watcher } diff --git a/modules/multi-runner/tests/config-effective.tftest.hcl b/modules/multi-runner/tests/config-effective.tftest.hcl new file mode 100644 index 0000000000..e2d8123e3d --- /dev/null +++ b/modules/multi-runner/tests/config-effective.tftest.hcl @@ -0,0 +1,252 @@ +mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } + + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/test-role" + } + } + + mock_resource "aws_cloudwatch_event_bus" { + defaults = { + arn = "arn:aws:events:eu-west-1:123456789012:event-bus/test" + } + } + + mock_resource "aws_cloudwatch_event_rule" { + defaults = { + arn = "arn:aws:events:eu-west-1:123456789012:rule/test" + } + } + + mock_resource "aws_lambda_function" { + defaults = { + arn = "arn:aws:lambda:eu-west-1:123456789012:function:test" + } + } + + mock_resource "aws_sqs_queue" { + defaults = { + arn = "arn:aws:sqs:eu-west-1:123456789012:test" + } + } + + mock_resource "aws_s3_bucket" { + defaults = { + arn = "arn:aws:s3:::test-runner-binaries" + id = "test-runner-binaries" + } + } + + mock_resource "aws_apigatewayv2_api" { + defaults = { + execution_arn = "arn:aws:execute-api:eu-west-1:123456789012:test" + } + } +} + +mock_provider "random" {} +mock_provider "null" {} + +variables { + aws_region = "eu-west-1" + vpc_id = "vpc-test" + subnet_ids = ["subnet-test"] + + multi_runner_config = {} + + github_app = { + key_base64_ssm = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/tests/github-app/key" + name = "/tests/github-app/key" + } + id_ssm = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/tests/github-app/id" + name = "/tests/github-app/id" + } + webhook_secret_ssm = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/tests/github-app/webhook-secret" + name = "/tests/github-app/webhook-secret" + } + } + + lambda_s3_bucket = "test-lambda-artifacts" + runners_lambda_zip = "README.md" + runners_lambda_s3_key = "runners.zip" + webhook_lambda_s3_key = "webhook.zip" + syncer_lambda_s3_key = "runner-binaries-syncer.zip" +} + +run "v1_effective_config_contains_derived_runner_labels" { + command = plan + + variables { + experimental_multi_runner_config = {} + + multi_runner_config = { + stable = { + runner_config = { + runner_os = "linux" + runner_architecture = "x64" + instance_types = ["m5.large"] + runners_maximum_count = 1 + } + matcherConfig = { + labelMatchers = [["stable-label"]] + } + } + } + } + + assert { + condition = toset(local.effective_config.multi_runner_config["stable"].runner.labels) == toset([ + "linux", + "self-hosted", + "stable-label", + "x64", + ]) + error_message = "The effective v1 configuration must contain the translated runner labels." + } +} + +run "v2_effective_config_contains_derived_values" { + command = apply + + variables { + experimental_global_config = { + runner = { + os = "linux" + architecture = "x64" + } + } + + experimental_global_config_github = { + app = { + key_base64 = "experimental-app-key" + id = "experimental-app-id" + webhook_secret = "experimental-webhook-secret" + } + } + + experimental_global_config_lambda = { + artifact = { + s3 = { + bucket = "global-lambda-artifacts" + } + } + subnet_ids = ["subnet-lambda"] + security_group_ids = ["sg-lambda"] + } + + experimental_global_config_orchestration_provider = { + webhook = { + lambda = { + artifact = { + s3 = { + key = "global-runners.zip" + } + } + webhook = { + artifact = { + s3 = { + key = "global-webhook.zip" + } + } + } + } + queue = { + encryption = { + kms_data_key_reuse_period_seconds = 300 + kms_master_key_id = "kms-global-queue" + sqs_managed_sse_enabled = null + } + } + } + } + + experimental_global_config_ssm = { + kms_key_id = "kms-global-ssm" + housekeeper = { + lambda = { + artifact = { + s3 = { + key = "global-housekeeper.zip" + } + } + } + } + } + + experimental_global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-global" + subnet_ids = ["subnet-global"] + runner_binaries = { + enabled = true + syncer = { + artifact = { + s3 = { + key = "runner-binaries-syncer.zip" + } + } + } + } + } + } + } + + experimental_multi_runner_config = { + lane = { + runner = { + extra_labels = ["lane-label"] + } + orchestration_provider = { + webhook = { + matcherConfig = { + labelMatchers = [["matcher-label"]] + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = ["m5.large"] + binaries_syncer = { + enabled = true + } + } + } + } + } + } + } + + assert { + condition = ( + toset(local.effective_config.multi_runner_config["lane"].runner.labels) == toset([ + "lane-label", + "linux", + "matcher-label", + "self-hosted", + "x64", + ]) + && local.effective_config.multi_runner_config["lane"].lambda.artifact.s3.bucket == "global-lambda-artifacts" + && local.effective_config.multi_runner_config["lane"].orchestration_provider.webhook.lambda.artifact.s3.key == "global-runners.zip" + && local.effective_config.multi_runner_config["lane"].orchestration_provider.webhook.queue.kms_key_id == "kms-global-queue" + && local.effective_config.multi_runner_config["lane"].ssm.kms_key_id == "kms-global-ssm" + && toset(keys(local.resolved_runner_binary_targets_by_key)) == toset(["linux_x64"]) + ) + error_message = "The effective v2 configuration must contain global values, derived labels, and the resolved runner-binary target map." + } +} diff --git a/modules/multi-runner/tests/config-resolution.tftest.hcl b/modules/multi-runner/tests/config-resolution.tftest.hcl new file mode 100644 index 0000000000..a1e389a35e --- /dev/null +++ b/modules/multi-runner/tests/config-resolution.tftest.hcl @@ -0,0 +1,462 @@ +mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } + + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/test-role" + } + } + + mock_resource "aws_cloudwatch_event_bus" { + defaults = { + arn = "arn:aws:events:eu-west-1:123456789012:event-bus/test" + } + } + + mock_resource "aws_cloudwatch_event_rule" { + defaults = { + arn = "arn:aws:events:eu-west-1:123456789012:rule/test" + } + } + + mock_resource "aws_lambda_function" { + defaults = { + arn = "arn:aws:lambda:eu-west-1:123456789012:function:test" + } + } + + mock_resource "aws_sqs_queue" { + defaults = { + arn = "arn:aws:sqs:eu-west-1:123456789012:test" + } + } + + mock_resource "aws_s3_bucket" { + defaults = { + arn = "arn:aws:s3:::test-lambda-artifacts" + id = "test-lambda-artifacts" + } + } + + mock_resource "aws_apigatewayv2_api" { + defaults = { + execution_arn = "arn:aws:execute-api:eu-west-1:123456789012:test" + } + } +} + +mock_provider "random" {} +mock_provider "null" {} + +variables { + aws_region = "eu-west-1" + prefix = "test" + aws_partition = "aws" + + experimental_global_config_github = { + app = { + key_base64 = "experimental-app-key" + id = "experimental-app-id" + webhook_secret = "experimental-webhook-secret" + } + } + + experimental_global_config_lambda = { + artifact = { + s3 = { + bucket = "test-lambda-artifacts" + } + } + } + + experimental_global_config_orchestration_provider = { + webhook = { + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + webhook = { + artifact = { + s3 = { + key = "webhook.zip" + } + } + } + } + } + } + + experimental_global_config_ssm = { + housekeeper = { + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + } + } + } + + experimental_global_config_compute_provider = { + aws = { + ec2 = { + runner_binaries = { + syncer = { + artifact = { + s3 = { + key = "runner-binaries-syncer.zip" + } + } + } + } + } + } + } +} + +run "v1_stable_inputs_translate_into_effective_base" { + command = plan + + variables { + vpc_id = "vpc-stable" + subnet_ids = ["subnet-stable"] + + github_app = { + key_base64_ssm = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/tests/github-app/key" + name = "/tests/github-app/key" + } + id_ssm = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/tests/github-app/id" + name = "/tests/github-app/id" + } + webhook_secret_ssm = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/tests/github-app/webhook-secret" + name = "/tests/github-app/webhook-secret" + } + } + + lambda_s3_bucket = "test-lambda-artifacts" + runners_lambda_zip = "README.md" + runners_lambda_s3_key = "runners.zip" + webhook_lambda_s3_key = "webhook.zip" + syncer_lambda_s3_key = "runner-binaries-syncer.zip" + + tags = { + source = "v1" + } + + experimental_global_config = { + tags = { + source = "v2-must-not-leak" + } + } + + experimental_multi_runner_config = {} + + multi_runner_config = { + stable = { + runner_config = { + runner_os = "linux" + runner_architecture = "x64" + instance_types = ["m5.large"] + runners_maximum_count = 2 + runner_group_name = "v1-lane" + } + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64"]] + } + } + } + } + + assert { + condition = ( + !local.use_v2_config + && local.normalized_config.tags.source == "v1" + && keys(local.resolved_config.multi_runner_config) == ["stable"] + && local.resolved_config.tags.source == "v1" + && local.resolved_config.multi_runner_config["stable"].runner.os == "linux" + && local.resolved_config.multi_runner_config["stable"].runner.architecture == "x64" + && local.resolved_config.multi_runner_config["stable"].runner.group_name == "v1-lane" + && local.resolved_config.multi_runner_config["stable"].orchestration_provider.webhook.runner.maximum_count == 2 + && toset(local.resolved_config.multi_runner_config["stable"].compute_provider.aws.ec2.instance_types) == toset(["m5.large"]) + && toset(local.effective_config.multi_runner_config["stable"].runner.labels) == toset(["linux", "self-hosted", "x64"]) + ) + error_message = "Stable v1 inputs must translate into the effective experimental base without leaking v2 globals." + } + + assert { + condition = ( + keys(module.runners) == ["stable"] + && length(module.runner_configs) == 0 + && keys(output.runners_map) == ["stable"] + && length(output.runners_map_v2) == 0 + ) + error_message = "Stable v1 configurations must route through module.runners and not the experimental runner-config module." + } +} + +run "v2_experimental_inputs_resolve_lane_over_global" { + command = plan + + variables { + tags = { + source = "v1-must-not-leak" + } + + multi_runner_config = { + stable = { + runner_config = { + runner_os = "windows" + runner_architecture = "x64" + instance_types = ["m5.large"] + runners_maximum_count = 1 + enable_runner_binaries_syncer = false + } + matcherConfig = { + labelMatchers = [["stable"]] + } + } + } + + experimental_global_config = { + tags = { + source = "v2" + } + runner = { + os = "linux" + architecture = "arm64" + group_name = "global-group" + } + } + + experimental_global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-global" + subnet_ids = ["subnet-global"] + instance_termination_watcher = { + features = { + runner_deregistration = { + enabled = false + } + spot_termination_handler = { + enabled = false + } + spot_termination_notification_watcher = { + enabled = false + } + } + } + runner_binaries = { + enabled = false + syncer = { + artifact = { + s3 = { + key = "runner-binaries-syncer.zip" + } + } + } + } + } + } + } + + experimental_global_config_observability = { + metrics = { + enabled = true + metric = { + github_app_rate_limit = { + enabled = false + } + job_retry = { + enabled = false + } + } + } + } + + experimental_global_config_orchestration_provider = { + webhook = { + eventbridge = { + enabled = false + } + lambda = { + artifact = { + s3 = { + key = "global-runners.zip" + } + } + webhook = { + artifact = { + s3 = { + key = "global-webhook.zip" + } + } + } + } + } + } + + experimental_global_config_ssm = { + housekeeper = { + lambda = { + artifact = { + s3 = { + key = "global-housekeeper.zip" + } + } + } + } + } + + experimental_multi_runner_config = { + lane = { + runner = { + group_name = "lane-group" + } + orchestration_provider = { + webhook = { + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "arm64"]] + dynamic_labels_enabled = true + } + } + } + observability = { + metrics = { + enabled = false + metric = { + github_app_rate_limit = { + enabled = true + } + job_retry = { + enabled = true + } + } + } + } + ssm = { + housekeeper = { + lambda = { + artifact = { + s3 = { + key = "lane-housekeeper.zip" + } + } + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = ["c7g.large"] + subnet_ids = ["subnet-lane"] + on_demand_failover_for_errors = ["InsufficientInstanceCapacity"] + } + } + } + } + } + } + + assert { + condition = ( + local.use_v2_config + && local.normalized_config.tags.source == "v2" + && toset(keys(local.resolved_config.multi_runner_config)) == toset(["lane"]) + && local.resolved_config.tags.source == "v2" + && local.resolved_config.multi_runner_config["lane"].runner.os == "linux" + && local.resolved_config.multi_runner_config["lane"].runner.architecture == "arm64" + && local.resolved_config.multi_runner_config["lane"].runner.group_name == "lane-group" + && local.resolved_config.multi_runner_config["lane"].compute_provider.aws.ec2.vpc_id == "vpc-global" + && toset(local.resolved_config.multi_runner_config["lane"].compute_provider.aws.ec2.subnet_ids) == toset(["subnet-lane"]) + && local.resolved_config.multi_runner_config["lane"].orchestration_provider.webhook.matcherConfig.dynamic_labels_enabled + && !local.resolved_config.multi_runner_config["lane"].observability.metrics.enabled + && local.resolved_config.multi_runner_config["lane"].observability.metrics.metric.github_app_rate_limit.enabled + && local.resolved_config.multi_runner_config["lane"].observability.metrics.metric.job_retry.enabled + && tolist(local.resolved_config.multi_runner_config["lane"].compute_provider.aws.ec2.on_demand_failover_for_errors) == tolist(["InsufficientInstanceCapacity"]) + && !local.resolved_config.orchestration_provider.webhook.eventbridge.enabled + && !local.resolved_config.compute_provider.aws.ec2.instance_termination_watcher.features.spot_termination_handler.enabled + && !local.resolved_config.compute_provider.aws.ec2.instance_termination_watcher.features.spot_termination_notification_watcher.enabled + && !local.resolved_config.compute_provider.aws.ec2.instance_termination_watcher.features.runner_deregistration.enabled + && local.resolved_config.multi_runner_config["lane"].ssm.housekeeper.lambda.artifact.zip == null + && local.resolved_config.multi_runner_config["lane"].ssm.housekeeper.lambda.artifact.s3.key == "lane-housekeeper.zip" + && toset(local.effective_config.multi_runner_config["lane"].runner.labels) == toset(["arm64", "linux", "self-hosted"]) + ) + error_message = "Experimental v2 inputs must resolve lane overrides before experimental global defaults." + } + + assert { + condition = ( + length(module.runners) == 0 + && keys(module.runner_configs) == ["lane"] + && length(output.runners_map) == 0 + && keys(output.runners_map_v2) == ["lane"] + ) + error_message = "Experimental v2 configurations must route through module.runner_configs and skip the legacy runners module." + } +} + +run "v2_inputs_do_not_require_legacy_arguments" { + command = plan + + variables { + experimental_global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-v2" + subnet_ids = ["subnet-v2"] + runner_binaries = { + enabled = false + } + } + } + } + experimental_multi_runner_config = { + lane = { + orchestration_provider = { + webhook = { + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64"]] + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = ["m5.large"] + binaries_syncer = { + enabled = false + } + } + } + } + } + } + } + + assert { + condition = ( + local.use_v2_config + && keys(module.runner_configs) == ["lane"] + && length(module.runners) == 0 + && local.resolved_config.multi_runner_config["lane"].compute_provider.aws.ec2.vpc_id == "vpc-v2" + ) + error_message = "The v2 interface must work without the stable v1 GitHub App, VPC, subnet, or runner configuration inputs." + } +} diff --git a/modules/multi-runner/tests/config-translation.tftest.hcl b/modules/multi-runner/tests/config-translation.tftest.hcl new file mode 100644 index 0000000000..e58f2e3944 --- /dev/null +++ b/modules/multi-runner/tests/config-translation.tftest.hcl @@ -0,0 +1,778 @@ +mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } + + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/test-role" + } + } + + mock_resource "aws_cloudwatch_event_bus" { + defaults = { + arn = "arn:aws:events:eu-west-1:123456789012:event-bus/test" + } + } + + mock_resource "aws_cloudwatch_event_rule" { + defaults = { + arn = "arn:aws:events:eu-west-1:123456789012:rule/test" + } + } + + mock_resource "aws_lambda_function" { + defaults = { + arn = "arn:aws:lambda:eu-west-1:123456789012:function:test" + } + } + + mock_resource "aws_sqs_queue" { + defaults = { + arn = "arn:aws:sqs:eu-west-1:123456789012:test" + } + } + + mock_resource "aws_s3_bucket" { + defaults = { + arn = "arn:aws:s3:::test-runner-binaries" + id = "test-runner-binaries" + } + } + + mock_resource "aws_apigatewayv2_api" { + defaults = { + execution_arn = "arn:aws:execute-api:eu-west-1:123456789012:test" + } + } +} + +mock_provider "random" {} +mock_provider "null" {} + +variables { + aws_region = "eu-west-1" + vpc_id = "vpc-stable" + subnet_ids = ["subnet-stable"] + + github_app = { + key_base64_ssm = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/tests/github-app/key" + name = "/tests/github-app/key" + } + id_ssm = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/tests/github-app/id" + name = "/tests/github-app/id" + } + webhook_secret_ssm = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/tests/github-app/webhook-secret" + name = "/tests/github-app/webhook-secret" + } + } + + multi_runner_config = {} + + lambda_s3_bucket = "test-lambda-artifacts" + runners_lambda_zip = "README.md" + runners_lambda_s3_key = "runners.zip" + webhook_lambda_s3_key = "webhook.zip" + syncer_lambda_s3_key = "runner-binaries-syncer.zip" + + experimental_global_config_github = { + app = { + key_base64 = "experimental-app-key" + id = "experimental-app-id" + webhook_secret = "experimental-webhook-secret" + } + } + + experimental_global_config_lambda = { + artifact = { + s3 = { + bucket = "test-lambda-artifacts" + } + } + } + + experimental_global_config_orchestration_provider = { + webhook = { + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + webhook = { + artifact = { + s3 = { + key = "webhook.zip" + } + } + } + } + } + } + + experimental_global_config_ssm = { + housekeeper = { + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + } + } + } + + experimental_global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-experimental-default" + subnet_ids = ["subnet-experimental-default"] + runner_binaries = { + syncer = { + artifact = { + s3 = { + key = "runner-binaries-syncer.zip" + } + } + } + } + } + } + } +} + +run "empty_experimental_map_translates_stable_inputs" { + command = plan + + variables { + tags = { + source = "stable" + } + + role_path = "/stable/" + role_permissions_boundary = "arn:aws:iam::123456789012:policy/stable-boundary" + queue_selection_strategy = "random" + repository_white_list = ["example/repository"] + additional_github_apps = [{ + id = "additional-app-id" + key_base64 = "additional-app-key" + installation_id = "additional-installation-id" + }] + ghes_url = "https://github.example.test" + ghes_ssl_verify = false + user_agent = "stable-test-agent" + eventbridge = { enable = false, accept_events = ["workflow_job"] } + matcher_config_parameter_store_tier = "Advanced" + scale_up_lambda_memory_size = 1024 + runners_scale_up_lambda_timeout = 45 + scale_down_lambda_memory_size = 768 + runners_scale_down_lambda_timeout = 75 + webhook_lambda_memory_size = 384 + webhook_lambda_timeout = 20 + pool_lambda_timeout = 90 + pool_lambda_reserved_concurrent_executions = 2 + lambda_event_source_mapping_batch_size = 25 + lambda_event_source_mapping_maximum_batching_window_in_seconds = 10 + webhook_lambda_apigateway_access_log_settings = { + destination_arn = "arn:aws:logs:eu-west-1:123456789012:log-group:test" + format = "$context.requestId" + } + kms_key_arn = "arn:aws:kms:eu-west-1:123456789012:key/stable" + queue_encryption = { + kms_data_key_reuse_period_seconds = 300 + kms_master_key_id = "arn:aws:kms:eu-west-1:123456789012:key/queue" + sqs_managed_sse_enabled = null + } + ssm_paths = { + root = "legacy-root" + app = "legacy-app" + runners = "legacy-runners" + webhook = "legacy-webhook" + } + parameter_store_tags = { owner = "stable-test" } + runners_ssm_housekeeper = { + schedule_expression = "rate(2 days)" + enabled = false + lambda_memory_size = 640 + lambda_timeout = 70 + config = { + tokenPath = "/stable/tokens" + minimumDaysOld = 5 + dryRun = true + } + } + log_level = "debug" + logging_retention_in_days = 30 + logging_kms_key_id = "arn:aws:kms:eu-west-1:123456789012:key/logs" + log_class = "INFREQUENT_ACCESS" + tracing_config = { + mode = "Active" + capture_http_requests = true + capture_error = true + } + metrics = { + enable = true + namespace = "StableTest" + metric = { + enable_github_app_rate_limit = false + enable_job_retry = false + enable_spot_termination_warning = false + } + } + enable_managed_runner_security_group = false + runner_egress_rules = [{ + cidr_blocks = ["10.0.0.0/8"] + ipv6_cidr_blocks = [] + prefix_list_ids = [] + from_port = 443 + protocol = "tcp" + security_groups = [] + self = false + to_port = 443 + description = "stable-test" + }] + runner_additional_security_group_ids = ["sg-stable"] + cloudwatch_config = "{\"metrics\":{}}" + instance_profile_path = "/stable/instance-profile/" + key_name = "stable-key" + associate_public_ipv4_address = true + instance_termination_watcher = { + enable = true + enable_runner_deregistration = false + environment_variables = { MODE = "stable-test" } + memory_size = 256 + timeout = 40 + zip = "watcher.zip" + s3_key = "watcher.zip" + s3_object_version = "watcher-version" + } + runner_binaries_s3_sse_configuration = { + rule = { + bucket_key_enabled = true + apply_server_side_encryption_by_default = { + sse_algorithm = "aws:kms" + kms_master_key_id = "arn:aws:kms:eu-west-1:123456789012:key/binaries" + } + } + } + runner_binaries_s3_tags = { component = "stable-test" } + runner_binaries_s3_versioning = "Enabled" + state_event_rule_binaries_syncer = "DISABLED" + + experimental_global_config = { + tags = { + source = "experimental-ignored" + } + roles = { + path = "/experimental-ignored/" + } + } + + experimental_global_config_github = { + user_agent = "experimental-ignored" + } + + experimental_global_config_lambda = { + runtime = "nodejs22.x" + } + + experimental_global_config_orchestration_provider = { + webhook = { + queue_selection_strategy = "first" + github = { + repository_white_list = ["ignored/repository"] + } + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + webhook = { + artifact = { + s3 = { + key = "webhook.zip" + } + } + } + } + } + } + + experimental_global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-experimental-ignored" + subnet_ids = ["subnet-experimental-ignored"] + runner_binaries = { + syncer = { + artifact = { + s3 = { + key = "runner-binaries-syncer.zip" + } + } + } + } + } + } + } + + experimental_multi_runner_config = {} + + multi_runner_config = { + stable = { + runner_config = { + runner_os = "linux" + runner_architecture = "x64" + instance_types = ["m5.large"] + runners_maximum_count = 2 + runner_group_name = "stable-group" + runner_iam_role_managed_policy_arns = [ + "arn:aws:iam::123456789012:policy/stable-runner", + ] + } + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64"]] + } + } + } + } + + assert { + condition = ( + !local.use_v2_config + && toset(keys(local.normalized_config.multi_runner_config)) == toset(["stable"]) + && tomap(local.normalized_config.tags) == tomap(var.tags) + && local.normalized_config.roles.path == var.role_path + && local.normalized_config.roles.permissions_boundary == var.role_permissions_boundary + && local.normalized_config.github.app.key_base64_ssm == var.github_app.key_base64_ssm + && local.normalized_config.github.app.id_ssm == var.github_app.id_ssm + && local.normalized_config.github.app.webhook_secret_ssm == var.github_app.webhook_secret_ssm + && local.normalized_config.github.user_agent == var.user_agent + && jsonencode(local.normalized_config.github.additional_apps) == jsonencode(var.additional_github_apps) + && local.normalized_config.github.enterprise_server.url == var.ghes_url + && local.normalized_config.github.enterprise_server.ssl_verify == var.ghes_ssl_verify + && local.normalized_config.lambda.runtime == var.lambda_runtime + && local.normalized_config.lambda.artifact.s3.bucket == var.lambda_s3_bucket + && local.normalized_config.lambda.architecture == var.lambda_architecture + && local.normalized_config.orchestration_provider.webhook.queue_selection_strategy == var.queue_selection_strategy + && local.normalized_config.orchestration_provider.webhook.eventbridge.enabled == var.eventbridge.enable + && tolist(local.normalized_config.orchestration_provider.webhook.eventbridge.accept_events) == tolist(var.eventbridge.accept_events) + && local.normalized_config.orchestration_provider.webhook.matcher_config_parameter_store_tier == var.matcher_config_parameter_store_tier + && tolist(local.normalized_config.orchestration_provider.webhook.github.repository_white_list) == tolist(var.repository_white_list) + && local.normalized_config.orchestration_provider.webhook.lambda.scale.up.memory_size == var.scale_up_lambda_memory_size + && local.normalized_config.orchestration_provider.webhook.lambda.scale.down.timeout == var.runners_scale_down_lambda_timeout + && local.normalized_config.orchestration_provider.webhook.lambda.webhook.memory_size == var.webhook_lambda_memory_size + && local.normalized_config.orchestration_provider.webhook.lambda.pool.timeout == var.pool_lambda_timeout + && jsonencode(local.normalized_config.orchestration_provider.webhook.queue.encryption) == jsonencode(var.queue_encryption) + && local.normalized_config.ssm.paths.root == "/${var.ssm_paths.root}/${var.prefix}" + && local.normalized_config.ssm.paths.tokens == "${var.ssm_paths.runners}/tokens" + && local.normalized_config.ssm.kms_key_id == var.kms_key_arn + && local.normalized_config.ssm.housekeeper.state == "DISABLED" + && local.normalized_config.ssm.housekeeper.config.minimumDaysOld == var.runners_ssm_housekeeper.config.minimumDaysOld + && local.normalized_config.observability.logs.level == var.log_level + && local.normalized_config.observability.logs.retention_in_days == var.logging_retention_in_days + && local.normalized_config.observability.logs.kms_key_id == var.logging_kms_key_id + && jsonencode(local.normalized_config.observability.tracing) == jsonencode(var.tracing_config) + && local.normalized_config.observability.metrics.namespace == var.metrics.namespace + && local.normalized_config.compute_provider.aws.ec2.vpc_id == var.vpc_id + && tolist(local.normalized_config.compute_provider.aws.ec2.subnet_ids) == tolist(var.subnet_ids) + && local.normalized_config.compute_provider.aws.ec2.managed_security_group_enabled == var.enable_managed_runner_security_group + && jsonencode(local.normalized_config.compute_provider.aws.ec2.egress_rules) == jsonencode(var.runner_egress_rules) + && jsonencode(local.normalized_config.compute_provider.aws.ec2.additional_security_group_ids) == jsonencode(var.runner_additional_security_group_ids) + && local.normalized_config.compute_provider.aws.ec2.cloudwatch_agent.config == var.cloudwatch_config + && local.normalized_config.compute_provider.aws.ec2.instance_profile_path == var.instance_profile_path + && local.normalized_config.compute_provider.aws.ec2.key_name == var.key_name + && local.normalized_config.compute_provider.aws.ec2.associate_public_ipv4_address == var.associate_public_ipv4_address + ) + error_message = "An empty experimental runner map must translate stable global inputs across every canonical section." + } + + assert { + condition = ( + local.stable_to_experimental.tags.source == var.tags.source + && local.stable_to_experimental.roles.path == var.role_path + && local.stable_to_experimental.github.user_agent == var.user_agent + && local.stable_to_experimental.lambda.artifact.s3.bucket == var.lambda_s3_bucket + && local.stable_to_experimental.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.batch_size == var.lambda_event_source_mapping_batch_size + && local.stable_to_experimental.orchestration_provider.webhook.lambda.scale.down.idle_config == [] + && local.stable_to_experimental.ssm.parameters.tags.owner == var.parameter_store_tags.owner + && local.stable_to_experimental.ssm.housekeeper.lambda.memory_size == var.runners_ssm_housekeeper.lambda_memory_size + && local.stable_to_experimental.compute_provider.aws.ec2.runner_binaries.s3.encryption.sse_algorithm == "aws:kms" + && local.stable_to_experimental.compute_provider.aws.ec2.runner_binaries.s3.encryption.kms_master_key_id == "arn:aws:kms:eu-west-1:123456789012:key/binaries" + && local.stable_to_experimental.compute_provider.aws.ec2.instance_termination_watcher.enabled == var.instance_termination_watcher.enable + ) + error_message = "The stable-to-experimental adapter must preserve nested legacy values without relying on the selector." + } + + assert { + condition = ( + local.normalized_config.multi_runner_config["stable"].runner.os == "linux" + && local.normalized_config.multi_runner_config["stable"].runner.architecture == "x64" + && local.normalized_config.multi_runner_config["stable"].runner.group_name == "stable-group" + && local.normalized_config.multi_runner_config["stable"].runner.iam.managed_policy_arns["legacy-0"] == "arn:aws:iam::123456789012:policy/stable-runner" + && local.normalized_config.multi_runner_config["stable"].orchestration_provider.webhook.runner.maximum_count == 2 + && jsonencode(local.normalized_config.multi_runner_config["stable"].orchestration_provider.webhook.matcherConfig.labelMatchers) == jsonencode([["self-hosted", "linux", "x64"]]) + && toset(local.normalized_config.multi_runner_config["stable"].compute_provider.aws.ec2.instance_types) == toset(["m5.large"]) + ) + error_message = "Stable runner entries must translate into the canonical runner, orchestration, and compute-provider blocks." + } +} + +run "non_empty_experimental_map_is_authoritative" { + command = plan + + variables { + tags = { + source = "stable-ignored" + } + + multi_runner_config = { + stable = { + runner_config = { + runner_os = "linux" + runner_architecture = "x64" + instance_types = ["m5.large"] + runners_maximum_count = 1 + enable_runner_binaries_syncer = false + } + matcherConfig = { + labelMatchers = [["stable"]] + } + } + } + + experimental_global_config = { + tags = { + source = "experimental" + } + runner = { + os = "linux" + architecture = "arm64" + } + } + + experimental_global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-experimental-default" + subnet_ids = ["subnet-experimental-default"] + runner_binaries = { + syncer = { + artifact = { + s3 = { + key = "runner-binaries-syncer.zip" + } + } + } + } + } + } + } + + experimental_multi_runner_config = { + experimental = { + orchestration_provider = { + webhook = { + matcherConfig = { + labelMatchers = [["experimental"]] + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = ["c7g.large"] + } + } + } + } + } + } + + assert { + condition = ( + local.use_v2_config + && toset(keys(local.normalized_config.multi_runner_config)) == toset(["experimental"]) + && local.normalized_config.tags.source == "experimental" + && toset(local.normalized_config.multi_runner_config["experimental"].compute_provider.aws.ec2.instance_types) == toset(["c7g.large"]) + && flatten(local.normalized_config.multi_runner_config["experimental"].orchestration_provider.webhook.matcherConfig.labelMatchers) == ["experimental"] + ) + error_message = "A non-empty experimental runner map must be authoritative and must not merge stable lanes or flat defaults." + } + + assert { + condition = jsonencode(local.normalized_config) == jsonencode(local.experimental) + error_message = "A non-empty experimental runner map must select the experimental object without leaking stable flat inputs." + } + +} + +run "lane_values_override_experimental_globals" { + command = plan + + variables { + experimental_global_config = { + tags = { + scope = "global" + precedence = "global" + } + + runner = { + os = "linux" + architecture = "x64" + group_name = "global-group" + iam = { + managed_policy_arns = { + global = "arn:aws:iam::123456789012:policy/global-runner" + } + additional_trust_policy_json = "{}" + } + } + } + + experimental_global_config_orchestration_provider = { + webhook = { + runner = { + maximum_count = 4 + } + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + webhook = { + artifact = { + s3 = { + key = "webhook.zip" + } + } + } + } + } + } + + experimental_global_config_observability = { + logs = { + level = "debug" + retention_in_days = 30 + } + } + + experimental_global_config_ssm = { + housekeeper = { + lambda = { + artifact = { + s3 = { + key = "global-housekeeper.zip" + } + } + } + } + } + + experimental_global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-experimental" + subnet_ids = ["subnet-global"] + runner_binaries = { + syncer = { + artifact = { + s3 = { + key = "runner-binaries-syncer.zip" + } + } + } + } + tags = { + precedence = "global" + global = "true" + } + } + } + } + + experimental_multi_runner_config = { + lane = { + tags = { + precedence = "lane" + lane = "true" + } + runner = { + group_name = "lane-group" + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external-runner" + } + } + } + orchestration_provider = { + webhook = { + runner = { + maximum_count = 7 + } + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64", "lane"]] + } + } + } + observability = { + logs = { + level = "warn" + } + } + ssm = { + housekeeper = { + lambda = { + artifact = { + s3 = { + key = "lane-housekeeper.zip" + } + } + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = ["m7i.large"] + subnet_ids = ["subnet-lane"] + instance_profile = { + name = "lane-runner-profile" + } + tags = { + precedence = "lane" + provider = "lane" + } + } + } + } + } + } + } + + assert { + condition = ( + local.resolved_config.multi_runner_config["lane"].runner.os == "linux" + && local.resolved_config.multi_runner_config["lane"].runner.architecture == "x64" + && local.resolved_config.multi_runner_config["lane"].runner.group_name == "lane-group" + && local.resolved_config.multi_runner_config["lane"].orchestration_provider.webhook.runner.maximum_count == 7 + && local.resolved_config.multi_runner_config["lane"].observability.logs.level == "warn" + && local.resolved_config.multi_runner_config["lane"].observability.logs.retention_in_days == 30 + && local.resolved_config.multi_runner_config["lane"].ssm.housekeeper.lambda.artifact.zip == null + && local.resolved_config.multi_runner_config["lane"].ssm.housekeeper.lambda.artifact.s3.key == "lane-housekeeper.zip" + ) + error_message = "Lane values must override experimental globals while omitted values inherit their global defaults." + } + + assert { + condition = ( + tomap(local.resolved_config.multi_runner_config["lane"].tags) == tomap({ + scope = "global" + precedence = "lane" + lane = "true" + }) + && local.resolved_config.multi_runner_config["lane"].compute_provider.aws.ec2.vpc_id == "vpc-experimental" + && toset(local.resolved_config.multi_runner_config["lane"].compute_provider.aws.ec2.subnet_ids) == toset(["subnet-lane"]) + && tomap(local.resolved_config.multi_runner_config["lane"].compute_provider.aws.ec2.tags) == tomap({ + precedence = "lane" + global = "true" + provider = "lane" + }) + ) + error_message = "Tags and EC2 defaults must merge from experimental globals with lane values taking precedence." + } + + assert { + condition = ( + local.resolved_config.multi_runner_config["lane"].runner.iam.role.arn == "arn:aws:iam::123456789012:role/external-runner" + && length(local.resolved_config.multi_runner_config["lane"].runner.iam.managed_policy_arns) == 0 + && local.resolved_config.multi_runner_config["lane"].runner.iam.additional_trust_policy_json == null + ) + error_message = "An externally managed runner role must suppress inherited managed policies and trust-policy additions." + } +} + +run "global_external_runner_role_suppresses_inherited_iam_overrides" { + command = plan + + variables { + experimental_global_config = { + runner = { + os = "linux" + architecture = "x64" + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/global-external-runner" + } + managed_policy_arns = { + global = "arn:aws:iam::123456789012:policy/global-runner" + } + additional_trust_policy_json = "{\"Version\":\"2012-10-17\"}" + } + } + } + + experimental_global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-global" + subnet_ids = ["subnet-global"] + runner_binaries = { + syncer = { + artifact = { + s3 = { + key = "runner-binaries-syncer.zip" + } + } + } + } + } + } + } + + experimental_multi_runner_config = { + lane = { + orchestration_provider = { + webhook = { + matcherConfig = { + labelMatchers = [["self-hosted"]] + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = ["m5.large"] + instance_profile = { + name = "global-runner-profile" + } + } + } + } + } + } + } + + assert { + condition = ( + local.resolved_config.multi_runner_config["lane"].runner.iam.role.arn == "arn:aws:iam::123456789012:role/global-external-runner" + && length(local.resolved_config.multi_runner_config["lane"].runner.iam.managed_policy_arns) == 0 + && local.resolved_config.multi_runner_config["lane"].runner.iam.additional_trust_policy_json == null + ) + error_message = "A global external runner role must suppress inherited managed policies and trust-policy additions." + } +} diff --git a/modules/multi-runner/validations.tf b/modules/multi-runner/validations.tf new file mode 100644 index 0000000000..a738297e53 --- /dev/null +++ b/modules/multi-runner/validations.tf @@ -0,0 +1,82 @@ +locals { + common_validation_errors = concat( + alltrue([ + for app in var.additional_github_apps : + (app.key_base64 != null || app.key_base64_ssm != null) && + (app.id != null || app.id_ssm != null) + ]) ? [] : ["Each additional GitHub app must provide either key_base64 or key_base64_ssm, and either id or id_ssm."], + contains(["STANDARD", "INFREQUENT_ACCESS"], var.log_class) ? [] : ["`log_class` must be either `STANDARD` or `INFREQUENT_ACCESS`."], + contains(["first", "random", "all"], var.queue_selection_strategy) ? [] : ["`queue_selection_strategy` value not valid. Valid values are 'first', 'random', 'all'."], + contains(["silly", "trace", "debug", "info", "warn", "error", "fatal"], var.log_level) ? [] : ["`log_level` value not valid. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'."], + contains(["arm64", "x86_64"], var.lambda_architecture) ? [] : ["`lambda_architecture` value is not valid, valid values are: `arm64` and `x86_64`."], + contains(["ENABLED", "DISABLED", "ENABLED_WITH_ALL_CLOUDTRAIL_MANAGEMENT_EVENTS"], var.state_event_rule_binaries_syncer) ? [] : ["`state_event_rule_binaries_syncer` value is not valid, valid values are: `ENABLED`, `DISABLED`, `ENABLED_WITH_ALL_CLOUDTRAIL_MANAGEMENT_EVENTS`."], + var.queue_encryption == null || var.queue_encryption.sqs_managed_sse_enabled != null && var.queue_encryption.kms_master_key_id == null && var.queue_encryption.kms_data_key_reuse_period_seconds == null || var.queue_encryption.sqs_managed_sse_enabled == null && var.queue_encryption.kms_master_key_id != null ? [] : ["Invalid configuration for `queue_encryption`. Valid configurations are encryption disabled, enabled via SSE. Or encryption via KMS."], + contains(["Standard", "Advanced"], var.matcher_config_parameter_store_tier) ? [] : ["`matcher_config_parameter_store_tier` value is not valid, valid values are: `Standard`, and `Advanced`."], + !var.iam_overrides.override_instance_profile || var.iam_overrides.instance_profile_name != null ? [] : ["instance_profile_name must be provided when override_instance_profile is true."], + !var.iam_overrides.override_runner_role || var.iam_overrides.runner_role_arn != null ? [] : ["runner_role_arn must be provided when override_runner_role is true."] + ) +} + +resource "terraform_data" "validate_v1" { + count = local.use_v2_config ? 0 : 1 + + lifecycle { + precondition { + condition = length(local.common_validation_errors) == 0 + error_message = join("\n", local.common_validation_errors) + } + + precondition { + condition = ( + (var.github_app.key_base64 != null || var.github_app.key_base64_ssm != null) && + (var.github_app.id != null || var.github_app.id_ssm != null) && + (var.github_app.webhook_secret != null || var.github_app.webhook_secret_ssm != null) && + var.vpc_id != null && + var.subnet_ids != null && + length(var.multi_runner_config) > 0 + ) + error_message = "Stable v1 configuration requires github_app, vpc_id, subnet_ids, and multi_runner_config." + } + } +} + +resource "terraform_data" "validate_v2" { + count = local.use_v2_config ? 1 : 0 + + lifecycle { + precondition { + condition = length(local.common_validation_errors) == 0 + error_message = join("\n", local.common_validation_errors) + } + + precondition { + condition = ( + ( + try(var.experimental_global_config_github.app.key_base64, null) != null || + try(var.experimental_global_config_github.app.key_base64_ssm, null) != null + ) && ( + try(var.experimental_global_config_github.app.id, null) != null || + try(var.experimental_global_config_github.app.id_ssm, null) != null + ) && ( + try(var.experimental_global_config_github.app.webhook_secret, null) != null || + try(var.experimental_global_config_github.app.webhook_secret_ssm, null) != null + ) + ) + error_message = "Experimental v2 configuration requires a complete GitHub App under experimental_global_config_github.app." + } + + precondition { + condition = alltrue([ + for config in local.resolved_config.multi_runner_config : ( + try(config.orchestration_provider.webhook != null, false) && + try(length(config.orchestration_provider.webhook.matcherConfig.labelMatchers) > 0, false) && + try(config.compute_provider.aws.ec2 != null, false) && + try(length(config.compute_provider.aws.ec2.instance_types) > 0, false) && + try(config.compute_provider.aws.ec2.vpc_id != null, false) && + try(length(config.compute_provider.aws.ec2.subnet_ids) > 0, false) + ) + ]) + error_message = "Each experimental v2 runner lane requires a webhook matcher, EC2 instance_types, vpc_id, and at least one subnet." + } + } +} diff --git a/modules/multi-runner/variables.experimental.compute-provider.tf b/modules/multi-runner/variables.experimental.compute-provider.tf new file mode 100644 index 0000000000..9bd4ea570d --- /dev/null +++ b/modules/multi-runner/variables.experimental.compute-provider.tf @@ -0,0 +1,137 @@ +# Experimental global compute-provider configuration. +variable "experimental_global_config_compute_provider" { + description = "Experimental global compute-provider configuration." + type = object({ + selections = optional(map(object({ + namespace = string + type = string + })), null) + aws = optional(object({ + ec2 = optional(object({ + vpc_id = optional(string, null) + subnet_ids = optional(list(string), null) + managed_security_group_enabled = optional(bool, true) + egress_rules = optional(list(object({ + cidr_blocks = list(string) + ipv6_cidr_blocks = list(string) + prefix_list_ids = list(string) + from_port = number + protocol = string + security_groups = list(string) + self = bool + to_port = number + description = string + })), [{ + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + prefix_list_ids = null + from_port = 0 + protocol = "-1" + security_groups = null + self = null + to_port = 0 + description = null + }]) + additional_security_group_ids = optional(list(string), []) + cloudwatch_agent = optional(object({ + config = optional(string, null) + }), {}) + instance_profile_path = optional(string, null) + key_name = optional(string, null) + associate_public_ipv4_address = optional(bool, false) + tags = optional(map(string), {}) + ami = optional(object({ + housekeeper = optional(object({ + enabled = optional(bool, false) + cleanup_config = optional(object({ + maxItems = optional(number) + minimumDaysOld = optional(number) + amiFilters = optional(list(object({ + Name = string + Values = list(string) + }))) + launchTemplateNames = optional(list(string)) + ssmParameterNames = optional(list(string)) + dryRun = optional(bool) + }), {}) + artifact = optional(object({ + zip = optional(string, null) + s3 = optional(object({ + key = string + object_version = optional(string, null) + }), null) + }), {}) + lambda = optional(object({ + memory_size = optional(number, 256) + timeout = optional(number, 300) + }), {}) + schedule = optional(object({ + expression = optional(string, "cron(11 7 * * ? *)") + }), {}) + }), {}) + }), {}) + instance_termination_watcher = optional(object({ + enabled = optional(bool, false) + features = optional(object({ + runner_deregistration = optional(object({ + enabled = optional(bool, true) + }), {}) + spot_termination_handler = optional(object({ + enabled = optional(bool, true) + }), {}) + spot_termination_notification_watcher = optional(object({ + enabled = optional(bool, true) + }), {}) + }), {}) + environment_variables = optional(map(string), {}) + artifact = optional(object({ + zip = optional(string, null) + s3 = optional(object({ + key = string + object_version = optional(string, null) + }), null) + }), {}) + lambda = optional(object({ + memory_size = optional(number, null) + timeout = optional(number, null) + }), {}) + }), {}) + runner_binaries = optional(object({ + enabled = optional(bool, true) + s3 = optional(object({ + encryption = optional(object({ + enabled = optional(bool, true) + bucket_key_enabled = optional(bool, null) + sse_algorithm = optional(string, "AES256") + kms_master_key_id = optional(string, null) + }), {}) + tags = optional(map(string), {}) + versioning = optional(string, "Disabled") + logging = optional(object({ + bucket = optional(string, null) + prefix = optional(string, null) + }), {}) + }), {}) + syncer = optional(object({ + artifact = optional(object({ + zip = optional(string, null) + s3 = optional(object({ + key = string + object_version = optional(string, null) + }), null) + }), {}) + lambda = optional(object({ + memory_size = optional(number, 256) + timeout = optional(number, 300) + }), {}) + schedule = optional(object({ + expression = optional(string, "cron(27 * * * ? *)") + state = optional(string, "ENABLED") + }), {}) + }), {}) + }), {}) + }), {}) + }), {}) + }) + default = {} +} diff --git a/modules/multi-runner/variables.experimental.github.tf b/modules/multi-runner/variables.experimental.github.tf new file mode 100644 index 0000000000..605e1b58de --- /dev/null +++ b/modules/multi-runner/variables.experimental.github.tf @@ -0,0 +1,37 @@ +# Experimental global GitHub configuration. +variable "experimental_global_config_github" { + description = "Experimental global GitHub configuration." + type = object({ + app = optional(object({ + key_base64 = optional(string) + key_base64_ssm = optional(object({ + arn = string + name = string + })) + id = optional(string) + id_ssm = optional(object({ + arn = string + name = string + })) + webhook_secret = optional(string) + webhook_secret_ssm = optional(object({ + arn = string + name = string + })) + }), null) + additional_apps = optional(list(object({ + key_base64 = optional(string) + key_base64_ssm = optional(object({ arn = string, name = string })) + id = optional(string) + id_ssm = optional(object({ arn = string, name = string })) + installation_id = optional(string) + installation_id_ssm = optional(object({ arn = string, name = string })) + })), []) + enterprise_server = optional(object({ + url = optional(string, null) + ssl_verify = optional(bool, true) + }), {}) + user_agent = optional(string, "github-aws-runners") + }) + default = {} +} diff --git a/modules/multi-runner/variables.experimental.global.tf b/modules/multi-runner/variables.experimental.global.tf new file mode 100644 index 0000000000..db030ab4d4 --- /dev/null +++ b/modules/multi-runner/variables.experimental.global.tf @@ -0,0 +1,39 @@ +# Experimental global defaults shared by all runner lanes. +variable "experimental_global_config" { + description = "Experimental global defaults shared by all runner lanes." + type = object({ + tags = optional(map(string), {}) + + roles = optional(object({ + path = optional(string, null) + permissions_boundary = optional(string, null) + }), {}) + + runner = optional(object({ + os = optional(string, null) + architecture = optional(string, null) + disable_default_labels = optional(bool, false) + extra_labels = optional(list(string), []) + group_name = optional(string, "Default") + name_prefix = optional(string, "") + run_as_root = optional(bool, false) + run_as = optional(string, "ec2-user") + auto_update_disabled = optional(bool, false) + tags = optional(map(string), {}) + hooks = optional(object({ + job_started = optional(string, "") + job_completed = optional(string, "") + }), {}) + iam = optional(object({ + role = optional(object({ + arn = string + }), null) + managed_policy_arns = optional(map(string), {}) + additional_trust_policy_json = optional(string, null) + path = optional(string, null) + permissions_boundary = optional(string, null) + }), {}) + }), {}) + }) + default = {} +} diff --git a/modules/multi-runner/variables.experimental.lambda.tf b/modules/multi-runner/variables.experimental.lambda.tf new file mode 100644 index 0000000000..6996cf4e51 --- /dev/null +++ b/modules/multi-runner/variables.experimental.lambda.tf @@ -0,0 +1,25 @@ +# Experimental global Lambda configuration. +variable "experimental_global_config_lambda" { + description = "Experimental global Lambda configuration." + type = object({ + artifact = optional(object({ + s3 = optional(object({ + bucket = optional(string, null) + }), {}) + }), {}) + runtime = optional(string, "nodejs24.x") + architecture = optional(string, "arm64") + principals = optional(list(object({ + type = string + identifiers = list(string) + })), []) + subnet_ids = optional(list(string), []) + security_group_ids = optional(list(string), []) + tags = optional(map(string), {}) + role = optional(object({ + path = optional(string, null) + permissions_boundary = optional(string, null) + }), {}) + }) + default = {} +} diff --git a/modules/multi-runner/variables.experimental.multi-runner.tf b/modules/multi-runner/variables.experimental.multi-runner.tf new file mode 100644 index 0000000000..7a7246bb73 --- /dev/null +++ b/modules/multi-runner/variables.experimental.multi-runner.tf @@ -0,0 +1,333 @@ +# Experimental per-runner and per-lane overrides. +variable "experimental_multi_runner_config" { + description = "Experimental per-runner and per-lane overrides." + type = map(object({ + tags = optional(map(string), {}) + + runner = optional(object({ + os = optional(string, null) + architecture = optional(string, null) + disable_default_labels = optional(bool, null) + extra_labels = optional(list(string), null) + group_name = optional(string, null) + name_prefix = optional(string, null) + run_as_root = optional(bool, null) + run_as = optional(string, null) + auto_update_disabled = optional(bool, null) + tags = optional(map(string), {}) + hooks = optional(object({ + job_started = optional(string, null) + job_completed = optional(string, null) + }), {}) + iam = optional(object({ + role = optional(object({ + arn = string + }), null) + managed_policy_arns = optional(map(string), null) + additional_trust_policy_json = optional(string, null) + path = optional(string, null) + permissions_boundary = optional(string, null) + }), {}) + }), {}) + + lambda = optional(object({ + runtime = optional(string, null) + architecture = optional(string, null) + subnet_ids = optional(list(string), null) + security_group_ids = optional(list(string), null) + tags = optional(map(string), {}) + role = optional(object({ + path = optional(string, null) + permissions_boundary = optional(string, null) + }), {}) + }), {}) + + orchestration_provider = object({ + webhook = optional(object({ + runner = optional(object({ + boot_time_in_minutes = optional(number, null) + ephemeral = optional(bool, null) + jit_config_enabled = optional(bool, null) + maximum_count = optional(number, null) + }), {}) + + github = optional(object({ + organization_runners = optional(bool, false) + }), {}) + + matcherConfig = object({ + labelMatchers = list(list(string)) + exactMatch = optional(bool, false) + bidirectionalLabelMatch = optional(bool, false) + priority = optional(number, 999) + dynamic_labels_enabled = optional(bool, false) + awsDynamicLabelsPolicy = optional(object({ + blocked_keys = optional(list(string), []) + restricted_keys = optional(map(object({ + allowed = optional(list(string), []) + denied = optional(list(string), []) + max = optional(string, null) + })), {}) + }), null) + }) + + queue = optional(object({ + delay_webhook_event = optional(number, null) + job_queue_retention_in_seconds = optional(number, null) + visibility_timeout_seconds = optional(number, null) + redrive_build_queue = optional(object({ + enabled = optional(bool, null) + maxReceiveCount = optional(number, null) + }), null) + tags = optional(map(string), {}) + }), {}) + + lambda = optional(object({ + scale = optional(object({ + up = optional(object({ + memory_size = optional(number, null) + timeout = optional(number, null) + reserved_concurrent_executions = optional(number, null) + job_queued_check_enabled = optional(bool, null) + event_source_mapping = optional(object({ + batch_size = optional(number, null) + maximum_batching_window_in_seconds = optional(number, null) + }), {}) + tags = optional(map(string), {}) + }), {}) + down = optional(object({ + memory_size = optional(number, null) + timeout = optional(number, null) + schedule_expression = optional(string, null) + minimum_running_time_in_minutes = optional(number, null) + idle_config = optional(list(object({ + cron = string + timeZone = string + idleCount = number + evictionStrategy = optional(string, "oldest_first") + })), null) + tags = optional(map(string), {}) + }), {}) + }), {}) + pool = optional(object({ + memory_size = optional(number, null) + timeout = optional(number, null) + reserved_concurrent_executions = optional(number, null) + config = optional(list(object({ + schedule_expression = string + schedule_expression_timezone = optional(string) + size = number + })), null) + include_busy_runners = optional(bool, null) + runner_owner = optional(string, null) + tags = optional(map(string), {}) + }), {}) + }), {}) + + job_retry = optional(object({ + enabled = optional(bool, false) + delay_in_seconds = optional(number, 300) + delay_backoff = optional(number, 2) + max_attempts = optional(number, 1) + tags = optional(map(string), {}) + lambda = optional(object({ + memory_size = optional(number, 256) + reserved_concurrent_executions = optional(number, 1) + timeout = optional(number, 30) + }), {}) + }), {}) + + }), null) + + }) + + ssm = optional(object({ + paths = optional(object({ + root = optional(string, null) + tokens = optional(string, null) + config = optional(string, null) + }), {}) + tags = optional(map(string), {}) + parameters = optional(object({ + tags = optional(map(string), {}) + }), {}) + housekeeper = optional(object({ + schedule_expression = optional(string, null) + state = optional(string, null) + tags = optional(map(string), {}) + lambda = optional(object({ + artifact = optional(object({ + zip = optional(string, null) + s3 = optional(object({ + key = string + object_version = optional(string, null) + }), null) + }), {}) + memory_size = optional(number, null) + timeout = optional(number, null) + }), {}) + config = optional(object({ + tokenPath = optional(string, null) + minimumDaysOld = optional(number, null) + dryRun = optional(bool, null) + }), {}) + }), {}) + }), {}) + + observability = optional(object({ + logs = optional(object({ + level = optional(string, null) + retention_in_days = optional(number, null) + kms_key_id = optional(string, null) + class = optional(string, null) + tags = optional(map(string), {}) + }), {}) + tracing = optional(object({ + mode = optional(string, null) + capture_http_requests = optional(bool, null) + capture_error = optional(bool, null) + }), {}) + metrics = optional(object({ + enabled = optional(bool, null) + namespace = optional(string, null) + metric = optional(object({ + github_app_rate_limit = optional(object({ + enabled = optional(bool, null) + }), {}) + job_retry = optional(object({ + enabled = optional(bool, null) + }), {}) + spot_termination_warning = optional(object({ + enabled = optional(bool, null) + }), {}) + }), {}) + }), {}) + }), {}) + + compute_provider = object({ + aws = optional(object({ + ec2 = optional(object({ + metadata_options = optional(object({ + instance_metadata_tags = optional(string, "enabled") + http_endpoint = optional(string, "enabled") + http_tokens = optional(string, "required") + http_put_response_hop_limit = optional(number, 1) + }), {}) + ami = optional(object({ + filter = optional(map(list(string)), { state = ["available"] }) + owners = optional(list(string), ["amazon"]) + id_ssm_parameter = optional(object({ + arn = string + }), null) + kms_key = optional(object({ + arn = string + }), null) + }), null) + block_device_mappings = optional(list(object({ + delete_on_termination = optional(bool, true) + device_name = optional(string, "/dev/xvda") + encrypted = optional(bool, true) + iops = optional(number) + kms_key_id = optional(string) + snapshot_id = optional(string) + throughput = optional(number) + volume_initialization_rate = optional(number) + volume_size = number + volume_type = optional(string, "gp3") + })), [{ + volume_size = 30 + }]) + create_service_linked_role_spot = optional(bool, false) + credit_specification = optional(string, null) + ebs_optimized = optional(bool, false) + cloudwatch_agent = optional(object({ + enabled = optional(bool, true) + config = optional(string, null) + }), {}) + binaries_syncer = optional(object({ + enabled = optional(bool, null) + }), {}) + detailed_monitoring_enabled = optional(bool, false) + ssm_enabled = optional(bool, false) + user_data = optional(object({ + enabled = optional(bool, true) + template = optional(string, null) + content = optional(string, null) + pre_install = optional(string, "") + post_install = optional(string, "") + debug_logging_enabled = optional(bool, false) + }), {}) + instance_allocation_strategy = optional(string, "lowest-price") + instance_max_spot_price = optional(string, null) + instance_target_capacity_type = optional(string, "spot") + instance_type_priorities = optional(map(number), null) + instance_types = list(string) + additional_security_group_ids = optional(list(string), null) + managed_security_group_enabled = optional(bool, null) + egress_rules = optional(list(object({ + cidr_blocks = list(string) + ipv6_cidr_blocks = list(string) + prefix_list_ids = list(string) + from_port = number + protocol = string + security_groups = list(string) + self = bool + to_port = number + description = string + })), null) + instance_profile_path = optional(string, null) + key_name = optional(string, null) + associate_public_ipv4_address = optional(bool, null) + instance_profile = optional(object({ + name = string + }), null) + on_demand_failover_for_errors = optional(list(string), []) + scale_errors = optional(list(string), [ + "UnfulfillableCapacity", + "MaxSpotInstanceCountExceeded", + "TargetCapacityLimitExceededException", + "RequestLimitExceeded", + "ResourceLimitExceeded", + "MaxSpotInstanceCountExceeded", + "MaxSpotFleetRequestCountExceeded", + "InsufficientInstanceCapacity", + "InsufficientCapacityOnHost", + ]) + subnet_ids = optional(list(string), null) + vpc_id = optional(string, null) + cpu_options = optional(object({ + core_count = optional(number) + threads_per_core = optional(number) + amd_sev_snp = optional(string) + nested_virtualization = optional(string) + }), null) + placement = optional(object({ + affinity = optional(string) + availability_zone = optional(string) + group_id = optional(string) + group_name = optional(string) + host_id = optional(string) + host_resource_group_arn = optional(string) + spread_domain = optional(string) + tenancy = optional(string) + partition_number = optional(number) + }), null) + license_specifications = optional(list(object({ + license_configuration_arn = string + })), []) + use_dedicated_host = optional(bool, false) + log_files = optional(list(object({ + log_group_name = string + prefix_log_group = bool + file_path = string + log_stream_name = string + log_class = optional(string, "STANDARD") + })), null) + tags = optional(map(string), {}) + }), null) + }), {}) + }) + + })) + default = {} +} diff --git a/modules/multi-runner/variables.experimental.observability.tf b/modules/multi-runner/variables.experimental.observability.tf new file mode 100644 index 0000000000..ba251d16b6 --- /dev/null +++ b/modules/multi-runner/variables.experimental.observability.tf @@ -0,0 +1,34 @@ +# Experimental global observability configuration. +variable "experimental_global_config_observability" { + description = "Experimental global observability configuration." + type = object({ + logs = optional(object({ + level = optional(string, "info") + retention_in_days = optional(number, 180) + kms_key_id = optional(string, null) + class = optional(string, "STANDARD") + tags = optional(map(string), {}) + }), {}) + tracing = optional(object({ + mode = optional(string, null) + capture_http_requests = optional(bool, false) + capture_error = optional(bool, false) + }), {}) + metrics = optional(object({ + enabled = optional(bool, false) + namespace = optional(string, "GitHub Runners") + metric = optional(object({ + github_app_rate_limit = optional(object({ + enabled = optional(bool, true) + }), {}) + job_retry = optional(object({ + enabled = optional(bool, true) + }), {}) + spot_termination_warning = optional(object({ + enabled = optional(bool, true) + }), {}) + }), {}) + }), {}) + }) + default = {} +} diff --git a/modules/multi-runner/variables.experimental.orchestration-provider.tf b/modules/multi-runner/variables.experimental.orchestration-provider.tf new file mode 100644 index 0000000000..b077e6dd9b --- /dev/null +++ b/modules/multi-runner/variables.experimental.orchestration-provider.tf @@ -0,0 +1,113 @@ +# Experimental global orchestration-provider configuration. +variable "experimental_global_config_orchestration_provider" { + description = "Experimental global orchestration-provider configuration." + type = object({ + webhook = optional(object({ + queue_selection_strategy = optional(string, "first") + eventbridge = optional(object({ + enabled = optional(bool, true) + accept_events = optional(list(string), []) + }), {}) + matcher_config_parameter_store_tier = optional(string, "Standard") + runner = optional(object({ + boot_time_in_minutes = optional(number, 5) + ephemeral = optional(bool, false) + jit_config_enabled = optional(bool, null) + maximum_count = optional(number, null) + }), {}) + + github = optional(object({ + repository_white_list = optional(list(string), []) + }), {}) + + lambda = optional(object({ + artifact = optional(object({ + zip = optional(string, null) + s3 = optional(object({ + key = string + object_version = optional(string, null) + }), null) + }), {}) + scale = optional(object({ + up = optional(object({ + memory_size = optional(number, 512) + timeout = optional(number, 30) + reserved_concurrent_executions = optional(number, 1) + job_queued_check_enabled = optional(bool, null) + event_source_mapping = optional(object({ + batch_size = optional(number, 10) + maximum_batching_window_in_seconds = optional(number, 0) + }), {}) + tags = optional(map(string), {}) + }), {}) + down = optional(object({ + memory_size = optional(number, 512) + timeout = optional(number, 60) + schedule_expression = optional(string, "cron(*/5 * * * ? *)") + minimum_running_time_in_minutes = optional(number, null) + idle_config = optional(list(object({ + cron = string + timeZone = string + idleCount = number + evictionStrategy = optional(string, "oldest_first") + })), []) + tags = optional(map(string), {}) + }), {}) + }), {}) + webhook = optional(object({ + artifact = optional(object({ + zip = optional(string, null) + s3 = optional(object({ + key = string + object_version = optional(string, null) + }), null) + }), {}) + api_gateway_access_log_settings = optional(object({ + destination_arn = string + format = string + }), null) + memory_size = optional(number, 256) + timeout = optional(number, 10) + tags = optional(map(string), {}) + }), {}) + pool = optional(object({ + memory_size = optional(number, 512) + timeout = optional(number, 60) + reserved_concurrent_executions = optional(number, 1) + config = optional(list(object({ + schedule_expression = string + schedule_expression_timezone = optional(string) + size = number + })), []) + include_busy_runners = optional(bool, false) + runner_owner = optional(string, null) + tags = optional(map(string), {}) + }), {}) + }), {}) + + queue = optional(object({ + delay_webhook_event = optional(number, 30) + job_queue_retention_in_seconds = optional(number, 86400) + visibility_timeout_seconds = optional(number, 180) + redrive_build_queue = optional(object({ + enabled = optional(bool, false) + maxReceiveCount = optional(number, null) + }), { + enabled = false + maxReceiveCount = null + }) + tags = optional(map(string), {}) + encryption = optional(object({ + kms_data_key_reuse_period_seconds = number + kms_master_key_id = string + sqs_managed_sse_enabled = bool + }), { + kms_data_key_reuse_period_seconds = null + kms_master_key_id = null + sqs_managed_sse_enabled = true + }) + }), {}) + }), {}) + }) + default = {} +} diff --git a/modules/multi-runner/variables.experimental.ssm.tf b/modules/multi-runner/variables.experimental.ssm.tf new file mode 100644 index 0000000000..64dd30ee89 --- /dev/null +++ b/modules/multi-runner/variables.experimental.ssm.tf @@ -0,0 +1,40 @@ +# Experimental global SSM configuration. +variable "experimental_global_config_ssm" { + description = "Experimental global SSM configuration." + type = object({ + paths = optional(object({ + root = optional(string, null) + app = optional(string, "app") + webhook = optional(string, "webhook") + tokens = optional(string, "runners/tokens") + config = optional(string, "runners/config") + }), {}) + kms_key_id = optional(string, null) + tags = optional(map(string), {}) + parameters = optional(object({ + tags = optional(map(string), {}) + }), {}) + housekeeper = optional(object({ + schedule_expression = optional(string, "rate(1 day)") + state = optional(string, "ENABLED") + tags = optional(map(string), {}) + lambda = optional(object({ + artifact = optional(object({ + zip = optional(string, null) + s3 = optional(object({ + key = string + object_version = optional(string, null) + }), null) + }), {}) + memory_size = optional(number, 512) + timeout = optional(number, 60) + }), {}) + config = optional(object({ + tokenPath = optional(string, null) + minimumDaysOld = optional(number, 1) + dryRun = optional(bool, false) + }), {}) + }), {}) + }) + default = {} +} diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index a47cd2a83c..cc4884b967 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -1,6 +1,8 @@ variable "github_app" { description = < v + if v.orchestration_provider.webhook != null + } + + runner_matcher_config = { + for k, v in local.webhook_runner_config : k => { + id = aws_sqs_queue.queued_builds[k].id + arn = aws_sqs_queue.queued_builds[k].arn + computeProvider = "ec2" + matcherConfig = { + labelMatchers = v.orchestration_provider.webhook.matcherConfig.labelMatchers + exactMatch = v.orchestration_provider.webhook.matcherConfig.exactMatch + bidirectionalLabelMatch = v.orchestration_provider.webhook.matcherConfig.bidirectionalLabelMatch + priority = v.orchestration_provider.webhook.matcherConfig.priority + enableDynamicLabels = v.orchestration_provider.webhook.matcherConfig.dynamic_labels_enabled + awsDynamicLabelsPolicy = v.orchestration_provider.webhook.matcherConfig.awsDynamicLabelsPolicy + } + } + } +} + module "webhook" { - source = "../webhook" - prefix = var.prefix - tags = local.tags - kms_key_arn = var.kms_key_arn - eventbridge = var.eventbridge - runner_matcher_config = local.runner_config - matcher_config_parameter_store_tier = var.matcher_config_parameter_store_tier + source = "../webhook" + prefix = var.prefix + tags = local.tags + kms_key_arn = local.effective_config.ssm.kms_key_id + eventbridge = { + enable = local.effective_config.orchestration_provider.webhook.eventbridge.enabled + accept_events = local.effective_config.orchestration_provider.webhook.eventbridge.accept_events + } + runner_matcher_config = local.runner_matcher_config + matcher_config_parameter_store_tier = local.effective_config.orchestration_provider.webhook.matcher_config_parameter_store_tier ssm_paths = { root = local.ssm_root_path - webhook = var.ssm_paths.webhook + webhook = local.effective_config.ssm.paths.webhook } github_app_parameters = { webhook_secret = local.github_app_parameters.webhook_secret } - lambda_s3_bucket = var.lambda_s3_bucket - webhook_lambda_s3_key = var.webhook_lambda_s3_key - webhook_lambda_s3_object_version = var.webhook_lambda_s3_object_version - webhook_lambda_apigateway_access_log_settings = var.webhook_lambda_apigateway_access_log_settings - lambda_runtime = var.lambda_runtime - lambda_architecture = var.lambda_architecture - lambda_zip = var.webhook_lambda_zip - lambda_timeout = var.webhook_lambda_timeout - lambda_memory_size = var.webhook_lambda_memory_size - lambda_tags = var.lambda_tags - tracing_config = var.tracing_config - logging_retention_in_days = var.logging_retention_in_days - logging_kms_key_id = var.logging_kms_key_id - log_class = var.log_class - - role_path = var.role_path - role_permissions_boundary = var.role_permissions_boundary - repository_white_list = var.repository_white_list - queue_selection_strategy = var.queue_selection_strategy - - lambda_subnet_ids = var.lambda_subnet_ids - lambda_security_group_ids = var.lambda_security_group_ids + lambda_s3_bucket = try(local.effective_config.lambda.artifact.s3.bucket, null) + webhook_lambda_s3_key = try(local.effective_config.orchestration_provider.webhook.lambda.webhook.artifact.s3.key, null) + webhook_lambda_s3_object_version = try(local.effective_config.orchestration_provider.webhook.lambda.webhook.artifact.s3.object_version, null) + webhook_lambda_apigateway_access_log_settings = local.effective_config.orchestration_provider.webhook.lambda.webhook.api_gateway_access_log_settings + lambda_runtime = local.effective_config.lambda.runtime + lambda_architecture = local.effective_config.lambda.architecture + lambda_zip = local.effective_config.orchestration_provider.webhook.lambda.webhook.artifact.zip + lambda_timeout = local.effective_config.orchestration_provider.webhook.lambda.webhook.timeout + lambda_memory_size = local.effective_config.orchestration_provider.webhook.lambda.webhook.memory_size + lambda_tags = local.effective_config.orchestration_provider.webhook.lambda.webhook.tags + tracing_config = local.effective_config.observability.tracing + logging_retention_in_days = local.effective_config.observability.logs.retention_in_days + logging_kms_key_id = local.effective_config.observability.logs.kms_key_id + log_class = local.effective_config.observability.logs.class + + role_path = local.effective_config.roles.path + role_permissions_boundary = local.effective_config.roles.permissions_boundary + repository_white_list = local.effective_config.orchestration_provider.webhook.github.repository_white_list + queue_selection_strategy = local.effective_config.orchestration_provider.webhook.queue_selection_strategy + + lambda_subnet_ids = local.effective_config.lambda.subnet_ids + lambda_security_group_ids = local.effective_config.lambda.security_group_ids aws_partition = var.aws_partition - log_level = var.log_level + log_level = local.effective_config.observability.logs.level } diff --git a/modules/orchestration-providers/webhook/README.md b/modules/orchestration-providers/webhook/README.md new file mode 100644 index 0000000000..53d0115ebb --- /dev/null +++ b/modules/orchestration-providers/webhook/README.md @@ -0,0 +1,61 @@ +# Webhook orchestration provider + +This internal module owns the event-driven runner demand controls used by `runner-config`: scale-up, scale-down, scheduled pool reconciliation, and optional queued-job retry. It receives the common GitHub, Lambda, runner-registration, SSM, observability, and selected compute-provider contracts from the parent configuration module, then resolves webhook-specific defaults and tag precedence before invoking its leaf modules. Lifecycle, boot time, and capacity are provider-owned under `config.runner`; the provider resolves the lifecycle contract for runner bootstrap, forwards capacity to scale-up and pool, and forwards boot time to scale-down and pool. It also combines the shared Lambda artifact bucket with its own `config.lambda.artifact` zip or S3 key/version shared by scale, pool, and job-retry; provider-specific artifact fields do not leak into the common Lambda contract. + +`runner-config` selects this provider when `orchestration_provider.webhook` is the one populated orchestration block. The parent continues to own common runner resources, shared SSM configuration, and compute-provider selection. A future orchestration provider should be implemented as a sibling module with the same parent-facing resource boundary; it should not add its stateful resources to this webhook module. + +The scale-down lifecycle is documented in the [scale-down state diagram](./scale-down-state-diagram.md). + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [job\_retry](#module\_job\_retry) | ./job-retry | n/a | +| [pool](#module\_pool) | ./pool | n/a | +| [scale\_runners](#module\_scale\_runners) | ./scale-runners | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.validate_config](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | AWS partition used to construct ARNs. | `string` | `"aws"` | no | +| [config](#input\_config) | Provider-owned webhook values supplied from `orchestration_provider.webhook`. The parent resolves inherited input values before calling this module; this provider still resolves the documented JIT, artifact, and tag-precedence fallbacks.

- `runner`: Runner lifecycle, boot timeout, and capacity settings owned by webhook orchestration.
- `runner.boot_time_in_minutes`: Expected runner boot duration used by scale-down and pool controls.
- `runner.ephemeral`: Registers runners in ephemeral mode.
- `runner.jit_config_enabled`: Explicitly enables or disables just-in-time configuration. Null follows `runner.ephemeral`.
- `runner.maximum_count`: Maximum number of runners managed for this runner configuration.
- `github.organization_runners`: Registers runners at organization scope when true; otherwise registration is repository-scoped.
- `queue.build.arn`: ARN of the runner configuration's build queue.
- `queue.build.url`: URL of the runner configuration's build queue.
- `queue.kms_key_id`: Optional KMS key ARN encrypting the build queue. This is independent from the Parameter Store KMS key.
- `queue.tags`: Tags inherited by queue-related provider resources before component-specific overrides.
- `lambda.artifact`: Runner-control artifact shared by scale, pool, and job-retry components. At most one of `zip` or `s3` may be selected; no selection uses the packaged runner archive.
- `lambda.artifact.zip`: Optional local path to the runner-control Lambda archive.
- `lambda.artifact.s3`: Optional S3 object selector in the common `lambda.artifact.s3.bucket`. Wrapper presence must be known during planning and selecting it requires a non-null common bucket.
- `lambda.artifact.s3.key`: Object key of the runner-control Lambda archive.
- `lambda.artifact.s3.object_version`: Optional object version of the runner-control Lambda archive.
- `lambda.scale.up.memory_size`: Memory allocated to the scale-up Lambda in MB.
- `lambda.scale.up.timeout`: Scale-up Lambda timeout in seconds.
- `lambda.scale.up.reserved_concurrent_executions`: Reserved concurrency for scale-up. Use `-1` for unreserved concurrency.
- `lambda.scale.up.job_queued_check_enabled`: Enables queued-job verification before scaling. Null follows the resolved runner mode.
- `lambda.scale.up.event_source_mapping.batch_size`: Maximum build-queue records delivered per scale-up invocation.
- `lambda.scale.up.event_source_mapping.maximum_batching_window_in_seconds`: Maximum batching window for build-queue records.
- `lambda.scale.up.tags`: Tags applied within scale-up resource scopes after common provider tags.
- `lambda.scale.down.memory_size`: Memory allocated to the scale-down Lambda in MB.
- `lambda.scale.down.timeout`: Scale-down Lambda timeout in seconds.
- `lambda.scale.down.schedule_expression`: EventBridge schedule expression that invokes scale-down.
- `lambda.scale.down.minimum_running_time_in_minutes`: Optional minimum runner age before scale-down may terminate it. Null selects the operating-system default.
- `lambda.scale.down.idle_config`: Time-based desired idle-runner configurations.
- `lambda.scale.down.idle_config[].cron`: Cron expression identifying when the idle configuration applies.
- `lambda.scale.down.idle_config[].timeZone`: IANA time zone used to evaluate the cron expression.
- `lambda.scale.down.idle_config[].idleCount`: Number of idle runners retained during the matching period.
- `lambda.scale.down.idle_config[].evictionStrategy`: Selection strategy used when excess idle runners are removed.
- `lambda.scale.down.tags`: Tags applied within scale-down resource scopes after common provider tags.
- `lambda.pool.memory_size`: Memory allocated to the pool Lambda in MB.
- `lambda.pool.timeout`: Pool Lambda timeout in seconds.
- `lambda.pool.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. Use `-1` for unreserved concurrency.
- `lambda.pool.config`: Scheduled target pool sizes. An empty list disables the pool component.
- `lambda.pool.config[].schedule_expression`: Scheduler expression that activates the target size.
- `lambda.pool.config[].schedule_expression_timezone`: Optional IANA time zone used to evaluate the schedule.
- `lambda.pool.config[].size`: Desired number of runners for the schedule.
- `lambda.pool.include_busy_runners`: Includes busy runners when reconciling scheduled pool capacity.
- `lambda.pool.runner_owner`: Optional GitHub organization or repository owner used for pooled runners.
- `lambda.pool.tags`: Tags applied within pool resource scopes after common provider tags.
- `job_retry.enabled`: Creates the retry queue, Lambda function, event-source mapping, and related IAM resources.
- `job_retry.delay_in_seconds`: Initial delay before a queued-job retry check.
- `job_retry.delay_backoff`: Multiplier applied to the delay after each unsuccessful check.
- `job_retry.max_attempts`: Maximum retry-check attempts before the message is no longer republished.
- `job_retry.tags`: Tags applied within job-retry resource scopes after common provider tags.
- `job_retry.lambda.memory_size`: Memory allocated to the job-retry Lambda in MB.
- `job_retry.lambda.reserved_concurrent_executions`: Reserved concurrency for job retry. Use `-1` for unreserved concurrency.
- `job_retry.lambda.timeout`: Job-retry Lambda timeout in seconds and visibility timeout for its retry queue. |
object({
runner = object({
boot_time_in_minutes = number
ephemeral = bool
jit_config_enabled = optional(bool, null)
maximum_count = number
})
github = object({
organization_runners = bool
})
queue = object({
build = object({
arn = string
url = string
})
kms_key_id = optional(string, null)
tags = optional(map(string), {})
})
lambda = object({
artifact = object({
zip = optional(string, null)
s3 = optional(object({
key = string
object_version = optional(string, null)
}), null)
})
scale = object({
up = object({
memory_size = number
timeout = number
reserved_concurrent_executions = number
job_queued_check_enabled = optional(bool, null)
event_source_mapping = object({
batch_size = number
maximum_batching_window_in_seconds = number
})
tags = optional(map(string), {})
})
down = object({
memory_size = number
timeout = number
schedule_expression = string
minimum_running_time_in_minutes = optional(number, null)
idle_config = list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = string
}))
tags = optional(map(string), {})
})
})
pool = object({
memory_size = number
timeout = number
reserved_concurrent_executions = number
config = list(object({
schedule_expression = string
schedule_expression_timezone = optional(string)
size = number
}))
include_busy_runners = bool
runner_owner = optional(string, null)
tags = optional(map(string), {})
})
})
job_retry = object({
enabled = bool
delay_in_seconds = number
delay_backoff = number
max_attempts = number
tags = optional(map(string), {})
lambda = object({
memory_size = number
reserved_concurrent_executions = number
timeout = number
})
})
})
| n/a | yes | +| [github](#input\_github) | Common GitHub API client and GitHub App Parameter Store references. |
object({
app_parameters = object({
key_base64 = list(map(string))
id = list(map(string))
installation_id = list(object({ name = string, arn = string }))
})
enterprise_server = object({
url = optional(string, null)
ssl_verify = bool
})
user_agent = optional(string, null)
})
| n/a | yes | +| [lambda](#input\_lambda) | Common Lambda substrate. Only the shared artifact bucket crosses this boundary; the webhook provider owns its archive key, version, and local zip selection. |
object({
artifact = object({
s3 = object({
bucket = optional(string, null)
})
})
runtime = string
architecture = string
subnet_ids = list(string)
security_group_ids = list(string)
tags = optional(map(string), {})
role = object({
path = string
permissions_boundary = optional(string, null)
principals = optional(list(object({
type = string
identifiers = list(string)
})), [])
})
})
| n/a | yes | +| [observability](#input\_observability) | Common logging, tracing, and metrics configuration consumed by webhook controls. |
object({
logs = object({
level = string
retention_in_days = number
kms_key_id = optional(string, null)
class = string
tags = optional(map(string), {})
})
tracing = object({
mode = optional(string, null)
capture_http_requests = bool
capture_error = bool
})
metrics = object({
enabled = bool
namespace = string
metric = object({
github_app_rate_limit = object({
enabled = bool
})
job_retry = object({
enabled = bool
})
})
})
})
| n/a | yes | +| [prefix](#input\_prefix) | Prefix used to identify resources created for this webhook orchestration provider. | `string` | n/a | yes | +| [runner](#input\_runner) | Common runner registration values consumed by webhook demand controls. Lifecycle, boot timeout, and capacity remain provider-owned under config.runner. |
object({
os = string
auto_update_disabled = bool
labels = list(string)
group_name = string
name_prefix = string
})
| n/a | yes | +| [runner\_provider](#input\_runner\_provider) | Selected compute-provider capabilities consumed by webhook scale-up, scale-down, and pool controls. |
object({
type = string
scale_up = object({
environment_variables = map(string)
iam_policy_json = string
additional_iam_policy_json = optional(string, null)
managed_policy = optional(object({
arn = string
}), null)
})
scale_down = object({
environment_variables = map(string)
iam_policy_json = string
})
pool = object({
environment_variables = map(string)
iam_policy_json = string
managed_policy_enabled = bool
managed_policy_arn = optional(string, null)
})
})
| n/a | yes | +| [ssm](#input\_ssm) | Resolved Parameter Store paths, optional decrypt key, and runtime parameter tags. |
object({
token_path = string
token_path_arn = string
config_path = string
config_path_arn = string
kms_key_id = optional(string, null)
parameter_store_tags = string
})
| n/a | yes | +| [tags](#input\_tags) | Base tags available to webhook-provider resources. Component-specific tags override this map within their documented scopes. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [job\_retry](#output\_job\_retry) | Job-retry resources. Null when job retry is disabled. | +| [pool](#output\_pool) | Scheduled pool resources. Null when no pool schedule is configured. | +| [runner\_lifecycle](#output\_runner\_lifecycle) | Effective webhook-owned runner lifecycle consumed by runner-config bootstrap parameters. | +| [scale\_down](#output\_scale\_down) | Scale-down control-plane resources. | +| [scale\_up](#output\_scale\_up) | Scale-up control-plane resources. | + diff --git a/modules/orchestration-providers/webhook/job-retry.tf b/modules/orchestration-providers/webhook/job-retry.tf new file mode 100644 index 0000000000..651e12c9ea --- /dev/null +++ b/modules/orchestration-providers/webhook/job-retry.tf @@ -0,0 +1,48 @@ +module "job_retry" { + source = "./job-retry" + count = local.job_retry_enabled ? 1 : 0 + + config = { + prefix = local.resolved_config.prefix + aws_partition = var.aws_partition + lambda = { + artifact = local.resolved_config.lambda.artifact + runtime = local.resolved_config.lambda.runtime + architecture = local.resolved_config.lambda.architecture + memory_size = local.resolved_config.job_retry.lambda.memory_size + timeout = local.resolved_config.job_retry.lambda.timeout + reserved_concurrent_executions = local.resolved_config.job_retry.lambda.reserved_concurrent_executions + environment_variables = {} + vpc = { + subnet_ids = local.resolved_config.lambda.subnet_ids + security_group_ids = local.resolved_config.lambda.security_group_ids + } + role = local.resolved_config.lambda.role + } + runner = { + name_prefix = local.resolved_config.runner.name_prefix + } + github = local.resolved_config.github + queue = { + build = local.resolved_config.queue.build + kms_key_id = local.resolved_config.queue.kms_key_id + event_source_mapping = local.resolved_config.queue.event_source_mapping + encryption = { + sqs_managed_sse_enabled = true + kms_master_key_id = null + kms_data_key_reuse_period_seconds = null + } + } + ssm = { + kms_key_id = local.resolved_config.ssm.kms_key_id + } + observability = local.resolved_config.observability + tags = { + resources = local.job_retry_tags + lambda = local.job_retry_lambda_tags + log_group = local.job_retry_log_tags + queue = local.job_retry_queue_tags + event_source_mapping = local.job_retry_queue_tags + } + } +} diff --git a/modules/orchestration-providers/webhook/job-retry/README.md b/modules/orchestration-providers/webhook/job-retry/README.md new file mode 100644 index 0000000000..9c6e4e0f52 --- /dev/null +++ b/modules/orchestration-providers/webhook/job-retry/README.md @@ -0,0 +1,63 @@ +# Module - Job Retry + +This module is listening to a SQS queue where the scale-up lambda publishes messages for jobs that needs to trigger a retry if still queued. The job retry module lambda function is handling the messages, checking if the job is queued. Next for queued jobs a message is published to the build queue for the scale-up lambda. The scale-up lambda will handle the message as any other workflow job event. + +## Usages + +The module is an inner module used by the webhook orchestration provider when the opt-in feature for job retry is enabled. The module is not intended to be used standalone. + + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.21 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.21 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_role.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.job_retry_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.job_retry_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.job_retry_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_lambda_event_source_mapping.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource | +| [aws_lambda_function.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_lambda_permission.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [aws_sqs_queue.job_retry_check_queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | +| [aws_sqs_queue_policy.job_retry_check_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | +| [terraform_data.validate_config](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [aws_iam_policy_document.deny_insecure_transport](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.job_retry_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [config](#input\_config) | Provider-neutral job-retry configuration assembled by runner-config.

- `prefix`: Prefix used to name job-retry resources.
- `aws_partition`: AWS partition used to construct the Lambda VPC managed-policy ARN.
- `lambda.artifact.zip`: Resolved local control-plane archive.
- `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive.
- `lambda.artifact.s3.key`: Object key of the Lambda archive.
- `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive.
- `lambda.runtime`: Runtime used by the job-retry Lambda.
- `lambda.architecture`: Instruction-set architecture used by the job-retry Lambda.
- `lambda.memory_size`: Memory allocated to the job-retry Lambda.
- `lambda.timeout`: Lambda timeout and retry-queue visibility timeout in seconds.
- `lambda.reserved_concurrent_executions`: Reserved concurrency for the Lambda. Use `-1` for unreserved concurrency.
- `lambda.environment_variables`: Additional Lambda environment variables. Required job-retry variables override matching keys.
- `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration.
- `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration.
- `lambda.role.path`: IAM path used for the job-retry Lambda role.
- `lambda.role.permissions_boundary`: Optional permissions boundary for the Lambda role.
- `lambda.role.principals`: Extra principals allowed to assume the Lambda role, for example during local testing.
- `runner.name_prefix`: Prefix used to identify runners belonging to this runner configuration.
- `github.organization_runners`: Enables organization runners.
- `github.enterprise_server.url`: Optional GitHub Enterprise Server URL.
- `github.enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server requests.
- `github.user_agent`: Optional User-Agent sent to GitHub.
- `github.app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys.
- `github.app_parameters.id`: Ordered Parameter Store references for GitHub App IDs.
- `github.app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs.
- `queue.build`: URL and ARN of the build queue to which retry messages are published.
- `queue.kms_key_id`: Optional KMS key ARN used to encrypt the build queue. This is distinct from the Parameter Store key.
- `queue.event_source_mapping.batch_size`: Maximum records delivered per job-retry invocation.
- `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum event batching window.
- `queue.encryption`: Server-side encryption configuration for the retry queue.
- `ssm.kms_key_id`: Optional KMS key ARN used by the job-retry IAM policy. Its value may be unknown until apply.
- `observability.logs`: Logging level, retention, encryption, and log-class configuration.
- `observability.tracing`: Lambda X-Ray and tracing-helper configuration.
- `observability.metrics`: Metrics enablement, namespace, and job-retry metric configuration.
- `tags.resources`: Tags for the job-retry Lambda role and component resources.
- `tags.lambda`: Tags for the job-retry Lambda function.
- `tags.log_group`: Tags for the job-retry log group.
- `tags.queue`: Tags for the retry queue.
- `tags.event_source_mapping`: Tags for the retry-queue event-source mapping. |
object({
prefix = string
aws_partition = string
lambda = object({
artifact = object({
zip = string
s3 = object({
bucket = optional(string, null)
key = optional(string, null)
object_version = optional(string, null)
})
})
runtime = string
architecture = string
memory_size = number
timeout = number
reserved_concurrent_executions = number
environment_variables = map(string)
vpc = object({
subnet_ids = list(string)
security_group_ids = list(string)
})
role = object({
path = string
permissions_boundary = optional(string, null)
principals = list(object({
type = string
identifiers = list(string)
}))
})
})
runner = object({
name_prefix = string
})
github = object({
organization_runners = bool
enterprise_server = object({
url = optional(string, null)
ssl_verify = optional(bool, true)
})
user_agent = optional(string, null)
app_parameters = object({
key_base64 = list(map(string))
id = list(map(string))
installation_id = list(object({ name = string, arn = string }))
})
})
queue = object({
build = object({
url = string
arn = string
})
kms_key_id = optional(string, null)
event_source_mapping = object({
batch_size = number
maximum_batching_window_in_seconds = number
})
encryption = object({
sqs_managed_sse_enabled = bool
kms_master_key_id = optional(string, null)
kms_data_key_reuse_period_seconds = optional(number, null)
})
})
ssm = object({
kms_key_id = optional(string, null)
})
observability = object({
logs = object({
level = string
retention_in_days = number
kms_key_id = optional(string, null)
class = string
})
tracing = object({
mode = optional(string, null)
capture_http_requests = bool
capture_error = bool
})
metrics = object({
enabled = bool
namespace = string
metric = object({
github_app_rate_limit = object({
enabled = bool
})
job_retry = object({
enabled = bool
})
})
})
})
tags = object({
resources = map(string)
lambda = map(string)
log_group = map(string)
queue = map(string)
event_source_mapping = map(string)
})
})
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [job\_retry\_check\_queue](#output\_job\_retry\_check\_queue) | Queue consumed by the job-retry Lambda. | +| [lambda](#output\_lambda) | Job-retry Lambda resources. | + diff --git a/modules/orchestration-providers/webhook/job-retry/iam-policies.tf b/modules/orchestration-providers/webhook/job-retry/iam-policies.tf new file mode 100644 index 0000000000..0e79e8a265 --- /dev/null +++ b/modules/orchestration-providers/webhook/job-retry/iam-policies.tf @@ -0,0 +1,122 @@ +# IAM policies attached to the job-retry Lambda role. +data "aws_iam_policy_document" "lambda_assume_role" { + statement { + sid = "WebhookJobRetryAssumeRole" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + + dynamic "principals" { + for_each = var.config.lambda.role.principals + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + } +} + +data "aws_iam_policy_document" "job_retry_logging" { + statement { + sid = "WebhookJobRetryWriteLogs" + effect = "Allow" + + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + + resources = ["${aws_cloudwatch_log_group.job_retry.arn}*"] + } +} + +data "aws_iam_policy_document" "lambda_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + + # AWS X-Ray write/read trace APIs do not support resource-level permissions. + statement { + sid = "AllowXRay" + effect = "Allow" + actions = [ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments", + ] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "job_retry" { + statement { + sid = "WebhookJobRetryReadGitHubAppParameters" + effect = "Allow" + + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + + resources = concat( + [for p in var.config.github.app_parameters.id : p.arn], + [for p in var.config.github.app_parameters.key_base64 : p.arn], + [for p in var.config.github.app_parameters.installation_id : p.arn if p != null], + ) + } + + statement { + sid = "WebhookJobRetryConsumeRetryQueue" + effect = "Allow" + + actions = [ + "sqs:ReceiveMessage", + "sqs:GetQueueAttributes", + "sqs:DeleteMessage", + ] + + resources = [aws_sqs_queue.job_retry_check_queue.arn] + } + + statement { + sid = "WebhookJobRetryPublishBuildQueue" + effect = "Allow" + + actions = [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + ] + + resources = [var.config.queue.build.arn] + } + + dynamic "statement" { + for_each = var.config.ssm.kms_key_id == null ? [] : [var.config.ssm.kms_key_id] + iterator = kms_key + + content { + sid = "WebhookJobRetryDecryptParameterStore" + effect = "Allow" + actions = ["kms:Decrypt"] + resources = [kms_key.value] + } + } + + dynamic "statement" { + for_each = var.config.queue.kms_key_id == null ? [] : [var.config.queue.kms_key_id] + iterator = kms_key + + content { + sid = "WebhookJobRetryEncryptBuildQueueMessage" + effect = "Allow" + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + resources = [kms_key.value] + } + } +} diff --git a/modules/orchestration-providers/webhook/job-retry/job-retry.tf b/modules/orchestration-providers/webhook/job-retry/job-retry.tf new file mode 100644 index 0000000000..a536cfe196 --- /dev/null +++ b/modules/orchestration-providers/webhook/job-retry/job-retry.tf @@ -0,0 +1,179 @@ +# Provider-neutral job-retry queue and Lambda resources. +locals { + name = "job-retry" + vpc_enabled = ( + length(var.config.lambda.vpc.subnet_ids) > 0 && + length(var.config.lambda.vpc.security_group_ids) > 0 + ) + + lambda_environment_variables = { + ENVIRONMENT = var.config.prefix + LOG_LEVEL = var.config.observability.logs.level + PREFIX = var.config.prefix + POWERTOOLS_LOGGER_LOG_EVENT = var.config.observability.logs.level == "debug" ? "true" : "false" + POWERTOOLS_SERVICE_NAME = local.name + POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error + POWERTOOLS_METRICS_NAMESPACE = var.config.observability.metrics.namespace + } + + job_retry_environment_variables = { + ENABLE_ORGANIZATION_RUNNERS = var.config.github.organization_runners + ENABLE_METRIC_JOB_RETRY = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.job_retry.enabled + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.github_app_rate_limit.enabled + GHES_URL = var.config.github.enterprise_server.url + NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 + USER_AGENT = var.config.github.user_agent + JOB_QUEUE_SCALE_UP_URL = var.config.queue.build.url + PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.config.github.app_parameters.id : p.name]) + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.config.github.app_parameters.key_base64 : p.name]) + PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.config.github.app_parameters.installation_id : p != null ? p.name : ""]) + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + } + + environment_variables = merge( + local.lambda_environment_variables, + var.config.lambda.environment_variables, + local.job_retry_environment_variables, + ) +} + +resource "aws_sqs_queue_policy" "job_retry_check_queue_policy" { + queue_url = aws_sqs_queue.job_retry_check_queue.id + policy = data.aws_iam_policy_document.deny_insecure_transport.json +} + +resource "aws_sqs_queue" "job_retry_check_queue" { + name = "${var.config.prefix}-job-retry" + visibility_timeout_seconds = var.config.lambda.timeout + + sqs_managed_sse_enabled = var.config.queue.encryption.sqs_managed_sse_enabled + kms_master_key_id = var.config.queue.encryption.kms_master_key_id + kms_data_key_reuse_period_seconds = var.config.queue.encryption.kms_data_key_reuse_period_seconds + + tags = var.config.tags.queue +} + +resource "aws_lambda_function" "job_retry" { + s3_bucket = var.config.lambda.artifact.s3.bucket + s3_key = var.config.lambda.artifact.s3.key + s3_object_version = var.config.lambda.artifact.s3.object_version + filename = var.config.lambda.artifact.s3.bucket == null ? var.config.lambda.artifact.zip : null + source_code_hash = var.config.lambda.artifact.s3.bucket == null ? filebase64sha256(var.config.lambda.artifact.zip) : null + function_name = "${var.config.prefix}-${local.name}" + role = aws_iam_role.job_retry.arn + handler = "index.jobRetryCheck" + runtime = var.config.lambda.runtime + timeout = var.config.lambda.timeout + memory_size = var.config.lambda.memory_size + reserved_concurrent_executions = var.config.lambda.reserved_concurrent_executions + architectures = [var.config.lambda.architecture] + + environment { + variables = local.environment_variables + } + + dynamic "vpc_config" { + for_each = local.vpc_enabled ? [true] : [] + + content { + security_group_ids = var.config.lambda.vpc.security_group_ids + subnet_ids = var.config.lambda.vpc.subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.config.observability.tracing.mode != null ? [true] : [] + + content { + mode = var.config.observability.tracing.mode + } + } + + tags = var.config.tags.lambda +} + +resource "aws_cloudwatch_log_group" "job_retry" { + name = "/aws/lambda/${aws_lambda_function.job_retry.function_name}" + retention_in_days = var.config.observability.logs.retention_in_days + kms_key_id = var.config.observability.logs.kms_key_id + log_group_class = var.config.observability.logs.class + tags = var.config.tags.log_group +} + +resource "aws_iam_role" "job_retry" { + name = "${substr("${var.config.prefix}-${local.name}", 0, 54)}-${substr(md5("${var.config.prefix}-${local.name}"), 0, 8)}" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json + path = var.config.lambda.role.path + permissions_boundary = var.config.lambda.role.permissions_boundary + tags = var.config.tags.resources +} + +resource "aws_iam_role_policy" "job_retry_logging" { + name = "logging-policy" + role = aws_iam_role.job_retry.name + policy = data.aws_iam_policy_document.job_retry_logging.json +} + +resource "aws_iam_role_policy_attachment" "job_retry_vpc_execution_role" { + count = local.vpc_enabled ? 1 : 0 + role = aws_iam_role.job_retry.name + policy_arn = "arn:${var.config.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +resource "aws_iam_role_policy" "job_retry_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + name = "xray-policy" + policy = data.aws_iam_policy_document.lambda_xray[0].json + role = aws_iam_role.job_retry.name +} + +resource "aws_lambda_event_source_mapping" "job_retry" { + event_source_arn = aws_sqs_queue.job_retry_check_queue.arn + function_name = aws_lambda_function.job_retry.arn + batch_size = var.config.queue.event_source_mapping.batch_size + maximum_batching_window_in_seconds = var.config.queue.event_source_mapping.maximum_batching_window_in_seconds + tags = var.config.tags.event_source_mapping +} + +resource "aws_lambda_permission" "job_retry" { + statement_id = "AllowExecutionFromSQS" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.job_retry.function_name + principal = "sqs.amazonaws.com" + source_arn = aws_sqs_queue.job_retry_check_queue.arn +} + +resource "aws_iam_role_policy" "job_retry" { + name = "job_retry-policy" + role = aws_iam_role.job_retry.name + policy = data.aws_iam_policy_document.job_retry.json +} + +data "aws_iam_policy_document" "deny_insecure_transport" { + statement { + sid = "DenyInsecureTransport" + + effect = "Deny" + + principals { + type = "AWS" + identifiers = ["*"] + } + + actions = [ + "sqs:*" + ] + + resources = [ + aws_sqs_queue.job_retry_check_queue.arn + ] + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} diff --git a/modules/orchestration-providers/webhook/job-retry/outputs.tf b/modules/orchestration-providers/webhook/job-retry/outputs.tf new file mode 100644 index 0000000000..4f08cc4498 --- /dev/null +++ b/modules/orchestration-providers/webhook/job-retry/outputs.tf @@ -0,0 +1,13 @@ +output "lambda" { + description = "Job-retry Lambda resources." + value = { + function = aws_lambda_function.job_retry + log_group = aws_cloudwatch_log_group.job_retry + role = aws_iam_role.job_retry + } +} + +output "job_retry_check_queue" { + description = "Queue consumed by the job-retry Lambda." + value = aws_sqs_queue.job_retry_check_queue +} diff --git a/modules/orchestration-providers/webhook/job-retry/tests/job-retry.tftest.hcl b/modules/orchestration-providers/webhook/job-retry/tests/job-retry.tftest.hcl new file mode 100644 index 0000000000..25d1dfaafc --- /dev/null +++ b/modules/orchestration-providers/webhook/job-retry/tests/job-retry.tftest.hcl @@ -0,0 +1,394 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/job-retry-test" + } + } + +} + +variables { + config = { + prefix = "job-retry-test" + aws_partition = "aws" + lambda = { + artifact = { + zip = "unused.zip" + s3 = { + bucket = "lambda-artifacts" + key = "job-retry.zip" + } + } + architecture = "arm64" + runtime = "nodejs24.x" + memory_size = 256 + timeout = 30 + reserved_concurrent_executions = 1 + environment_variables = { + CUSTOM_ENV = "preserved" + RUNNER_NAME_PREFIX = "caller-prefix-" + } + vpc = { + security_group_ids = ["sg-12345678"] + subnet_ids = ["subnet-12345678"] + } + role = { + path = "/job-retry-test/" + principals = [{ + type = "AWS" + identifiers = ["arn:aws:iam::123456789012:root"] + }] + } + } + runner = { + name_prefix = "required-prefix-" + } + github = { + organization_runners = false + enterprise_server = { + url = "https://experimental-job-retry.example.com" + ssl_verify = false + } + user_agent = "experimental-job-retry-user-agent" + app_parameters = { + key_base64 = [ + { + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + }, + { + name = "/github-runner/key-base64-2" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64-2" + }, + ] + id = [ + { + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + }, + { + name = "/github-runner/app-id-2" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id-2" + }, + ] + installation_id = [ + null, + { + name = "/github-runner/installation-id-2" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/installation-id-2" + }, + ] + } + } + queue = { + build = { + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + } + kms_key_id = "arn:aws:kms:eu-west-1:123456789012:key/build-queue-test" + event_source_mapping = { + batch_size = 10 + maximum_batching_window_in_seconds = 0 + } + encryption = { + sqs_managed_sse_enabled = true + } + } + ssm = { + kms_key_id = "arn:aws:kms:eu-west-1:123456789012:key/job-retry-test" + } + observability = { + logs = { + level = "trace" + class = "INFREQUENT_ACCESS" + retention_in_days = 180 + } + tracing = { + mode = "Active" + capture_http_requests = false + capture_error = false + } + metrics = { + enabled = false + namespace = "JobRetryTest" + metric = { + github_app_rate_limit = { + enabled = true + } + job_retry = { + enabled = true + } + } + } + } + tags = { + resources = { scope = "resources" } + lambda = { scope = "lambda" } + log_group = { scope = "log-group" } + queue = { scope = "queue" } + event_source_mapping = { scope = "event-source-mapping" } + } + } +} + +run "preserves_nested_job_retry_configuration" { + command = plan + + assert { + condition = output.lambda.function.environment[0].variables["CUSTOM_ENV"] == "preserved" + error_message = "Caller-provided job-retry environment variables must be preserved." + } + + assert { + condition = output.lambda.function.environment[0].variables["RUNNER_NAME_PREFIX"] == "required-prefix-" + error_message = "Required job-retry environment variables must override caller-provided values." + } + + assert { + condition = ( + output.lambda.function.environment[0].variables["GHES_URL"] == "https://experimental-job-retry.example.com" + && output.lambda.function.environment[0].variables["NODE_TLS_REJECT_UNAUTHORIZED"] == "0" + && output.lambda.function.environment[0].variables["USER_AGENT"] == "experimental-job-retry-user-agent" + && output.lambda.function.environment[0].variables["PARAMETER_GITHUB_APP_ID_NAME"] == "/github-runner/app-id:/github-runner/app-id-2" + && output.lambda.function.environment[0].variables["PARAMETER_GITHUB_APP_KEY_BASE64_NAME"] == "/github-runner/key-base64:/github-runner/key-base64-2" + && output.lambda.function.environment[0].variables["PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME"] == ":/github-runner/installation-id-2" + && contains(data.aws_iam_policy_document.job_retry.statement[0].resources, "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id-2") + && contains(data.aws_iam_policy_document.job_retry.statement[0].resources, "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64-2") + && contains(data.aws_iam_policy_document.job_retry.statement[0].resources, "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/installation-id-2") + ) + error_message = "Job retry must receive the nested GitHub connection settings, pass every app parameter, and grant access to every corresponding SSM ARN." + } + + assert { + condition = ( + toset(keys(output.lambda)) == toset(["function", "log_group", "role"]) + && output.lambda.function.s3_bucket == "lambda-artifacts" + && output.lambda.function.s3_key == "job-retry.zip" + && output.lambda.function.reserved_concurrent_executions == 1 + ) + error_message = "The nested Lambda configuration and direct resource output contract must be preserved." + } + + assert { + condition = ( + output.lambda.function.tags == tomap({ scope = "lambda" }) + && output.lambda.log_group.tags == tomap({ scope = "log-group" }) + && output.lambda.role.tags == tomap({ scope = "resources" }) + && output.job_retry_check_queue.tags == tomap({ scope = "queue" }) + && aws_lambda_event_source_mapping.job_retry.tags == tomap({ scope = "event-source-mapping" }) + ) + error_message = "Resolved nested tag maps must be applied to their owned resources." + } + + assert { + condition = ( + output.lambda.log_group.log_group_class == "INFREQUENT_ACCESS" + && length(data.aws_iam_policy_document.job_retry.statement) == 5 + && one([ + for statement in data.aws_iam_policy_document.job_retry.statement : statement + if statement.sid == "WebhookJobRetryDecryptParameterStore" + ]).resources == toset(["arn:aws:kms:eu-west-1:123456789012:key/job-retry-test"]) + && one([ + for statement in data.aws_iam_policy_document.job_retry.statement : statement + if statement.sid == "WebhookJobRetryDecryptParameterStore" + ]).actions == toset(["kms:Decrypt"]) + && one([ + for statement in data.aws_iam_policy_document.job_retry.statement : statement + if statement.sid == "WebhookJobRetryEncryptBuildQueueMessage" + ]).resources == toset(["arn:aws:kms:eu-west-1:123456789012:key/build-queue-test"]) + && one([ + for statement in data.aws_iam_policy_document.job_retry.statement : statement + if statement.sid == "WebhookJobRetryEncryptBuildQueueMessage" + ]).actions == toset(["kms:Decrypt", "kms:GenerateDataKey"]) + && length(aws_lambda_function.job_retry.vpc_config) == 1 + && length(aws_iam_role_policy_attachment.job_retry_vpc_execution_role) == 1 + && length(aws_iam_role_policy.job_retry_xray) == 1 + && length(data.aws_iam_policy_document.lambda_assume_role.statement[0].principals) == 2 + ) + error_message = "Logging, distinct Parameter Store/build-queue KMS grants, complete VPC, tracing, and extra role-principal configuration must be preserved." + } + + assert { + condition = ( + data.aws_iam_policy_document.lambda_xray[0].statement[0].sid == "AllowXRay" + && data.aws_iam_policy_document.lambda_xray[0].statement[0].resources == toset(["*"]) + && toset(data.aws_iam_policy_document.lambda_xray[0].statement[0].actions) == toset([ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments", + ]) + ) + error_message = "Only the resource-agnostic X-Ray APIs may retain a wildcard resource in the job-retry policies." + } + +} + +run "does_not_enable_partial_vpc_configuration" { + command = plan + + variables { + config = { + prefix = "job-retry-test" + aws_partition = "aws" + lambda = { + artifact = { + zip = "unused.zip" + s3 = { + bucket = "lambda-artifacts" + key = "job-retry.zip" + } + } + architecture = "arm64" + runtime = "nodejs24.x" + memory_size = 256 + timeout = 30 + reserved_concurrent_executions = 1 + environment_variables = {} + vpc = { + security_group_ids = [] + subnet_ids = ["subnet-12345678"] + } + role = { + path = "/job-retry-test/" + principals = [] + } + } + runner = { + name_prefix = "" + } + github = { + organization_runners = false + enterprise_server = {} + app_parameters = { + key_base64 = [{ + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + }] + id = [{ + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + }] + installation_id = [null] + } + } + queue = { + build = { + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + } + event_source_mapping = { + batch_size = 10 + maximum_batching_window_in_seconds = 0 + } + encryption = { + sqs_managed_sse_enabled = true + } + } + ssm = {} + observability = { + logs = { + level = "info" + class = "STANDARD" + retention_in_days = 180 + } + tracing = { + capture_http_requests = false + capture_error = false + } + metrics = { + enabled = false + namespace = "GitHub Runners" + metric = { + github_app_rate_limit = { + enabled = true + } + job_retry = { + enabled = true + } + } + } + } + tags = { + resources = {} + lambda = {} + log_group = {} + queue = {} + event_source_mapping = {} + } + } + } + + assert { + condition = ( + length(aws_lambda_function.job_retry.vpc_config) == 0 + && length(aws_iam_role_policy_attachment.job_retry_vpc_execution_role) == 0 + && length(data.aws_iam_policy_document.job_retry.statement) == 3 + && length([ + for statement in data.aws_iam_policy_document.job_retry.statement : statement + if contains(statement.actions, "kms:Decrypt") + ]) == 0 + ) + error_message = "Partial VPC inputs must stay disabled and a null KMS key must omit the KMS statement entirely." + } +} + +run "rejects_unsupported_lambda_architecture" { + command = plan + + plan_options { + target = [terraform_data.validate_config] + } + + variables { + config = merge(var.config, { + lambda = merge(var.config.lambda, { + architecture = "unsupported" + }) + }) + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_unsupported_log_level" { + command = plan + + plan_options { + target = [terraform_data.validate_config] + } + + variables { + config = merge(var.config, { + observability = merge(var.config.observability, { + logs = merge(var.config.observability.logs, { + level = "verbose" + }) + }) + }) + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_resource_prefix_longer_than_aws_limit" { + command = plan + + plan_options { + target = [terraform_data.validate_config] + } + + variables { + config = merge(var.config, { + prefix = "1234567890123456789012345678901234567890123456789012345" + }) + } + + expect_failures = [terraform_data.validate_config] +} diff --git a/modules/orchestration-providers/webhook/job-retry/validations.tf b/modules/orchestration-providers/webhook/job-retry/validations.tf new file mode 100644 index 0000000000..832f53ddd5 --- /dev/null +++ b/modules/orchestration-providers/webhook/job-retry/validations.tf @@ -0,0 +1,26 @@ +resource "terraform_data" "validate_config" { + lifecycle { + precondition { + condition = contains(["arm64", "x86_64"], var.config.lambda.architecture) + error_message = "config.lambda.architecture must be arm64 or x86_64." + } + + precondition { + condition = contains([ + "silly", + "trace", + "debug", + "info", + "warn", + "error", + "fatal", + ], var.config.observability.logs.level) + error_message = "config.observability.logs.level must be one of silly, trace, debug, info, warn, error, or fatal." + } + + precondition { + condition = length(var.config.prefix) + length("job-retry") <= 63 + error_message = "The length of config.prefix plus job-retry must be less than or equal to 63." + } + } +} diff --git a/modules/orchestration-providers/webhook/job-retry/variables.tf b/modules/orchestration-providers/webhook/job-retry/variables.tf new file mode 100644 index 0000000000..e8235265f8 --- /dev/null +++ b/modules/orchestration-providers/webhook/job-retry/variables.tf @@ -0,0 +1,147 @@ +variable "config" { + description = <<-EOT + Provider-neutral job-retry configuration assembled by runner-config. + + - `prefix`: Prefix used to name job-retry resources. + - `aws_partition`: AWS partition used to construct the Lambda VPC managed-policy ARN. + - `lambda.artifact.zip`: Resolved local control-plane archive. + - `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive. + - `lambda.artifact.s3.key`: Object key of the Lambda archive. + - `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive. + - `lambda.runtime`: Runtime used by the job-retry Lambda. + - `lambda.architecture`: Instruction-set architecture used by the job-retry Lambda. + - `lambda.memory_size`: Memory allocated to the job-retry Lambda. + - `lambda.timeout`: Lambda timeout and retry-queue visibility timeout in seconds. + - `lambda.reserved_concurrent_executions`: Reserved concurrency for the Lambda. Use `-1` for unreserved concurrency. + - `lambda.environment_variables`: Additional Lambda environment variables. Required job-retry variables override matching keys. + - `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration. + - `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration. + - `lambda.role.path`: IAM path used for the job-retry Lambda role. + - `lambda.role.permissions_boundary`: Optional permissions boundary for the Lambda role. + - `lambda.role.principals`: Extra principals allowed to assume the Lambda role, for example during local testing. + - `runner.name_prefix`: Prefix used to identify runners belonging to this runner configuration. + - `github.organization_runners`: Enables organization runners. + - `github.enterprise_server.url`: Optional GitHub Enterprise Server URL. + - `github.enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server requests. + - `github.user_agent`: Optional User-Agent sent to GitHub. + - `github.app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys. + - `github.app_parameters.id`: Ordered Parameter Store references for GitHub App IDs. + - `github.app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs. + - `queue.build`: URL and ARN of the build queue to which retry messages are published. + - `queue.kms_key_id`: Optional KMS key ARN used to encrypt the build queue. This is distinct from the Parameter Store key. + - `queue.event_source_mapping.batch_size`: Maximum records delivered per job-retry invocation. + - `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum event batching window. + - `queue.encryption`: Server-side encryption configuration for the retry queue. + - `ssm.kms_key_id`: Optional KMS key ARN used by the job-retry IAM policy. Its value may be unknown until apply. + - `observability.logs`: Logging level, retention, encryption, and log-class configuration. + - `observability.tracing`: Lambda X-Ray and tracing-helper configuration. + - `observability.metrics`: Metrics enablement, namespace, and job-retry metric configuration. + - `tags.resources`: Tags for the job-retry Lambda role and component resources. + - `tags.lambda`: Tags for the job-retry Lambda function. + - `tags.log_group`: Tags for the job-retry log group. + - `tags.queue`: Tags for the retry queue. + - `tags.event_source_mapping`: Tags for the retry-queue event-source mapping. + EOT + + type = object({ + prefix = string + aws_partition = string + lambda = object({ + artifact = object({ + zip = string + s3 = object({ + bucket = optional(string, null) + key = optional(string, null) + object_version = optional(string, null) + }) + }) + runtime = string + architecture = string + memory_size = number + timeout = number + reserved_concurrent_executions = number + environment_variables = map(string) + vpc = object({ + subnet_ids = list(string) + security_group_ids = list(string) + }) + role = object({ + path = string + permissions_boundary = optional(string, null) + principals = list(object({ + type = string + identifiers = list(string) + })) + }) + }) + runner = object({ + name_prefix = string + }) + github = object({ + organization_runners = bool + enterprise_server = object({ + url = optional(string, null) + ssl_verify = optional(bool, true) + }) + user_agent = optional(string, null) + app_parameters = object({ + key_base64 = list(map(string)) + id = list(map(string)) + installation_id = list(object({ name = string, arn = string })) + }) + }) + queue = object({ + build = object({ + url = string + arn = string + }) + kms_key_id = optional(string, null) + event_source_mapping = object({ + batch_size = number + maximum_batching_window_in_seconds = number + }) + encryption = object({ + sqs_managed_sse_enabled = bool + kms_master_key_id = optional(string, null) + kms_data_key_reuse_period_seconds = optional(number, null) + }) + }) + ssm = object({ + kms_key_id = optional(string, null) + }) + observability = object({ + logs = object({ + level = string + retention_in_days = number + kms_key_id = optional(string, null) + class = string + }) + tracing = object({ + mode = optional(string, null) + capture_http_requests = bool + capture_error = bool + }) + metrics = object({ + enabled = bool + namespace = string + metric = object({ + github_app_rate_limit = object({ + enabled = bool + }) + job_retry = object({ + enabled = bool + }) + }) + }) + }) + tags = object({ + resources = map(string) + lambda = map(string) + log_group = map(string) + queue = map(string) + event_source_mapping = map(string) + }) + }) + + nullable = false +} diff --git a/modules/orchestration-providers/webhook/job-retry/versions.tf b/modules/orchestration-providers/webhook/job-retry/versions.tf new file mode 100644 index 0000000000..fcec7c620d --- /dev/null +++ b/modules/orchestration-providers/webhook/job-retry/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.21" + } + } +} diff --git a/modules/orchestration-providers/webhook/main.tf b/modules/orchestration-providers/webhook/main.tf new file mode 100644 index 0000000000..d9b1722d98 --- /dev/null +++ b/modules/orchestration-providers/webhook/main.tf @@ -0,0 +1,66 @@ +locals { + packaged_runners_lambda_zip = "${path.module}/../../../lambdas/functions/control-plane/runners.zip" + runner_control_artifact_s3_selected = var.config.lambda.artifact.s3 != null + runner_control_artifact = { + zip = local.runner_control_artifact_s3_selected ? null : coalesce( + var.config.lambda.artifact.zip, + local.packaged_runners_lambda_zip, + ) + s3 = { + bucket = local.runner_control_artifact_s3_selected ? var.lambda.artifact.s3.bucket : null + key = try(var.config.lambda.artifact.s3.key, null) + object_version = try(var.config.lambda.artifact.s3.object_version, null) + } + } + + resolved_config = { + prefix = var.prefix + tags = var.tags + runner = merge(var.runner, var.config.runner, { + jit_config_enabled = ( + var.config.runner.jit_config_enabled == null + ? var.config.runner.ephemeral + : var.config.runner.jit_config_enabled + ) + }) + github = merge(var.github, var.config.github) + lambda = merge(var.lambda, { + artifact = local.runner_control_artifact + }) + queue = merge(var.config.queue, { + event_source_mapping = var.config.lambda.scale.up.event_source_mapping + }) + scale_up = var.config.lambda.scale.up + scale_down = var.config.lambda.scale.down + pool = var.config.lambda.pool + job_retry = var.config.job_retry + ssm = var.ssm + observability = var.observability + } + + common_tags = local.resolved_config.tags + lambda_tags = merge(local.common_tags, local.resolved_config.lambda.tags) + queue_tags = merge(local.common_tags, local.resolved_config.queue.tags) + observability_log_tags = merge(local.common_tags, local.resolved_config.observability.logs.tags) + + scale_up_tags = merge(local.common_tags, local.resolved_config.scale_up.tags) + scale_up_lambda_tags = merge(local.lambda_tags, local.resolved_config.scale_up.tags) + scale_up_log_tags = merge(local.observability_log_tags, local.resolved_config.scale_up.tags) + scale_up_queue_tags = merge(local.queue_tags, local.resolved_config.scale_up.tags) + + scale_down_tags = merge(local.common_tags, local.resolved_config.scale_down.tags) + scale_down_lambda_tags = merge(local.lambda_tags, local.resolved_config.scale_down.tags) + scale_down_log_tags = merge(local.observability_log_tags, local.resolved_config.scale_down.tags) + + pool_tags = merge(local.common_tags, local.resolved_config.pool.tags) + pool_lambda_tags = merge(local.lambda_tags, local.resolved_config.pool.tags) + pool_log_tags = merge(local.observability_log_tags, local.resolved_config.pool.tags) + + job_retry_enabled = local.resolved_config.job_retry.enabled + job_retry_tags = merge(local.common_tags, local.resolved_config.job_retry.tags) + job_retry_lambda_tags = merge(local.lambda_tags, local.resolved_config.job_retry.tags) + job_retry_log_tags = merge(local.observability_log_tags, local.resolved_config.job_retry.tags) + job_retry_queue_tags = merge(local.queue_tags, local.resolved_config.job_retry.tags) + + enable_job_queued_check = local.resolved_config.scale_up.job_queued_check_enabled == null ? !local.resolved_config.runner.ephemeral : local.resolved_config.scale_up.job_queued_check_enabled +} diff --git a/modules/orchestration-providers/webhook/outputs.tf b/modules/orchestration-providers/webhook/outputs.tf new file mode 100644 index 0000000000..0fb477ba7d --- /dev/null +++ b/modules/orchestration-providers/webhook/outputs.tf @@ -0,0 +1,30 @@ +output "scale_up" { + description = "Scale-up control-plane resources." + value = module.scale_runners.scale_up +} + +output "scale_down" { + description = "Scale-down control-plane resources." + value = module.scale_runners.scale_down +} + +output "pool" { + description = "Scheduled pool resources. Null when no pool schedule is configured." + value = one(module.pool[*].pool) +} + +output "job_retry" { + description = "Job-retry resources. Null when job retry is disabled." + value = local.job_retry_enabled ? { + lambda = one(module.job_retry[*].lambda) + queue = one(module.job_retry[*].job_retry_check_queue) + } : null +} + +output "runner_lifecycle" { + description = "Effective webhook-owned runner lifecycle consumed by runner-config bootstrap parameters." + value = { + ephemeral = local.resolved_config.runner.ephemeral + jit_config_enabled = local.resolved_config.runner.jit_config_enabled + } +} diff --git a/modules/orchestration-providers/webhook/pool.tf b/modules/orchestration-providers/webhook/pool.tf new file mode 100644 index 0000000000..6fb9ad3d34 --- /dev/null +++ b/modules/orchestration-providers/webhook/pool.tf @@ -0,0 +1,66 @@ +module "pool" { + count = length(local.resolved_config.pool.config) > 0 ? 1 : 0 + source = "./pool" + + config = { + prefix = local.resolved_config.prefix + ghes = { + ssl_verify = local.resolved_config.github.enterprise_server.ssl_verify + url = local.resolved_config.github.enterprise_server.url + } + user_agent = local.resolved_config.github.user_agent + github_app_parameters = local.resolved_config.github.app_parameters + runners_maximum_count = local.resolved_config.runner.maximum_count + kms_key_id = local.resolved_config.ssm.kms_key_id + lambda = { + log_level = local.resolved_config.observability.logs.level + logging_retention_in_days = local.resolved_config.observability.logs.retention_in_days + logging_kms_key_id = local.resolved_config.observability.logs.kms_key_id + log_class = local.resolved_config.observability.logs.class + reserved_concurrent_executions = local.resolved_config.pool.reserved_concurrent_executions + s3_bucket = local.resolved_config.lambda.artifact.s3.bucket + s3_key = local.resolved_config.lambda.artifact.s3.key + s3_object_version = local.resolved_config.lambda.artifact.s3.object_version + security_group_ids = local.resolved_config.lambda.security_group_ids + subnet_ids = local.resolved_config.lambda.subnet_ids + architecture = local.resolved_config.lambda.architecture + memory_size = local.resolved_config.pool.memory_size + runtime = local.resolved_config.lambda.runtime + timeout = local.resolved_config.pool.timeout + zip = local.resolved_config.lambda.artifact.zip + parameter_store_tags = local.resolved_config.ssm.parameter_store_tags + principals = local.resolved_config.lambda.role.principals + } + pool = local.resolved_config.pool.config + include_busy_runners = local.resolved_config.pool.include_busy_runners + role_path = local.resolved_config.lambda.role.path + role_permissions_boundary = local.resolved_config.lambda.role.permissions_boundary + runner = { + disable_runner_autoupdate = local.resolved_config.runner.auto_update_disabled + ephemeral = local.resolved_config.runner.ephemeral + enable_jit_config = local.resolved_config.runner.jit_config_enabled + labels = local.resolved_config.runner.labels + group_name = local.resolved_config.runner.group_name + name_prefix = local.resolved_config.runner.name_prefix + pool_owner = local.resolved_config.pool.runner_owner + boot_time_in_minutes = local.resolved_config.runner.boot_time_in_minutes + } + ssm_token_path = local.resolved_config.ssm.token_path + ssm_token_path_arn = local.resolved_config.ssm.token_path_arn + ssm_config_path = local.resolved_config.ssm.config_path + tags = local.pool_tags + lambda_tags = local.pool_lambda_tags + log_group_tags = local.pool_log_tags + arn_ssm_parameters_path_config = local.resolved_config.ssm.config_path_arn + } + + aws_partition = var.aws_partition + tracing_config = local.resolved_config.observability.tracing + runner_provider = { + type = var.runner_provider.type + environment_variables = var.runner_provider.pool.environment_variables + iam_policy_json = var.runner_provider.pool.iam_policy_json + managed_policy_enabled = var.runner_provider.pool.managed_policy_enabled + managed_policy_arn = var.runner_provider.pool.managed_policy_arn + } +} diff --git a/modules/orchestration-providers/webhook/pool/README.md b/modules/orchestration-providers/webhook/pool/README.md new file mode 100644 index 0000000000..877eec8039 --- /dev/null +++ b/modules/orchestration-providers/webhook/pool/README.md @@ -0,0 +1,66 @@ +# Pool module + +This module creates the AWS resources required to maintain a pool of runners. However terraform modules are always exposed and theoretically can be used anywhere. This module is seen as a strict inner module. + +## Why a submodule for the pool + +The pool is an opt-in feature. To be able to use the count on a module level to avoid counts per resources a module is created. All inputs of the module are already defined on a higher level. See the mapping of the variables in [`pool.tf`](../pool.tf) + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.21 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.21 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_role.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.pool_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.pool_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.pool_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_lambda_function.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_scheduler_schedule.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/scheduler_schedule) | resource | +| [aws_scheduler_schedule_group.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/scheduler_schedule_group) | resource | +| [terraform_data.validate_config](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [aws_iam_policy_document.lambda_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.pool_common](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.pool_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scheduler_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | (optional) partition for the arn if not 'aws' | `string` | `"aws"` | no | +| [config](#input\_config) | Configuration passed from the webhook orchestration provider to the pool Lambda and scheduler.

- `lambda`: Pool Lambda runtime and deployment configuration.
- `lambda.log_level`: Logging level used by the pool Lambda.
- `lambda.logging_retention_in_days`: Number of days to retain events in the pool Lambda log group.
- `lambda.logging_kms_key_id`: KMS key ID used to encrypt the pool Lambda log group.
- `lambda.log_class`: CloudWatch Logs class for the pool Lambda log group.
- `lambda.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. Use -1 for no reservation.
- `lambda.s3_bucket`: S3 bucket containing the pool Lambda deployment package.
- `lambda.s3_key`: S3 key of the pool Lambda deployment package.
- `lambda.s3_object_version`: S3 object version of the pool Lambda deployment package.
- `lambda.security_group_ids`: Security group IDs associated with the pool Lambda.
- `lambda.runtime`: AWS Lambda runtime used by the pool Lambda.
- `lambda.architecture`: AWS Lambda architecture used by the pool Lambda.
- `lambda.memory_size`: Memory allocated to the pool Lambda in MB.
- `lambda.timeout`: Pool Lambda timeout in seconds.
- `lambda.zip`: Local path to the pool Lambda deployment package when S3 is not used.
- `lambda.subnet_ids`: Subnet IDs in which the pool Lambda runs.
- `lambda.parameter_store_tags`: JSON-encoded tags supplied to the pool Lambda for SSM parameters it creates.
- `lambda.principals`: Additional principals allowed to assume the pool Lambda role.
- `tags`: Common tags added to pool resources.
- `ghes`: GitHub Enterprise Server connection configuration.
- `ghes.url`: GitHub Enterprise Server URL; null when using public GitHub.
- `ghes.ssl_verify`: Whether the pool Lambda verifies the GitHub Enterprise Server TLS certificate.
- `github_app_parameters`: Ordered SSM parameter metadata for GitHub App credentials.
- `github_app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys.
- `github_app_parameters.id`: Ordered Parameter Store references for GitHub App IDs.
- `github_app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs.
- `runner`: Runner registration configuration used by the pool Lambda.
- `runner.disable_runner_autoupdate`: Whether GitHub runner automatic updates are disabled.
- `runner.ephemeral`: Whether runners register as ephemeral runners.
- `runner.enable_jit_config`: Whether runners use just-in-time registration configuration.
- `runner.labels`: Labels assigned to runners created by the pool Lambda.
- `runner.group_name`: GitHub runner group assigned to runners created by the pool Lambda.
- `runner.name_prefix`: Prefix used for runner names.
- `runner.pool_owner`: GitHub organization or repository that owns the runner pool.
- `runner.boot_time_in_minutes`: Webhook-provider runner boot timeout used by pool reconciliation.
- `runners_maximum_count`: Webhook-provider runner capacity limit enforced by the pool Lambda.
- `prefix`: Prefix used to name pool resources.
- `pool`: Scheduled pool targets.
- `pool[*].schedule_expression`: EventBridge Scheduler expression for a pool target.
- `pool[*].schedule_expression_timezone`: Time zone used to evaluate the schedule expression.
- `pool[*].size`: Desired runner count for the scheduled pool target.
- `include_busy_runners`: Whether busy runners count toward the desired pool size.
- `role_permissions_boundary`: Permissions boundary applied to IAM roles created for the pool.
- `kms_key_id`: Optional customer-managed KMS key ARN that the pool Lambda may use to decrypt encrypted parameters.
- `role_path`: IAM path applied to roles created for the pool.
- `ssm_token_path`: SSM path under which runner registration tokens are stored.
- `ssm_token_path_arn`: ARN matching the runner registration-token SSM path.
- `ssm_config_path`: SSM path under which runner configuration is stored.
- `arn_ssm_parameters_path_config`: ARN matching the runner configuration SSM path.
- `lambda_tags`: Tags added specifically to the pool Lambda function, overriding common tags with the same key.
- `log_group_tags`: Tags added specifically to the pool Lambda log group, overriding common tags with the same key.
- `user_agent`: User-Agent header used for GitHub API requests. |
object({
lambda = object({
log_level = string
logging_retention_in_days = number
logging_kms_key_id = string
log_class = string
reserved_concurrent_executions = number
s3_bucket = string
s3_key = string
s3_object_version = string
security_group_ids = list(string)
runtime = string
architecture = string
memory_size = number
timeout = number
zip = string
subnet_ids = list(string)
parameter_store_tags = string
principals = optional(list(object({
type = string
identifiers = list(string)
})), [])
})
tags = map(string)
ghes = object({
url = string
ssl_verify = string
})
github_app_parameters = object({
key_base64 = list(map(string))
id = list(map(string))
installation_id = list(object({ name = string, arn = string }))
})
runner = object({
disable_runner_autoupdate = bool
ephemeral = bool
enable_jit_config = bool
labels = list(string)
group_name = string
name_prefix = string
pool_owner = string
boot_time_in_minutes = number
})
runners_maximum_count = number
prefix = string
pool = list(object({
schedule_expression = string
schedule_expression_timezone = string
size = number
}))
include_busy_runners = bool
role_permissions_boundary = string
kms_key_id = optional(string, null)
role_path = string
ssm_token_path = string
ssm_token_path_arn = string
ssm_config_path = string
arn_ssm_parameters_path_config = string
lambda_tags = map(string)
log_group_tags = optional(map(string), {})
user_agent = string
})
| n/a | yes | +| [runner\_provider](#input\_runner\_provider) | Compute provider integration used by the pool Lambda.

- `type`: Compute provider type passed to scheduled pool invocations.
- `environment_variables`: Provider-specific environment variables added to the pool Lambda.
- `iam_policy_json`: Provider-specific IAM policy document merged into the pool Lambda policy.
- `managed_policy_enabled`: Whether to attach a provider-specific managed IAM policy to the pool Lambda role.
- `managed_policy_arn`: ARN of the provider-specific managed IAM policy to attach when enabled. |
object({
type = string
environment_variables = map(string)
iam_policy_json = string
managed_policy_enabled = bool
managed_policy_arn = optional(string, null)
})
| n/a | yes | +| [tracing\_config](#input\_tracing\_config) | Tracing configuration for the pool Lambda.

- `mode`: AWS X-Ray tracing mode. A null value disables tracing.
- `capture_http_requests`: Whether Powertools tracing captures outgoing HTTP requests.
- `capture_error`: Whether Powertools tracing captures errors as tracing metadata. |
object({
mode = optional(string, null)
capture_http_requests = optional(bool, false)
capture_error = optional(bool, false)
})
| `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [pool](#output\_pool) | Scheduled pool Lambda resources. | + diff --git a/modules/orchestration-providers/webhook/pool/iam-policies.tf b/modules/orchestration-providers/webhook/pool/iam-policies.tf new file mode 100644 index 0000000000..f5a9285bce --- /dev/null +++ b/modules/orchestration-providers/webhook/pool/iam-policies.tf @@ -0,0 +1,77 @@ +# IAM policies attached to the pool Lambda role. +data "aws_iam_policy_document" "pool_common" { + statement { + sid = "WebhookPoolWriteRuntimeParameters" + effect = "Allow" + + actions = [ + "ssm:AddTagsToResource", + "ssm:PutParameter", + ] + + resources = [ + var.config.ssm_token_path_arn, + "${var.config.ssm_token_path_arn}/*", + var.config.arn_ssm_parameters_path_config, + "${var.config.arn_ssm_parameters_path_config}/*", + ] + } + + statement { + sid = "WebhookPoolReadRunnerConfigParameters" + effect = "Allow" + + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:GetParametersByPath", + ] + + resources = [ + var.config.arn_ssm_parameters_path_config, + "${var.config.arn_ssm_parameters_path_config}/*", + ] + } + + statement { + sid = "WebhookPoolReadGitHubAppParameters" + effect = "Allow" + + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + + resources = concat( + [for p in var.config.github_app_parameters.id : p.arn], + [for p in var.config.github_app_parameters.key_base64 : p.arn], + [for p in var.config.github_app_parameters.installation_id : p.arn if p != null], + ) + } + + dynamic "statement" { + for_each = var.config.kms_key_id == null ? [] : [var.config.kms_key_id] + iterator = kms_key + + content { + sid = "WebhookPoolDecryptParameterStore" + effect = "Allow" + actions = ["kms:Decrypt"] + resources = [kms_key.value] + } + } +} + +data "aws_iam_policy_document" "pool_logging" { + statement { + sid = "WebhookPoolWriteLogs" + effect = "Allow" + + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + + resources = ["${aws_cloudwatch_log_group.pool.arn}*"] + } +} diff --git a/modules/orchestration-providers/webhook/pool/outputs.tf b/modules/orchestration-providers/webhook/pool/outputs.tf new file mode 100644 index 0000000000..cfc429ecce --- /dev/null +++ b/modules/orchestration-providers/webhook/pool/outputs.tf @@ -0,0 +1,8 @@ +output "pool" { + description = "Scheduled pool Lambda resources." + value = { + lambda = aws_lambda_function.pool + log_group = aws_cloudwatch_log_group.pool + role = aws_iam_role.pool + } +} diff --git a/modules/orchestration-providers/webhook/pool/pool.tf b/modules/orchestration-providers/webhook/pool/pool.tf new file mode 100644 index 0000000000..cff2776e90 --- /dev/null +++ b/modules/orchestration-providers/webhook/pool/pool.tf @@ -0,0 +1,236 @@ +# Provider-neutral pool Lambda and scheduler wiring. +locals { + pool_name_prefix = ( + length("${var.config.prefix}-pool") <= 38 + ? "${var.config.prefix}-pool" + : "${substr("${var.config.prefix}-pool", 0, 29)}-${substr(md5("${var.config.prefix}-pool"), 0, 8)}" + ) + + common_environment_variables = { + DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate + ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral + ENABLE_JIT_CONFIG = var.config.runner.enable_jit_config + ENVIRONMENT = var.config.prefix + GHES_URL = var.config.ghes.url + USER_AGENT = var.config.user_agent + LOG_LEVEL = upper(var.config.lambda.log_level) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1 + PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.config.github_app_parameters.id : p.name]) + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.config.github_app_parameters.key_base64 : p.name]) + PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.config.github_app_parameters.installation_id : p != null ? p.name : ""]) + POWERTOOLS_LOGGER_LOG_EVENT = var.config.lambda.log_level == "debug" ? "true" : "false" + RUNNER_LABELS = lower(join(",", var.config.runner.labels)) + RUNNER_GROUP_NAME = var.config.runner.group_name + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + RUNNER_OWNER = var.config.runner.pool_owner + RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes + RUNNERS_MAXIMUM_COUNT = var.config.runners_maximum_count + SSM_TOKEN_PATH = var.config.ssm_token_path + SSM_CONFIG_PATH = var.config.ssm_config_path + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-pool" + POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error + SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags + INCLUDE_BUSY_RUNNERS = var.config.include_busy_runners + } +} + +resource "aws_lambda_function" "pool" { + + s3_bucket = var.config.lambda.s3_bucket != null ? var.config.lambda.s3_bucket : null + s3_key = var.config.lambda.s3_key != null ? var.config.lambda.s3_key : null + s3_object_version = var.config.lambda.s3_object_version != null ? var.config.lambda.s3_object_version : null + filename = var.config.lambda.s3_bucket == null ? var.config.lambda.zip : null + source_code_hash = var.config.lambda.s3_bucket == null ? filebase64sha256(var.config.lambda.zip) : null + function_name = "${var.config.prefix}-pool" + role = aws_iam_role.pool.arn + handler = "index.adjustPool" + architectures = [var.config.lambda.architecture] + runtime = var.config.lambda.runtime + timeout = var.config.lambda.timeout + reserved_concurrent_executions = var.config.lambda.reserved_concurrent_executions + memory_size = var.config.lambda.memory_size + tags = merge(var.config.tags, var.config.lambda_tags) + + environment { + variables = merge(var.runner_provider.environment_variables, local.common_environment_variables) + } + + dynamic "vpc_config" { + for_each = var.config.lambda.subnet_ids != null && var.config.lambda.security_group_ids != null ? [true] : [] + content { + security_group_ids = var.config.lambda.security_group_ids + subnet_ids = var.config.lambda.subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.tracing_config.mode != null ? [true] : [] + content { + mode = var.tracing_config.mode + } + } +} + +resource "aws_cloudwatch_log_group" "pool" { + name = "/aws/lambda/${aws_lambda_function.pool.function_name}" + retention_in_days = var.config.lambda.logging_retention_in_days + kms_key_id = var.config.lambda.logging_kms_key_id + log_group_class = var.config.lambda.log_class + tags = merge(var.config.tags, var.config.log_group_tags) +} + +resource "aws_iam_role" "pool" { + name = "${substr("${var.config.prefix}-pool-lambda", 0, 54)}-${substr(md5("${var.config.prefix}-pool-lambda"), 0, 8)}" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json + path = var.config.role_path + permissions_boundary = var.config.role_permissions_boundary + tags = var.config.tags +} + +resource "aws_iam_role_policy" "pool" { + name = "pool-policy" + role = aws_iam_role.pool.name + policy = data.aws_iam_policy_document.pool.json +} + +data "aws_iam_policy_document" "pool" { + source_policy_documents = [ + data.aws_iam_policy_document.pool_common.json, + var.runner_provider.iam_policy_json, + ] +} + +resource "aws_iam_role_policy" "pool_logging" { + name = "logging-policy" + role = aws_iam_role.pool.name + policy = data.aws_iam_policy_document.pool_logging.json +} + +resource "aws_iam_role_policy_attachment" "pool_vpc_execution_role" { + count = length(var.config.lambda.subnet_ids) > 0 ? 1 : 0 + role = aws_iam_role.pool.name + policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +data "aws_iam_policy_document" "lambda_assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + + dynamic "principals" { + for_each = var.config.lambda.principals + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + } +} + +resource "aws_iam_role_policy_attachment" "provider" { + count = var.runner_provider.managed_policy_enabled ? 1 : 0 + role = aws_iam_role.pool.name + policy_arn = var.runner_provider.managed_policy_arn +} + +# AWS X-Ray write/read trace APIs do not support resource-level permissions. +data "aws_iam_policy_document" "lambda_xray" { + count = var.tracing_config.mode != null ? 1 : 0 + statement { + actions = [ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments" + ] + effect = "Allow" + resources = [ + "*" + ] + sid = "AllowXRay" + } +} + +resource "aws_iam_role_policy" "pool_xray" { + count = var.tracing_config.mode != null ? 1 : 0 + name = "xray-policy" + policy = data.aws_iam_policy_document.lambda_xray[0].json + role = aws_iam_role.pool.name +} + +resource "aws_scheduler_schedule_group" "pool" { + name_prefix = local.pool_name_prefix + + tags = var.config.tags +} + +data "aws_iam_policy_document" "scheduler_assume" { + statement { + sid = "ScheduleGroupAssumeRole" + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["scheduler.amazonaws.com"] + } + + condition { + test = "StringEquals" + variable = "aws:SourceArn" + values = [aws_scheduler_schedule_group.pool.arn] + } + } +} + +data "aws_iam_policy_document" "scheduler" { + statement { + sid = "InvokePoolLambda" + actions = ["lambda:InvokeFunction"] + resources = [aws_lambda_function.pool.arn] + } +} + +resource "aws_iam_role" "scheduler" { + name_prefix = local.pool_name_prefix + + path = var.config.role_path + permissions_boundary = var.config.role_permissions_boundary + + assume_role_policy = data.aws_iam_policy_document.scheduler_assume.json + tags = var.config.tags +} + +resource "aws_iam_role_policy" "scheduler" { + name = "terraform" + role = aws_iam_role.scheduler.name + policy = data.aws_iam_policy_document.scheduler.json +} + +resource "aws_scheduler_schedule" "pool" { + for_each = { for i, v in var.config.pool : i => v } + + name = "${var.config.prefix}-pool-${each.key}-rule" + group_name = aws_scheduler_schedule_group.pool.name + + flexible_time_window { + mode = "OFF" + } + + schedule_expression = each.value.schedule_expression + schedule_expression_timezone = each.value.schedule_expression_timezone + + target { + arn = aws_lambda_function.pool.arn + role_arn = aws_iam_role.scheduler.arn + input = jsonencode({ + poolSize = each.value.size + type = var.runner_provider.type + }) + } +} diff --git a/modules/orchestration-providers/webhook/pool/tests/provider.tftest.hcl b/modules/orchestration-providers/webhook/pool/tests/provider.tftest.hcl new file mode 100644 index 0000000000..c04f435024 --- /dev/null +++ b/modules/orchestration-providers/webhook/pool/tests/provider.tftest.hcl @@ -0,0 +1,297 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"logs:CreateLogStream\",\"Resource\":\"*\"}]}" + } + } +} + +variables { + config = { + lambda = { + log_level = "info" + logging_retention_in_days = 14 + logging_kms_key_id = null + log_class = "STANDARD" + reserved_concurrent_executions = 1 + s3_bucket = "lambda-artifacts" + s3_key = "runners.zip" + s3_object_version = null + security_group_ids = [] + runtime = "nodejs24.x" + architecture = "arm64" + memory_size = 256 + timeout = 60 + zip = "runners.zip" + subnet_ids = [] + parameter_store_tags = "{}" + principals = [{ + type = "AWS" + identifiers = ["arn:aws:iam::123456789012:role/local-testing"] + }] + } + tags = { + Environment = "pool-test" + } + ghes = { + url = null + ssl_verify = true + } + github_app_parameters = { + key_base64 = [ + { + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + }, + { + name = "/github-runner/key-base64-2" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64-2" + }, + ] + id = [ + { + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + }, + { + name = "/github-runner/app-id-2" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id-2" + }, + ] + installation_id = [ + null, + { + name = "/github-runner/installation-id-2" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/installation-id-2" + }, + ] + } + runner = { + disable_runner_autoupdate = false + ephemeral = true + enable_jit_config = true + labels = ["self-hosted", "microvm"] + group_name = "default" + name_prefix = "microvm" + pool_owner = "example" + boot_time_in_minutes = 13 + } + runners_maximum_count = 10 + prefix = "pool-test" + pool = [{ + schedule_expression = "cron(0 8 * * ? *)" + schedule_expression_timezone = "UTC" + size = 2 + }] + include_busy_runners = false + role_permissions_boundary = null + kms_key_id = "arn:aws:kms:eu-west-1:123456789012:key/pool-test" + role_path = "/" + ssm_token_path = "/github-runner/tokens" + ssm_token_path_arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/tokens" + ssm_config_path = "/github-runner/config" + arn_ssm_parameters_path_config = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/config" + lambda_tags = {} + user_agent = "terraform-aws-github-runner" + } + + runner_provider = { + type = "microvm" + environment_variables = { + MICROVM_CLUSTER = "runner-cluster" + } + iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["microvm:CreateRunner"] + Resource = ["*"] + }] + }) + managed_policy_enabled = true + managed_policy_arn = "arn:aws:iam::123456789012:policy/microvm-pool" + } + + tracing_config = { + mode = "Active" + capture_http_requests = true + capture_error = true + } +} + +run "provider_supplies_only_compute_specific_pool_configuration" { + command = plan + + assert { + condition = ( + length(data.aws_iam_policy_document.lambda_assume_role_policy.statement[0].principals) == 2 && + contains(data.aws_iam_policy_document.lambda_assume_role_policy.statement[0].principals[*].type, "AWS") + ) + error_message = "The pool Lambda trust policy must include configured additional principals." + } + + assert { + condition = toset(keys(output.pool)) == toset(["lambda", "log_group", "role"]) + error_message = "The pool module must expose its resources through one nested output." + } + + assert { + condition = ( + aws_lambda_function.pool.environment[0].variables["RUNNER_OWNER"] == "example" + && aws_lambda_function.pool.environment[0].variables["RUNNERS_MAXIMUM_COUNT"] == "10" + && aws_lambda_function.pool.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "13" + ) + error_message = "The pool module must assemble common runner registration values and webhook-provider capacity and boot-time settings." + } + + assert { + condition = ( + aws_lambda_function.pool.environment[0].variables["PARAMETER_GITHUB_APP_ID_NAME"] == "/github-runner/app-id:/github-runner/app-id-2" + && aws_lambda_function.pool.environment[0].variables["PARAMETER_GITHUB_APP_KEY_BASE64_NAME"] == "/github-runner/key-base64:/github-runner/key-base64-2" + && aws_lambda_function.pool.environment[0].variables["PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME"] == ":/github-runner/installation-id-2" + && contains(data.aws_iam_policy_document.pool_common.statement[2].resources, "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id-2") + && contains(data.aws_iam_policy_document.pool_common.statement[2].resources, "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64-2") + && contains(data.aws_iam_policy_document.pool_common.statement[2].resources, "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/installation-id-2") + ) + error_message = "Pool must pass every GitHub App parameter and grant access to every corresponding SSM ARN." + } + + assert { + condition = aws_lambda_function.pool.environment[0].variables["MICROVM_CLUSTER"] == "runner-cluster" + error_message = "The pool module must merge compute-provider environment variables into the Lambda environment." + } + + assert { + condition = !contains(keys(aws_lambda_function.pool.environment[0].variables), "AMI_ID_SSM_PARAMETER_NAME") + error_message = "The common pool module must not add EC2-specific environment variables." + } + + assert { + condition = jsondecode(aws_scheduler_schedule.pool["0"].target[0].input).type == "microvm" + error_message = "The pool scheduler payload must select the configured compute provider." + } + + assert { + condition = length(data.aws_iam_policy_document.pool.source_policy_documents) == 2 + error_message = "The pool role policy must merge the common and compute-provider policy documents." + } + + assert { + condition = ( + length(data.aws_iam_policy_document.pool_common.statement) == 4 + && one([ + for statement in data.aws_iam_policy_document.pool_common.statement : statement + if statement.sid == "WebhookPoolDecryptParameterStore" + ]).resources == toset(["arn:aws:kms:eu-west-1:123456789012:key/pool-test"]) + ) + error_message = "The pool KMS policy statement must consume the scalar key ARN." + } + + assert { + condition = ( + one([ + for statement in data.aws_iam_policy_document.pool_common.statement : statement + if statement.sid == "WebhookPoolWriteRuntimeParameters" + ]).resources == toset([ + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/tokens", + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/tokens/*", + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/config", + "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/config/*", + ]) + && !contains(one([ + for statement in data.aws_iam_policy_document.pool_common.statement : statement + if statement.sid == "WebhookPoolWriteRuntimeParameters" + ]).resources, "*") + ) + error_message = "The pool Lambda must scope runtime SSM writes to the token and runner-config parameter paths." + } + + assert { + condition = ( + data.aws_iam_policy_document.lambda_xray[0].statement[0].sid == "AllowXRay" + && data.aws_iam_policy_document.lambda_xray[0].statement[0].resources == toset(["*"]) + && toset(data.aws_iam_policy_document.lambda_xray[0].statement[0].actions) == toset([ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments", + ]) + ) + error_message = "Only the resource-agnostic X-Ray APIs may retain a wildcard resource in the pool policies." + } + + assert { + condition = length(aws_iam_role_policy_attachment.provider) == 1 + error_message = "The optional compute-provider managed policy must be attached to the pool role." + } +} + +run "omits_optional_kms_statement" { + command = plan + + variables { + config = merge(var.config, { + kms_key_id = null + }) + } + + assert { + condition = ( + length(data.aws_iam_policy_document.pool_common.statement) == 3 + && length([ + for statement in data.aws_iam_policy_document.pool_common.statement : statement + if anytrue([for action in statement.actions : startswith(action, "kms:")]) + ]) == 0 + ) + error_message = "A null Parameter Store key must omit the optional pool KMS statement." + } +} + +run "rejects_empty_compute_provider_type" { + command = plan + + plan_options { + target = [terraform_data.validate_config] + } + + variables { + runner_provider = merge(var.runner_provider, { + type = " " + }) + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_invalid_compute_provider_policy" { + command = plan + + plan_options { + target = [terraform_data.validate_config] + } + + variables { + runner_provider = merge(var.runner_provider, { + iam_policy_json = "not-json" + }) + } + + expect_failures = [terraform_data.validate_config] +} + +run "requires_enabled_compute_provider_managed_policy_arn" { + command = plan + + plan_options { + target = [terraform_data.validate_config] + } + + variables { + runner_provider = merge(var.runner_provider, { + managed_policy_enabled = true + managed_policy_arn = null + }) + } + + expect_failures = [terraform_data.validate_config] +} diff --git a/modules/orchestration-providers/webhook/pool/validations.tf b/modules/orchestration-providers/webhook/pool/validations.tf new file mode 100644 index 0000000000..f18d251c23 --- /dev/null +++ b/modules/orchestration-providers/webhook/pool/validations.tf @@ -0,0 +1,18 @@ +resource "terraform_data" "validate_config" { + lifecycle { + precondition { + condition = trimspace(var.runner_provider.type) != "" + error_message = "The compute provider type must not be empty." + } + + precondition { + condition = can(jsondecode(var.runner_provider.iam_policy_json)) + error_message = "The compute provider IAM policy must be valid JSON." + } + + precondition { + condition = !var.runner_provider.managed_policy_enabled || var.runner_provider.managed_policy_arn != null + error_message = "The compute provider managed policy ARN must be set when its attachment is enabled." + } + } +} diff --git a/modules/orchestration-providers/webhook/pool/variables.tf b/modules/orchestration-providers/webhook/pool/variables.tf new file mode 100644 index 0000000000..e1f516c8ad --- /dev/null +++ b/modules/orchestration-providers/webhook/pool/variables.tf @@ -0,0 +1,161 @@ +variable "config" { + description = <<-EOF + Configuration passed from the webhook orchestration provider to the pool Lambda and scheduler. + + - `lambda`: Pool Lambda runtime and deployment configuration. + - `lambda.log_level`: Logging level used by the pool Lambda. + - `lambda.logging_retention_in_days`: Number of days to retain events in the pool Lambda log group. + - `lambda.logging_kms_key_id`: KMS key ID used to encrypt the pool Lambda log group. + - `lambda.log_class`: CloudWatch Logs class for the pool Lambda log group. + - `lambda.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. Use -1 for no reservation. + - `lambda.s3_bucket`: S3 bucket containing the pool Lambda deployment package. + - `lambda.s3_key`: S3 key of the pool Lambda deployment package. + - `lambda.s3_object_version`: S3 object version of the pool Lambda deployment package. + - `lambda.security_group_ids`: Security group IDs associated with the pool Lambda. + - `lambda.runtime`: AWS Lambda runtime used by the pool Lambda. + - `lambda.architecture`: AWS Lambda architecture used by the pool Lambda. + - `lambda.memory_size`: Memory allocated to the pool Lambda in MB. + - `lambda.timeout`: Pool Lambda timeout in seconds. + - `lambda.zip`: Local path to the pool Lambda deployment package when S3 is not used. + - `lambda.subnet_ids`: Subnet IDs in which the pool Lambda runs. + - `lambda.parameter_store_tags`: JSON-encoded tags supplied to the pool Lambda for SSM parameters it creates. + - `lambda.principals`: Additional principals allowed to assume the pool Lambda role. + - `tags`: Common tags added to pool resources. + - `ghes`: GitHub Enterprise Server connection configuration. + - `ghes.url`: GitHub Enterprise Server URL; null when using public GitHub. + - `ghes.ssl_verify`: Whether the pool Lambda verifies the GitHub Enterprise Server TLS certificate. + - `github_app_parameters`: Ordered SSM parameter metadata for GitHub App credentials. + - `github_app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys. + - `github_app_parameters.id`: Ordered Parameter Store references for GitHub App IDs. + - `github_app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs. + - `runner`: Runner registration configuration used by the pool Lambda. + - `runner.disable_runner_autoupdate`: Whether GitHub runner automatic updates are disabled. + - `runner.ephemeral`: Whether runners register as ephemeral runners. + - `runner.enable_jit_config`: Whether runners use just-in-time registration configuration. + - `runner.labels`: Labels assigned to runners created by the pool Lambda. + - `runner.group_name`: GitHub runner group assigned to runners created by the pool Lambda. + - `runner.name_prefix`: Prefix used for runner names. + - `runner.pool_owner`: GitHub organization or repository that owns the runner pool. + - `runner.boot_time_in_minutes`: Webhook-provider runner boot timeout used by pool reconciliation. + - `runners_maximum_count`: Webhook-provider runner capacity limit enforced by the pool Lambda. + - `prefix`: Prefix used to name pool resources. + - `pool`: Scheduled pool targets. + - `pool[*].schedule_expression`: EventBridge Scheduler expression for a pool target. + - `pool[*].schedule_expression_timezone`: Time zone used to evaluate the schedule expression. + - `pool[*].size`: Desired runner count for the scheduled pool target. + - `include_busy_runners`: Whether busy runners count toward the desired pool size. + - `role_permissions_boundary`: Permissions boundary applied to IAM roles created for the pool. + - `kms_key_id`: Optional customer-managed KMS key ARN that the pool Lambda may use to decrypt encrypted parameters. + - `role_path`: IAM path applied to roles created for the pool. + - `ssm_token_path`: SSM path under which runner registration tokens are stored. + - `ssm_token_path_arn`: ARN matching the runner registration-token SSM path. + - `ssm_config_path`: SSM path under which runner configuration is stored. + - `arn_ssm_parameters_path_config`: ARN matching the runner configuration SSM path. + - `lambda_tags`: Tags added specifically to the pool Lambda function, overriding common tags with the same key. + - `log_group_tags`: Tags added specifically to the pool Lambda log group, overriding common tags with the same key. + - `user_agent`: User-Agent header used for GitHub API requests. + EOF + type = object({ + lambda = object({ + log_level = string + logging_retention_in_days = number + logging_kms_key_id = string + log_class = string + reserved_concurrent_executions = number + s3_bucket = string + s3_key = string + s3_object_version = string + security_group_ids = list(string) + runtime = string + architecture = string + memory_size = number + timeout = number + zip = string + subnet_ids = list(string) + parameter_store_tags = string + principals = optional(list(object({ + type = string + identifiers = list(string) + })), []) + }) + tags = map(string) + ghes = object({ + url = string + ssl_verify = string + }) + github_app_parameters = object({ + key_base64 = list(map(string)) + id = list(map(string)) + installation_id = list(object({ name = string, arn = string })) + }) + runner = object({ + disable_runner_autoupdate = bool + ephemeral = bool + enable_jit_config = bool + labels = list(string) + group_name = string + name_prefix = string + pool_owner = string + boot_time_in_minutes = number + }) + runners_maximum_count = number + prefix = string + pool = list(object({ + schedule_expression = string + schedule_expression_timezone = string + size = number + })) + include_busy_runners = bool + role_permissions_boundary = string + kms_key_id = optional(string, null) + role_path = string + ssm_token_path = string + ssm_token_path_arn = string + ssm_config_path = string + arn_ssm_parameters_path_config = string + lambda_tags = map(string) + log_group_tags = optional(map(string), {}) + user_agent = string + }) +} + +variable "runner_provider" { + description = <<-EOF + Compute provider integration used by the pool Lambda. + + - `type`: Compute provider type passed to scheduled pool invocations. + - `environment_variables`: Provider-specific environment variables added to the pool Lambda. + - `iam_policy_json`: Provider-specific IAM policy document merged into the pool Lambda policy. + - `managed_policy_enabled`: Whether to attach a provider-specific managed IAM policy to the pool Lambda role. + - `managed_policy_arn`: ARN of the provider-specific managed IAM policy to attach when enabled. + EOF + type = object({ + type = string + environment_variables = map(string) + iam_policy_json = string + managed_policy_enabled = bool + managed_policy_arn = optional(string, null) + }) +} + +variable "aws_partition" { + description = "(optional) partition for the arn if not 'aws'" + type = string + default = "aws" +} + +variable "tracing_config" { + description = <<-EOF + Tracing configuration for the pool Lambda. + + - `mode`: AWS X-Ray tracing mode. A null value disables tracing. + - `capture_http_requests`: Whether Powertools tracing captures outgoing HTTP requests. + - `capture_error`: Whether Powertools tracing captures errors as tracing metadata. + EOF + type = object({ + mode = optional(string, null) + capture_http_requests = optional(bool, false) + capture_error = optional(bool, false) + }) + default = {} +} diff --git a/modules/orchestration-providers/webhook/pool/versions.tf b/modules/orchestration-providers/webhook/pool/versions.tf new file mode 100644 index 0000000000..fcec7c620d --- /dev/null +++ b/modules/orchestration-providers/webhook/pool/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.21" + } + } +} diff --git a/modules/orchestration-providers/webhook/scale-down-state-diagram.md b/modules/orchestration-providers/webhook/scale-down-state-diagram.md new file mode 100644 index 0000000000..2e94d9ba79 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-down-state-diagram.md @@ -0,0 +1,150 @@ +# GitHub Actions Runner Scale-Down State Diagram + + + +The scale-down Lambda function runs on a scheduled basis (every 5 minutes by default) to manage GitHub Actions runner instances. It performs a two-phase cleanup process: first terminating confirmed orphaned instances, then evaluating active runners to maintain the desired idle capacity while removing unnecessary instances. + +```mermaid +stateDiagram-v2 + [*] --> ScheduledExecution : Cron Trigger every 5 min + + ScheduledExecution --> Phase1_OrphanTermination : Start Phase 1 + + state Phase1_OrphanTermination { + [*] --> ListOrphanInstances : Query EC2 for ghr orphan true + + ListOrphanInstances --> CheckOrphanType : For each orphan + + state CheckOrphanType <> + CheckOrphanType --> HasRunnerIdTag : Has ghr github runner id + CheckOrphanType --> TerminateOrphan : No runner ID tag + + HasRunnerIdTag --> LastChanceCheck : Query GitHub API + + state LastChanceCheck <> + LastChanceCheck --> ConfirmedOrphan : Offline and busy + LastChanceCheck --> FalsePositive : Exists and not problematic + + ConfirmedOrphan --> TerminateOrphan + FalsePositive --> RemoveOrphanTag + + TerminateOrphan --> NextOrphan : Continue processing + RemoveOrphanTag --> NextOrphan + + NextOrphan --> CheckOrphanType : More orphans? + NextOrphan --> Phase2_ActiveRunners : All processed + } + + Phase1_OrphanTermination --> Phase2_ActiveRunners : Phase 1 Complete + + state Phase2_ActiveRunners { + [*] --> ListActiveRunners : Query non-orphan EC2 instances + + ListActiveRunners --> GroupByOwner : Sort by owner and repo + + GroupByOwner --> ProcessOwnerGroup : For each owner + + state ProcessOwnerGroup { + [*] --> SortByStrategy : Apply eviction strategy + SortByStrategy --> ProcessRunner : Oldest first or newest first + + ProcessRunner --> QueryGitHub : Get GitHub runners for owner + + QueryGitHub --> MatchRunner : Find runner by instance ID suffix + + state MatchRunner <> + MatchRunner --> FoundInGitHub : Runner exists in GitHub + MatchRunner --> NotFoundInGitHub : Runner not in GitHub + + state FoundInGitHub { + [*] --> CheckMinimumTime : Has minimum runtime passed? + + state CheckMinimumTime <> + CheckMinimumTime --> TooYoung : Runtime less than minimum + CheckMinimumTime --> CheckIdleQuota : Runtime greater than or equal to minimum + + TooYoung --> NextRunner + + state CheckIdleQuota <> + CheckIdleQuota --> KeepIdle : Idle quota available + CheckIdleQuota --> CheckBusyState : Quota full + + KeepIdle --> NextRunner + + state CheckBusyState <> + CheckBusyState --> KeepBusy : Runner busy + CheckBusyState --> TerminateIdle : Runner idle + + KeepBusy --> NextRunner + TerminateIdle --> DeregisterFromGitHub + DeregisterFromGitHub --> TerminateInstance + TerminateInstance --> NextRunner + } + + state NotFoundInGitHub { + [*] --> CheckBootTime : Has boot time exceeded? + + state CheckBootTime <> + CheckBootTime --> StillBooting : Boot time less than threshold + CheckBootTime --> MarkOrphan : Boot time greater than or equal to threshold + + StillBooting --> NextRunner + MarkOrphan --> TagAsOrphan : Set ghr orphan true + TagAsOrphan --> NextRunner + } + + NextRunner --> ProcessRunner : More runners in group? + NextRunner --> NextOwnerGroup : Group complete + } + + NextOwnerGroup --> ProcessOwnerGroup : More owner groups? + NextOwnerGroup --> ExecutionComplete : All groups processed + } + + Phase2_ActiveRunners --> ExecutionComplete : Phase 2 Complete + + ExecutionComplete --> [*] : Wait for next cron trigger + + note right of LastChanceCheck + Uses ghr github runner id tag + for precise GitHub API lookup + end note + + note right of MatchRunner + Matches GitHub runner name + ending with EC2 instance ID + end note + + note right of CheckMinimumTime + Minimum running time in minutes + (Linux: 5min, Windows: 15min, OSX: 20min) + end note + + note right of CheckBootTime + Runner boot time in minutes + Default configuration value + end note +``` + + + +## Key Decision Points + +| State | Condition | Action | +|-------|-----------|--------| +| **Orphan w/ Runner ID** | GitHub: offline + busy | Terminate (confirmed orphan) | +| **Orphan w/ Runner ID** | GitHub: exists + healthy | Remove orphan tag (false positive) | +| **Orphan w/o Runner ID** | Always | Terminate (no way to verify) | +| **Active Runner Found** | Runtime < minimum | Keep (too young) | +| **Active Runner Found** | Idle quota available | Keep as idle | +| **Active Runner Found** | Quota full + idle | Terminate + deregister | +| **Active Runner Found** | Quota full + busy | Keep running | +| **Active Runner Missing** | Boot time exceeded | Mark as orphan | +| **Active Runner Missing** | Still booting | Wait | + +## Configuration Parameters + +- **Cron Schedule**: `cron(*/5 * * * ? *)` (every 5 minutes) +- **Minimum Runtime**: Linux 5min, Windows 15min, OSX 20min +- **Boot Timeout**: Configurable via `orchestration_provider.webhook.runner.boot_time_in_minutes`; stable-v1 inputs are translated from `runner_boot_time_in_minutes`. +- **Idle Config**: Per-environment configuration for desired idle runners diff --git a/modules/orchestration-providers/webhook/scale-runners.tf b/modules/orchestration-providers/webhook/scale-runners.tf new file mode 100644 index 0000000000..caf79eefb5 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners.tf @@ -0,0 +1,61 @@ +module "scale_runners" { + source = "./scale-runners" + + aws_partition = var.aws_partition + + config = { + prefix = local.resolved_config.prefix + lambda = { + artifact = local.resolved_config.lambda.artifact + runtime = local.resolved_config.lambda.runtime + architecture = local.resolved_config.lambda.architecture + vpc = { + subnet_ids = local.resolved_config.lambda.subnet_ids + security_group_ids = local.resolved_config.lambda.security_group_ids + } + role = local.resolved_config.lambda.role + } + runner = local.resolved_config.runner + github = local.resolved_config.github + queue = { + build = local.resolved_config.queue.build + kms_key_id = local.resolved_config.queue.kms_key_id + event_source_mapping = local.resolved_config.queue.event_source_mapping + } + ssm = local.resolved_config.ssm + observability = { + logs = local.resolved_config.observability.logs + tracing = local.resolved_config.observability.tracing + metrics = local.resolved_config.observability.metrics + } + scale_up = merge(local.resolved_config.scale_up, { + job_queued_check_enabled = local.enable_job_queued_check + tags = { + resources = local.scale_up_tags + lambda = local.scale_up_lambda_tags + log_group = local.scale_up_log_tags + event_source_mapping = local.scale_up_queue_tags + } + }) + scale_down = merge(local.resolved_config.scale_down, { + tags = { + resources = local.scale_down_tags + lambda = local.scale_down_lambda_tags + log_group = local.scale_down_log_tags + } + }) + job_retry = { + enabled = local.job_retry_enabled + max_attempts = local.resolved_config.job_retry.max_attempts + delay_in_seconds = local.resolved_config.job_retry.delay_in_seconds + delay_backoff = local.resolved_config.job_retry.delay_backoff + queue = one(module.job_retry[*].job_retry_check_queue) + } + } + + runner_provider = { + type = var.runner_provider.type + scale_up = var.runner_provider.scale_up + scale_down = var.runner_provider.scale_down + } +} diff --git a/modules/orchestration-providers/webhook/scale-runners/README.md b/modules/orchestration-providers/webhook/scale-runners/README.md new file mode 100644 index 0000000000..3b096f9b85 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/README.md @@ -0,0 +1,79 @@ +# Scale runners module + +> This module is treated as an internal module; breaking changes do not trigger a major release bump. + +This provider-neutral child module owns the scale-up and scale-down Lambda functions, their event sources and schedules, and their IAM and logging resources. `runner-config` supplies common configuration through the webhook orchestration provider together with the selected compute provider's environment and IAM fragments. + +The module is an implementation detail of the experimental runner configuration. It is composed by the webhook orchestration provider and is not intended to be called directly. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_event_rule.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_target.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_cloudwatch_log_group.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_cloudwatch_log_group.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_role.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.job_retry_sqs_publish](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_down_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_down_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_up_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_up_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.service_linked_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.scale_down_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.scale_up_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_lambda_event_source_mapping.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource | +| [aws_lambda_function.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_lambda_function.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_lambda_permission.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [aws_lambda_permission.scale_runners_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [terraform_data.validate_config](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [aws_iam_policy_document.lambda_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_down_common](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_down_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up_common](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up_job_retry_publish](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | AWS partition used to construct IAM policy ARNs. | `string` | `"aws"` | no | +| [config](#input\_config) | Provider-neutral scale-up and scale-down configuration assembled by runner-config.

- `prefix`: Prefix used to name scaling resources.
- `lambda.artifact.zip`: Resolved local control-plane archive.
- `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive.
- `lambda.artifact.s3.key`: Object key of the Lambda archive.
- `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive.
- `lambda.runtime`: Runtime used by both scaling Lambdas.
- `lambda.architecture`: Instruction-set architecture used by both scaling Lambdas.
- `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration.
- `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration.
- `lambda.role.path`: IAM path used for the scaling Lambda roles.
- `lambda.role.permissions_boundary`: Optional permissions boundary for the scaling Lambda roles.
- `lambda.role.principals`: Additional principals allowed to assume the scaling Lambda roles.
- `runner.os`: Runner operating system used for the minimum-runtime default.
- `runner.auto_update_disabled`: Disables the GitHub runner application's built-in updater.
- `runner.ephemeral`: Registers runners in ephemeral mode.
- `runner.jit_config_enabled`: Enables or disables just-in-time runner configuration.
- `runner.labels`: Labels supplied when a runner is registered.
- `runner.group_name`: GitHub runner group used during registration.
- `runner.name_prefix`: Prefix added to registered runner names.
- `runner.boot_time_in_minutes`: Webhook-provider runner boot timeout used by scale-down.
- `runner.maximum_count`: Webhook-provider runner capacity limit for this runner configuration.
- `github.organization_runners`: Registers organization runners when true.
- `github.enterprise_server.url`: Optional GitHub Enterprise Server URL.
- `github.enterprise_server.ssl_verify`: Enables TLS verification for GitHub Enterprise Server.
- `github.user_agent`: Optional User-Agent sent to GitHub.
- `github.app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys.
- `github.app_parameters.id`: Ordered Parameter Store references for GitHub App IDs.
- `github.app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs.
- `queue.build.arn`: ARN of the build queue consumed by scale-up.
- `queue.kms_key_id`: Optional KMS key ARN used to encrypt the build queue. This is distinct from the Parameter Store key.
- `queue.event_source_mapping.batch_size`: Maximum records delivered per scale-up invocation.
- `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum event batching window.
- `ssm.token_path`: Parameter Store path used for registration tokens.
- `ssm.token_path_arn`: ARN of the Parameter Store path used for registration tokens.
- `ssm.config_path`: Parameter Store path used for persistent runner configuration.
- `ssm.config_path_arn`: ARN of the persistent runner configuration path.
- `ssm.kms_key_id`: Optional KMS key ARN used to decrypt shared parameters. Its value may be unknown until apply.
- `ssm.parameter_store_tags`: JSON-encoded tags applied to parameters created at runtime.
- `observability.logs`: Shared logging level, retention, encryption, and log-class configuration.
- `observability.tracing`: Lambda X-Ray and tracing-helper configuration.
- `observability.metrics`: Metrics enablement, namespace, and GitHub rate-limit metric configuration.
- `scale_up`: Scale-up Lambda sizing, concurrency, queued-job behavior, and resolved resource tag maps.
- `scale_up.tags.resources`: Tags for the scale-up IAM role and other component resources.
- `scale_up.tags.lambda`: Tags for the scale-up Lambda function.
- `scale_up.tags.log_group`: Tags for the scale-up log group.
- `scale_up.tags.event_source_mapping`: Tags for the build-queue event-source mapping.
- `scale_down`: Scale-down Lambda sizing, schedule, idle configuration, minimum runtime, and resolved resource tag maps.
- `scale_down.tags.resources`: Tags for the scale-down IAM role and EventBridge rule.
- `scale_down.tags.lambda`: Tags for the scale-down Lambda function.
- `scale_down.tags.log_group`: Tags for the scale-down log group.
- `job_retry.enabled`: Enables publishing retry checks from scale-up.
- `job_retry.queue`: Retry queue ARN and URL. Required when job retry is enabled.
- `job_retry.max_attempts`: Maximum queued-job retry attempts.
- `job_retry.delay_in_seconds`: Initial delay before checking the queued job.
- `job_retry.delay_backoff`: Multiplier applied to subsequent delays. |
object({
prefix = string
lambda = object({
artifact = object({
zip = string
s3 = object({
bucket = optional(string, null)
key = optional(string, null)
object_version = optional(string, null)
})
})
runtime = string
architecture = string
vpc = object({
subnet_ids = list(string)
security_group_ids = list(string)
})
role = object({
path = string
permissions_boundary = optional(string, null)
principals = optional(list(object({
type = string
identifiers = list(string)
})), [])
})
})
runner = object({
os = string
auto_update_disabled = bool
ephemeral = bool
jit_config_enabled = optional(bool, null)
labels = list(string)
group_name = string
name_prefix = string
boot_time_in_minutes = number
maximum_count = number
})
github = object({
organization_runners = bool
enterprise_server = object({
url = optional(string, null)
ssl_verify = bool
})
user_agent = optional(string, null)
app_parameters = object({
key_base64 = list(map(string))
id = list(map(string))
installation_id = list(object({ name = string, arn = string }))
})
})
queue = object({
build = object({
arn = string
})
kms_key_id = optional(string, null)
event_source_mapping = object({
batch_size = number
maximum_batching_window_in_seconds = number
})
})
ssm = object({
token_path = string
token_path_arn = string
config_path = string
config_path_arn = string
parameter_store_tags = string
kms_key_id = optional(string, null)
})
observability = object({
logs = object({
level = string
retention_in_days = number
kms_key_id = optional(string, null)
class = string
})
tracing = object({
mode = optional(string, null)
capture_http_requests = bool
capture_error = bool
})
metrics = object({
enabled = bool
namespace = string
metric = object({
github_app_rate_limit = object({
enabled = bool
})
})
})
})
scale_up = object({
memory_size = number
timeout = number
reserved_concurrent_executions = number
job_queued_check_enabled = bool
tags = object({
resources = map(string)
lambda = map(string)
log_group = map(string)
event_source_mapping = map(string)
})
})
scale_down = object({
memory_size = number
timeout = number
schedule_expression = string
minimum_running_time_in_minutes = optional(number, null)
idle_config = list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = string
}))
tags = object({
resources = map(string)
lambda = map(string)
log_group = map(string)
})
})
job_retry = object({
enabled = bool
max_attempts = number
delay_in_seconds = number
delay_backoff = number
queue = optional(object({
arn = string
url = string
}), null)
})
})
| n/a | yes | +| [runner\_provider](#input\_runner\_provider) | Selected compute-provider integration for the scaling control plane.

- `type`: Compute-provider discriminator supplied to both Lambdas.
- `scale_up.environment_variables`: Provider-specific scale-up environment variables.
- `scale_up.iam_policy_json`: Provider-specific IAM policy merged into the common scale-up policy.
- `scale_up.additional_iam_policy_json`: Optional additional provider policy attached separately to the scale-up role.
- `scale_up.managed_policy`: Optional provider-managed policy attachment. Object presence controls attachment creation.
- `scale_up.managed_policy.arn`: ARN of the provider-managed policy. The ARN may remain unknown until apply.
- `scale_down.environment_variables`: Provider-specific scale-down environment variables.
- `scale_down.iam_policy_json`: Provider-specific IAM policy merged into the common scale-down policy. |
object({
type = string
scale_up = object({
environment_variables = map(string)
iam_policy_json = string
additional_iam_policy_json = optional(string, null)
managed_policy = optional(object({
arn = string
}), null)
})
scale_down = object({
environment_variables = map(string)
iam_policy_json = string
})
})
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [scale\_down](#output\_scale\_down) | Scale-down Lambda resources. | +| [scale\_up](#output\_scale\_up) | Scale-up Lambda resources. | + diff --git a/modules/orchestration-providers/webhook/scale-runners/common-config.tf b/modules/orchestration-providers/webhook/scale-runners/common-config.tf new file mode 100644 index 0000000000..7c8a04d095 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/common-config.tf @@ -0,0 +1,20 @@ +locals { + vpc_enabled = ( + length(var.config.lambda.vpc.subnet_ids) > 0 && + length(var.config.lambda.vpc.security_group_ids) > 0 + ) + + job_retry_config = var.config.job_retry.enabled ? { + enable = true + maxAttempts = var.config.job_retry.max_attempts + delayInSeconds = var.config.job_retry.delay_in_seconds + delayBackoff = var.config.job_retry.delay_backoff + queueUrl = var.config.job_retry.queue.url + } : {} + + min_runtime_defaults = { + windows = 15 + linux = 5 + osx = 20 + } +} diff --git a/modules/orchestration-providers/webhook/scale-runners/lambda-iam-policies.tf b/modules/orchestration-providers/webhook/scale-runners/lambda-iam-policies.tf new file mode 100644 index 0000000000..0c9214d0d3 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/lambda-iam-policies.tf @@ -0,0 +1,36 @@ +data "aws_iam_policy_document" "lambda_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + + dynamic "principals" { + for_each = var.config.lambda.role.principals + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + } +} + +data "aws_iam_policy_document" "lambda_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + + # AWS X-Ray write/read trace APIs do not support resource-level permissions. + statement { + sid = "AllowXRay" + effect = "Allow" + actions = [ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments", + ] + resources = ["*"] + } +} diff --git a/modules/orchestration-providers/webhook/scale-runners/outputs.tf b/modules/orchestration-providers/webhook/scale-runners/outputs.tf new file mode 100644 index 0000000000..74d54d2101 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/outputs.tf @@ -0,0 +1,17 @@ +output "scale_up" { + description = "Scale-up Lambda resources." + value = { + lambda = aws_lambda_function.scale_up + log_group = aws_cloudwatch_log_group.scale_up + role = aws_iam_role.scale_up + } +} + +output "scale_down" { + description = "Scale-down Lambda resources." + value = { + lambda = aws_lambda_function.scale_down + log_group = aws_cloudwatch_log_group.scale_down + role = aws_iam_role.scale_down + } +} diff --git a/modules/orchestration-providers/webhook/scale-runners/scale-down-iam-policies.tf b/modules/orchestration-providers/webhook/scale-runners/scale-down-iam-policies.tf new file mode 100644 index 0000000000..b95cb9e686 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/scale-down-iam-policies.tf @@ -0,0 +1,46 @@ +data "aws_iam_policy_document" "scale_down_common" { + statement { + sid = "WebhookScaleDownReadGitHubAppParameters" + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + resources = concat( + [for p in var.config.github.app_parameters.id : p.arn], + [for p in var.config.github.app_parameters.key_base64 : p.arn], + [for p in var.config.github.app_parameters.installation_id : p.arn if p != null], + ) + } + + dynamic "statement" { + for_each = var.config.ssm.kms_key_id == null ? [] : [var.config.ssm.kms_key_id] + iterator = kms_key + + content { + sid = "WebhookScaleDownDecryptParameterStore" + effect = "Allow" + actions = ["kms:Decrypt"] + resources = [kms_key.value] + } + } +} + +data "aws_iam_policy_document" "scale_down" { + source_policy_documents = [ + data.aws_iam_policy_document.scale_down_common.json, + var.runner_provider.scale_down.iam_policy_json, + ] +} + +data "aws_iam_policy_document" "scale_down_logging" { + statement { + sid = "WebhookScaleDownWriteLogs" + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = ["${aws_cloudwatch_log_group.scale_down.arn}*"] + } +} diff --git a/modules/orchestration-providers/webhook/scale-runners/scale-down.tf b/modules/orchestration-providers/webhook/scale-runners/scale-down.tf new file mode 100644 index 0000000000..44e651d78c --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/scale-down.tf @@ -0,0 +1,116 @@ +resource "aws_lambda_function" "scale_down" { + s3_bucket = var.config.lambda.artifact.s3.bucket + s3_key = var.config.lambda.artifact.s3.key + s3_object_version = var.config.lambda.artifact.s3.object_version + filename = var.config.lambda.artifact.s3.bucket == null ? var.config.lambda.artifact.zip : null + source_code_hash = var.config.lambda.artifact.s3.bucket == null ? filebase64sha256(var.config.lambda.artifact.zip) : null + function_name = "${var.config.prefix}-scale-down" + role = aws_iam_role.scale_down.arn + handler = "index.scaleDownHandler" + runtime = var.config.lambda.runtime + timeout = var.config.scale_down.timeout + tags = var.config.scale_down.tags.lambda + memory_size = var.config.scale_down.memory_size + architectures = [var.config.lambda.architecture] + + environment { + variables = merge(var.runner_provider.scale_down.environment_variables, { + ENVIRONMENT = var.config.prefix + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.github_app_rate_limit.enabled + GHES_URL = var.config.github.enterprise_server.url + USER_AGENT = var.config.github.user_agent + LOG_LEVEL = upper(var.config.observability.logs.level) + MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.config.scale_down.minimum_running_time_in_minutes, local.min_runtime_defaults[var.config.runner.os]) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 + PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.config.github.app_parameters.id : p.name]) + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.config.github.app_parameters.key_base64 : p.name]) + PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.config.github.app_parameters.installation_id : p != null ? p.name : ""]) + POWERTOOLS_LOGGER_LOG_EVENT = var.config.observability.logs.level == "debug" ? "true" : "false" + SCALE_DOWN_CONFIG = jsonencode(var.config.scale_down.idle_config) + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-scale-down" + POWERTOOLS_METRICS_NAMESPACE = var.config.observability.metrics.namespace + POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error + COMPUTE_PROVIDER_TYPE = var.runner_provider.type + RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes + }) + } + + dynamic "vpc_config" { + for_each = local.vpc_enabled ? [true] : [] + + content { + security_group_ids = var.config.lambda.vpc.security_group_ids + subnet_ids = var.config.lambda.vpc.subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.config.observability.tracing.mode != null ? [true] : [] + + content { + mode = var.config.observability.tracing.mode + } + } +} + +resource "aws_cloudwatch_log_group" "scale_down" { + name = "/aws/lambda/${aws_lambda_function.scale_down.function_name}" + retention_in_days = var.config.observability.logs.retention_in_days + kms_key_id = var.config.observability.logs.kms_key_id + log_group_class = var.config.observability.logs.class + tags = var.config.scale_down.tags.log_group +} + +resource "aws_cloudwatch_event_rule" "scale_down" { + name = "${var.config.prefix}-scale-down-rule" + schedule_expression = var.config.scale_down.schedule_expression + tags = var.config.scale_down.tags.resources +} + +resource "aws_cloudwatch_event_target" "scale_down" { + rule = aws_cloudwatch_event_rule.scale_down.name + arn = aws_lambda_function.scale_down.arn +} + +resource "aws_lambda_permission" "scale_down" { + statement_id = "AllowExecutionFromCloudWatch" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.scale_down.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.scale_down.arn +} + +resource "aws_iam_role" "scale_down" { + name = "${substr("${var.config.prefix}-scale-down-lambda", 0, 54)}-${substr(md5("${var.config.prefix}-scale-down-lambda"), 0, 8)}" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json + path = var.config.lambda.role.path + permissions_boundary = var.config.lambda.role.permissions_boundary + tags = var.config.scale_down.tags.resources +} + +resource "aws_iam_role_policy" "scale_down" { + name = "scale-down-policy" + role = aws_iam_role.scale_down.name + policy = data.aws_iam_policy_document.scale_down.json +} + +resource "aws_iam_role_policy" "scale_down_logging" { + name = "logging-policy" + role = aws_iam_role.scale_down.name + policy = data.aws_iam_policy_document.scale_down_logging.json +} + +resource "aws_iam_role_policy_attachment" "scale_down_vpc_execution_role" { + count = local.vpc_enabled ? 1 : 0 + role = aws_iam_role.scale_down.name + policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +resource "aws_iam_role_policy" "scale_down_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + name = "xray-policy" + policy = data.aws_iam_policy_document.lambda_xray[0].json + role = aws_iam_role.scale_down.name +} diff --git a/modules/orchestration-providers/webhook/scale-runners/scale-up-iam-policies.tf b/modules/orchestration-providers/webhook/scale-runners/scale-up-iam-policies.tf new file mode 100644 index 0000000000..b3c87b8ad7 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/scale-up-iam-policies.tf @@ -0,0 +1,102 @@ +data "aws_iam_policy_document" "scale_up_common" { + statement { + sid = "WebhookScaleUpWriteRuntimeParameters" + effect = "Allow" + actions = [ + "ssm:PutParameter", + "ssm:AddTagsToResource", + ] + resources = [ + var.config.ssm.token_path_arn, + "${var.config.ssm.token_path_arn}/*", + var.config.ssm.config_path_arn, + "${var.config.ssm.config_path_arn}/*", + ] + } + + statement { + sid = "WebhookScaleUpReadGitHubAppAndRunnerConfigParameters" + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + resources = concat( + [for p in var.config.github.app_parameters.id : p.arn], + [for p in var.config.github.app_parameters.key_base64 : p.arn], + [for p in var.config.github.app_parameters.installation_id : p.arn if p != null], + [ + var.config.ssm.config_path_arn, + "${var.config.ssm.config_path_arn}/*", + ], + ) + } + + statement { + sid = "WebhookScaleUpConsumeBuildQueue" + effect = "Allow" + actions = [ + "sqs:ReceiveMessage", + "sqs:GetQueueAttributes", + "sqs:DeleteMessage", + ] + resources = [var.config.queue.build.arn] + } + + dynamic "statement" { + for_each = var.config.ssm.kms_key_id == null ? [] : [var.config.ssm.kms_key_id] + iterator = kms_key + + content { + sid = "WebhookScaleUpDecryptParameterStore" + effect = "Allow" + actions = ["kms:Decrypt"] + resources = [kms_key.value] + } + } + + dynamic "statement" { + for_each = var.config.queue.kms_key_id == null ? [] : [var.config.queue.kms_key_id] + iterator = kms_key + + content { + sid = "WebhookScaleUpDecryptBuildQueue" + effect = "Allow" + actions = ["kms:Decrypt"] + resources = [kms_key.value] + } + } +} + +data "aws_iam_policy_document" "scale_up" { + source_policy_documents = [ + data.aws_iam_policy_document.scale_up_common.json, + var.runner_provider.scale_up.iam_policy_json, + ] +} + +data "aws_iam_policy_document" "scale_up_logging" { + statement { + sid = "WebhookScaleUpWriteLogs" + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = ["${aws_cloudwatch_log_group.scale_up.arn}*"] + } +} + +data "aws_iam_policy_document" "scale_up_job_retry_publish" { + count = var.config.job_retry.enabled ? 1 : 0 + + statement { + sid = "WebhookScaleUpPublishJobRetryQueue" + effect = "Allow" + actions = [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + ] + resources = [var.config.job_retry.queue.arn] + } +} diff --git a/modules/orchestration-providers/webhook/scale-runners/scale-up.tf b/modules/orchestration-providers/webhook/scale-runners/scale-up.tf new file mode 100644 index 0000000000..2997aeac21 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/scale-up.tf @@ -0,0 +1,146 @@ +resource "aws_lambda_function" "scale_up" { + s3_bucket = var.config.lambda.artifact.s3.bucket + s3_key = var.config.lambda.artifact.s3.key + s3_object_version = var.config.lambda.artifact.s3.object_version + filename = var.config.lambda.artifact.s3.bucket == null ? var.config.lambda.artifact.zip : null + source_code_hash = var.config.lambda.artifact.s3.bucket == null ? filebase64sha256(var.config.lambda.artifact.zip) : null + function_name = "${var.config.prefix}-scale-up" + role = aws_iam_role.scale_up.arn + handler = "index.scaleUpHandler" + runtime = var.config.lambda.runtime + timeout = var.config.scale_up.timeout + reserved_concurrent_executions = var.config.scale_up.reserved_concurrent_executions + memory_size = var.config.scale_up.memory_size + tags = var.config.scale_up.tags.lambda + architectures = [var.config.lambda.architecture] + + environment { + variables = merge(var.runner_provider.scale_up.environment_variables, { + DISABLE_RUNNER_AUTOUPDATE = var.config.runner.auto_update_disabled + ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral + ENABLE_JIT_CONFIG = var.config.runner.jit_config_enabled + ENABLE_JOB_QUEUED_CHECK = var.config.scale_up.job_queued_check_enabled + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.github_app_rate_limit.enabled + ENABLE_ORGANIZATION_RUNNERS = var.config.github.organization_runners + ENVIRONMENT = var.config.prefix + GHES_URL = var.config.github.enterprise_server.url + USER_AGENT = var.config.github.user_agent + LOG_LEVEL = upper(var.config.observability.logs.level) + MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.config.scale_down.minimum_running_time_in_minutes, local.min_runtime_defaults[var.config.runner.os]) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 + PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.config.github.app_parameters.id : p.name]) + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.config.github.app_parameters.key_base64 : p.name]) + PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.config.github.app_parameters.installation_id : p != null ? p.name : ""]) + POWERTOOLS_LOGGER_LOG_EVENT = var.config.observability.logs.level == "debug" ? "true" : "false" + POWERTOOLS_METRICS_NAMESPACE = var.config.observability.metrics.namespace + POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error + RUNNER_LABELS = lower(join(",", var.config.runner.labels)) + RUNNER_GROUP_NAME = var.config.runner.group_name + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + COMPUTE_PROVIDER_TYPE = var.runner_provider.type + RUNNERS_MAXIMUM_COUNT = var.config.runner.maximum_count + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-scale-up" + SSM_TOKEN_PATH = var.config.ssm.token_path + SSM_CONFIG_PATH = var.config.ssm.config_path + SSM_PARAMETER_STORE_TAGS = var.config.ssm.parameter_store_tags + JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) + }) + } + + dynamic "vpc_config" { + for_each = local.vpc_enabled ? [true] : [] + + content { + security_group_ids = var.config.lambda.vpc.security_group_ids + subnet_ids = var.config.lambda.vpc.subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.config.observability.tracing.mode != null ? [true] : [] + + content { + mode = var.config.observability.tracing.mode + } + } +} + +resource "aws_cloudwatch_log_group" "scale_up" { + name = "/aws/lambda/${aws_lambda_function.scale_up.function_name}" + retention_in_days = var.config.observability.logs.retention_in_days + kms_key_id = var.config.observability.logs.kms_key_id + log_group_class = var.config.observability.logs.class + tags = var.config.scale_up.tags.log_group +} + +resource "aws_lambda_event_source_mapping" "scale_up" { + event_source_arn = var.config.queue.build.arn + function_name = aws_lambda_function.scale_up.arn + function_response_types = ["ReportBatchItemFailures"] + batch_size = var.config.queue.event_source_mapping.batch_size + maximum_batching_window_in_seconds = var.config.queue.event_source_mapping.maximum_batching_window_in_seconds + tags = var.config.scale_up.tags.event_source_mapping +} + +resource "aws_lambda_permission" "scale_runners_lambda" { + statement_id = "AllowExecutionFromSQS" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.scale_up.function_name + principal = "sqs.amazonaws.com" + source_arn = var.config.queue.build.arn +} + +resource "aws_iam_role" "scale_up" { + name = "${substr("${var.config.prefix}-scale-up-lambda", 0, 54)}-${substr(md5("${var.config.prefix}-scale-up-lambda"), 0, 8)}" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json + path = var.config.lambda.role.path + permissions_boundary = var.config.lambda.role.permissions_boundary + tags = var.config.scale_up.tags.resources +} + +resource "aws_iam_role_policy" "scale_up" { + name = "scale-up-policy" + role = aws_iam_role.scale_up.name + policy = data.aws_iam_policy_document.scale_up.json +} + +resource "aws_iam_role_policy" "scale_up_logging" { + name = "logging-policy" + role = aws_iam_role.scale_up.name + policy = data.aws_iam_policy_document.scale_up_logging.json +} + +resource "aws_iam_role_policy" "service_linked_role" { + count = var.runner_provider.scale_up.additional_iam_policy_json != null ? 1 : 0 + name = "service_linked_role" + role = aws_iam_role.scale_up.name + policy = var.runner_provider.scale_up.additional_iam_policy_json +} + +resource "aws_iam_role_policy_attachment" "scale_up_vpc_execution_role" { + count = local.vpc_enabled ? 1 : 0 + role = aws_iam_role.scale_up.name + policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +resource "aws_iam_role_policy_attachment" "provider" { + count = var.runner_provider.scale_up.managed_policy != null ? 1 : 0 + role = aws_iam_role.scale_up.name + policy_arn = var.runner_provider.scale_up.managed_policy.arn +} + +resource "aws_iam_role_policy" "scale_up_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + name = "xray-policy" + policy = data.aws_iam_policy_document.lambda_xray[0].json + role = aws_iam_role.scale_up.name +} + +resource "aws_iam_role_policy" "job_retry_sqs_publish" { + count = var.config.job_retry.enabled ? 1 : 0 + name = "publish-retry-check-sqs-policy" + role = aws_iam_role.scale_up.name + policy = data.aws_iam_policy_document.scale_up_job_retry_publish[0].json +} diff --git a/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl b/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl new file mode 100644 index 0000000000..acca05980b --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl @@ -0,0 +1,460 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/scale-runners-test" + } + } +} + +variables { + aws_partition = "aws-us-gov" + + config = { + prefix = "scale-runners-test" + lambda = { + artifact = { + zip = "runners.zip" + s3 = { + bucket = "lambda-artifacts" + key = "runners.zip" + object_version = "test-version" + } + } + runtime = "nodejs24.x" + architecture = "arm64" + vpc = { + subnet_ids = ["subnet-12345678"] + security_group_ids = ["sg-12345678"] + } + role = { + path = "/scale-runners-test/" + permissions_boundary = "arn:aws-us-gov:iam::123456789012:policy/permissions-boundary" + principals = [{ + type = "AWS" + identifiers = ["arn:aws-us-gov:iam::123456789012:role/local-testing"] + }] + } + } + runner = { + os = "windows" + auto_update_disabled = true + ephemeral = true + jit_config_enabled = true + labels = ["Self-Hosted", "MicroVM"] + group_name = "test-group" + name_prefix = "test-runner-" + boot_time_in_minutes = 12 + maximum_count = 7 + } + github = { + organization_runners = true + enterprise_server = { + url = "https://github.example.com" + ssl_verify = false + } + user_agent = "scale-runners-test" + app_parameters = { + key_base64 = [ + { + name = "/github-runner/key-base64" + arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/key-base64" + }, + { + name = "/github-runner/key-base64-2" + arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/key-base64-2" + }, + ] + id = [ + { + name = "/github-runner/app-id" + arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/app-id" + }, + { + name = "/github-runner/app-id-2" + arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/app-id-2" + }, + ] + installation_id = [ + null, + { + name = "/github-runner/installation-id-2" + arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/installation-id-2" + }, + ] + } + } + queue = { + build = { + arn = "arn:aws-us-gov:sqs:us-gov-west-1:123456789012:build-queue" + } + kms_key_id = "arn:aws-us-gov:kms:us-gov-west-1:123456789012:key/build-queue-test" + event_source_mapping = { + batch_size = 25 + maximum_batching_window_in_seconds = 5 + } + } + ssm = { + token_path = "/github-runner/tokens" + token_path_arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/tokens" + config_path = "/github-runner/config" + config_path_arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/config" + parameter_store_tags = jsonencode([{ + Key = "Environment" + Value = "test" + }]) + kms_key_id = "arn:aws-us-gov:kms:us-gov-west-1:123456789012:key/scale-runners-test" + } + observability = { + logs = { + level = "debug" + retention_in_days = 14 + kms_key_id = "arn:aws-us-gov:kms:us-gov-west-1:123456789012:key/logs" + class = "INFREQUENT_ACCESS" + } + tracing = { + mode = "Active" + capture_http_requests = true + capture_error = true + } + metrics = { + enabled = true + namespace = "ScaleRunnersTest" + metric = { + github_app_rate_limit = { + enabled = true + } + } + } + } + scale_up = { + memory_size = 768 + timeout = 90 + reserved_concurrent_executions = 2 + job_queued_check_enabled = true + tags = { + resources = { Scope = "scale-up" } + lambda = { Scope = "scale-up-lambda" } + log_group = { Scope = "scale-up-log" } + event_source_mapping = { Scope = "scale-up-queue" } + } + } + scale_down = { + memory_size = 640 + timeout = 75 + schedule_expression = "rate(10 minutes)" + minimum_running_time_in_minutes = null + idle_config = [{ + cron = "* * * * *" + timeZone = "UTC" + idleCount = 2 + evictionStrategy = "oldest_first" + }] + tags = { + resources = { Scope = "scale-down" } + lambda = { Scope = "scale-down-lambda" } + log_group = { Scope = "scale-down-log" } + } + } + job_retry = { + enabled = true + max_attempts = 4 + delay_in_seconds = 120 + delay_backoff = 3 + queue = { + arn = "arn:aws-us-gov:sqs:us-gov-west-1:123456789012:job-retry" + url = "https://sqs.us-gov-west-1.amazonaws.com/123456789012/job-retry" + } + } + } + + runner_provider = { + type = "microvm" + scale_up = { + environment_variables = { + MICROVM_CLUSTER = "runner-cluster" + } + iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["microvm:CreateRunner"] + Resource = ["*"] + }] + }) + additional_iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["iam:CreateServiceLinkedRole"] + Resource = ["*"] + }] + }) + managed_policy = { + arn = "arn:aws-us-gov:iam::123456789012:policy/microvm-scale-up" + } + } + scale_down = { + environment_variables = { + MICROVM_CLUSTER = "runner-cluster" + } + iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["microvm:DeleteRunner"] + Resource = ["*"] + }] + }) + } + } +} + +run "assembles_provider_neutral_scaling_control_plane" { + command = plan + + assert { + condition = ( + length(data.aws_iam_policy_document.lambda_assume_role.statement[0].principals) == 2 && + contains(data.aws_iam_policy_document.lambda_assume_role.statement[0].principals[*].type, "AWS") + ) + error_message = "The scaling Lambda trust policy must include configured additional principals." + } + + assert { + condition = ( + toset(keys(output.scale_up)) == toset(["lambda", "log_group", "role"]) + && toset(keys(output.scale_down)) == toset(["lambda", "log_group", "role"]) + ) + error_message = "Scale runners must expose nested scale-up and scale-down Lambda resource contracts." + } + + assert { + condition = ( + aws_lambda_function.scale_up.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "microvm" + && aws_lambda_function.scale_down.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "microvm" + && aws_lambda_function.scale_up.environment[0].variables["RUNNERS_MAXIMUM_COUNT"] == "7" + && aws_lambda_function.scale_down.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "12" + && aws_lambda_function.scale_up.environment[0].variables["MICROVM_CLUSTER"] == "runner-cluster" + && aws_lambda_function.scale_down.environment[0].variables["MICROVM_CLUSTER"] == "runner-cluster" + && !contains(keys(aws_lambda_function.scale_up.environment[0].variables), "INSTANCE_TYPES") + ) + error_message = "The common scaling Lambdas must select the compute provider while injecting webhook-owned capacity and boot-time settings." + } + + assert { + condition = ( + aws_lambda_function.scale_up.environment[0].variables["LOG_LEVEL"] == "DEBUG" + && aws_lambda_function.scale_up.environment[0].variables["RUNNER_LABELS"] == "self-hosted,microvm" + && aws_lambda_function.scale_up.environment[0].variables["MINIMUM_RUNNING_TIME_IN_MINUTES"] == "15" + && aws_lambda_function.scale_down.environment[0].variables["MINIMUM_RUNNING_TIME_IN_MINUTES"] == "15" + && aws_lambda_function.scale_up.environment[0].variables["NODE_TLS_REJECT_UNAUTHORIZED"] == "0" + && jsondecode(aws_lambda_function.scale_up.environment[0].variables["SSM_PARAMETER_STORE_TAGS"])[0].Value == "test" + ) + error_message = "Scale runners must assemble shared runner, logging, TLS, lifetime, and Parameter Store environment variables." + } + + assert { + condition = ( + aws_lambda_function.scale_up.environment[0].variables["PARAMETER_GITHUB_APP_ID_NAME"] == "/github-runner/app-id:/github-runner/app-id-2" + && aws_lambda_function.scale_down.environment[0].variables["PARAMETER_GITHUB_APP_KEY_BASE64_NAME"] == "/github-runner/key-base64:/github-runner/key-base64-2" + && aws_lambda_function.scale_up.environment[0].variables["PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME"] == ":/github-runner/installation-id-2" + && contains(data.aws_iam_policy_document.scale_up_common.statement[1].resources, "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/app-id-2") + && contains(data.aws_iam_policy_document.scale_down_common.statement[0].resources, "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/key-base64-2") + && contains(data.aws_iam_policy_document.scale_down_common.statement[0].resources, "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/installation-id-2") + ) + error_message = "Scale-up and scale-down must pass every GitHub App parameter and grant access to every corresponding SSM ARN." + } + + assert { + condition = ( + jsondecode(aws_lambda_function.scale_up.environment[0].variables["JOB_RETRY_CONFIG"]).queueUrl == "https://sqs.us-gov-west-1.amazonaws.com/123456789012/job-retry" + && jsondecode(aws_lambda_function.scale_up.environment[0].variables["JOB_RETRY_CONFIG"]).maxAttempts == "4" + && jsondecode(aws_lambda_function.scale_down.environment[0].variables["SCALE_DOWN_CONFIG"])[0].idleCount == 2 + ) + error_message = "Scale runners must preserve job-retry and idle-runner configuration at the Lambda boundary." + } + + assert { + condition = ( + aws_lambda_function.scale_up.memory_size == 768 + && aws_lambda_function.scale_up.timeout == 90 + && aws_lambda_function.scale_up.reserved_concurrent_executions == 2 + && aws_lambda_function.scale_down.memory_size == 640 + && aws_lambda_function.scale_down.timeout == 75 + && aws_cloudwatch_log_group.scale_up.log_group_class == "INFREQUENT_ACCESS" + && aws_cloudwatch_log_group.scale_down.retention_in_days == 14 + ) + error_message = "The child module must preserve Lambda sizing and log-group configuration." + } + + assert { + condition = ( + aws_lambda_event_source_mapping.scale_up.event_source_arn == "arn:aws-us-gov:sqs:us-gov-west-1:123456789012:build-queue" + && aws_lambda_event_source_mapping.scale_up.batch_size == 25 + && aws_lambda_event_source_mapping.scale_up.maximum_batching_window_in_seconds == 5 + && aws_lambda_event_source_mapping.scale_up.tags["Scope"] == "scale-up-queue" + && aws_cloudwatch_event_rule.scale_down.schedule_expression == "rate(10 minutes)" + && aws_cloudwatch_event_rule.scale_down.tags["Scope"] == "scale-down" + ) + error_message = "Scale-up queue and scale-down schedule triggers must remain owned by the child module." + } + + assert { + condition = ( + aws_lambda_function.scale_up.tags["Scope"] == "scale-up-lambda" + && aws_cloudwatch_log_group.scale_up.tags["Scope"] == "scale-up-log" + && aws_iam_role.scale_up.tags["Scope"] == "scale-up" + && aws_lambda_function.scale_down.tags["Scope"] == "scale-down-lambda" + && aws_cloudwatch_log_group.scale_down.tags["Scope"] == "scale-down-log" + && aws_iam_role.scale_down.tags["Scope"] == "scale-down" + ) + error_message = "Resolved component tag maps must reach the resources owned by scale runners." + } + + assert { + condition = ( + length(aws_lambda_function.scale_up.vpc_config) == 1 + && length(aws_lambda_function.scale_down.vpc_config) == 1 + && length(aws_iam_role_policy_attachment.scale_up_vpc_execution_role) == 1 + && length(aws_iam_role_policy_attachment.scale_down_vpc_execution_role) == 1 + && aws_iam_role_policy_attachment.scale_up_vpc_execution_role[0].policy_arn == "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + ) + error_message = "A complete Lambda VPC configuration must configure both Lambdas and their partition-aware execution policies." + } + + assert { + condition = ( + length(aws_lambda_function.scale_up.tracing_config) == 1 + && length(aws_lambda_function.scale_down.tracing_config) == 1 + && length(aws_iam_role_policy.scale_up_xray) == 1 + && length(aws_iam_role_policy.scale_down_xray) == 1 + ) + error_message = "Active tracing must configure both Lambdas and attach their X-Ray policies." + } + + assert { + condition = ( + length(aws_iam_role_policy.service_linked_role) == 1 + && length(aws_iam_role_policy_attachment.provider) == 1 + && aws_iam_role_policy_attachment.provider[0].policy_arn == "arn:aws-us-gov:iam::123456789012:policy/microvm-scale-up" + && length(aws_iam_role_policy.job_retry_sqs_publish) == 1 + ) + error_message = "Optional compute-provider and job-retry IAM integrations must be attached to the scale-up role." + } + + assert { + condition = ( + length(data.aws_iam_policy_document.scale_up.source_policy_documents) == 2 + && length(data.aws_iam_policy_document.scale_down.source_policy_documents) == 2 + && length(data.aws_iam_policy_document.scale_up_common.statement) == 5 + && length(data.aws_iam_policy_document.scale_down_common.statement) == 2 + && one([ + for statement in data.aws_iam_policy_document.scale_up_common.statement : statement + if statement.sid == "WebhookScaleUpDecryptParameterStore" + ]).resources == toset(["arn:aws-us-gov:kms:us-gov-west-1:123456789012:key/scale-runners-test"]) + && one([ + for statement in data.aws_iam_policy_document.scale_up_common.statement : statement + if statement.sid == "WebhookScaleUpDecryptBuildQueue" + ]).resources == toset(["arn:aws-us-gov:kms:us-gov-west-1:123456789012:key/build-queue-test"]) + && one([ + for statement in data.aws_iam_policy_document.scale_up_common.statement : statement + if statement.sid == "WebhookScaleUpDecryptBuildQueue" + ]).actions == toset(["kms:Decrypt"]) + && one([ + for statement in data.aws_iam_policy_document.scale_down_common.statement : statement + if statement.sid == "WebhookScaleDownDecryptParameterStore" + ]).resources == toset(["arn:aws-us-gov:kms:us-gov-west-1:123456789012:key/scale-runners-test"]) + && length(data.aws_iam_policy_document.scale_up_job_retry_publish) == 1 + ) + error_message = "Common, provider, distinct Parameter Store/build-queue KMS, and retry IAM fragments must retain their conditional plan shape." + } + + assert { + condition = ( + one([ + for statement in data.aws_iam_policy_document.scale_up_common.statement : statement + if statement.sid == "WebhookScaleUpWriteRuntimeParameters" + ]).resources == toset([ + "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/tokens", + "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/tokens/*", + "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/config", + "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/config/*", + ]) + && !contains(one([ + for statement in data.aws_iam_policy_document.scale_up_common.statement : statement + if statement.sid == "WebhookScaleUpWriteRuntimeParameters" + ]).resources, "*") + ) + error_message = "Scale-up must scope runtime SSM writes to the token and runner-config parameter paths." + } + + assert { + condition = ( + data.aws_iam_policy_document.lambda_xray[0].statement[0].sid == "AllowXRay" + && data.aws_iam_policy_document.lambda_xray[0].statement[0].resources == toset(["*"]) + && toset(data.aws_iam_policy_document.lambda_xray[0].statement[0].actions) == toset([ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments", + ]) + ) + error_message = "Only the resource-agnostic X-Ray APIs may retain a wildcard resource in the scale-runner common policies." + } +} + +run "omits_optional_kms_statements" { + command = plan + + variables { + config = merge(var.config, { + queue = merge(var.config.queue, { + kms_key_id = null + }) + ssm = merge(var.config.ssm, { + kms_key_id = null + }) + }) + } + + assert { + condition = ( + length(data.aws_iam_policy_document.scale_up_common.statement) == 3 + && length(data.aws_iam_policy_document.scale_down_common.statement) == 1 + && length([ + for statement in data.aws_iam_policy_document.scale_up_common.statement : statement + if anytrue([for action in statement.actions : startswith(action, "kms:")]) + ]) == 0 + && length([ + for statement in data.aws_iam_policy_document.scale_down_common.statement : statement + if anytrue([for action in statement.actions : startswith(action, "kms:")]) + ]) == 0 + ) + error_message = "Null Parameter Store and build-queue keys must omit every optional scale-runner KMS statement." + } +} + +run "requires_job_retry_queue_when_enabled" { + command = plan + + plan_options { + target = [terraform_data.validate_config] + } + + variables { + config = merge(var.config, { + job_retry = merge(var.config.job_retry, { + enabled = true + queue = null + }) + }) + } + + expect_failures = [terraform_data.validate_config] +} diff --git a/modules/orchestration-providers/webhook/scale-runners/validations.tf b/modules/orchestration-providers/webhook/scale-runners/validations.tf new file mode 100644 index 0000000000..50cdc1a136 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/validations.tf @@ -0,0 +1,8 @@ +resource "terraform_data" "validate_config" { + lifecycle { + precondition { + condition = !var.config.job_retry.enabled || var.config.job_retry.queue != null + error_message = "config.job_retry.queue must be set when config.job_retry.enabled is true." + } + } +} diff --git a/modules/orchestration-providers/webhook/scale-runners/variables.tf b/modules/orchestration-providers/webhook/scale-runners/variables.tf new file mode 100644 index 0000000000..e191e303d1 --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/variables.tf @@ -0,0 +1,233 @@ +variable "aws_partition" { + description = "AWS partition used to construct IAM policy ARNs." + type = string + default = "aws" +} + +variable "config" { + description = <<-EOT + Provider-neutral scale-up and scale-down configuration assembled by runner-config. + + - `prefix`: Prefix used to name scaling resources. + - `lambda.artifact.zip`: Resolved local control-plane archive. + - `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive. + - `lambda.artifact.s3.key`: Object key of the Lambda archive. + - `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive. + - `lambda.runtime`: Runtime used by both scaling Lambdas. + - `lambda.architecture`: Instruction-set architecture used by both scaling Lambdas. + - `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration. + - `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration. + - `lambda.role.path`: IAM path used for the scaling Lambda roles. + - `lambda.role.permissions_boundary`: Optional permissions boundary for the scaling Lambda roles. + - `lambda.role.principals`: Additional principals allowed to assume the scaling Lambda roles. + - `runner.os`: Runner operating system used for the minimum-runtime default. + - `runner.auto_update_disabled`: Disables the GitHub runner application's built-in updater. + - `runner.ephemeral`: Registers runners in ephemeral mode. + - `runner.jit_config_enabled`: Enables or disables just-in-time runner configuration. + - `runner.labels`: Labels supplied when a runner is registered. + - `runner.group_name`: GitHub runner group used during registration. + - `runner.name_prefix`: Prefix added to registered runner names. + - `runner.boot_time_in_minutes`: Webhook-provider runner boot timeout used by scale-down. + - `runner.maximum_count`: Webhook-provider runner capacity limit for this runner configuration. + - `github.organization_runners`: Registers organization runners when true. + - `github.enterprise_server.url`: Optional GitHub Enterprise Server URL. + - `github.enterprise_server.ssl_verify`: Enables TLS verification for GitHub Enterprise Server. + - `github.user_agent`: Optional User-Agent sent to GitHub. + - `github.app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys. + - `github.app_parameters.id`: Ordered Parameter Store references for GitHub App IDs. + - `github.app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs. + - `queue.build.arn`: ARN of the build queue consumed by scale-up. + - `queue.kms_key_id`: Optional KMS key ARN used to encrypt the build queue. This is distinct from the Parameter Store key. + - `queue.event_source_mapping.batch_size`: Maximum records delivered per scale-up invocation. + - `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum event batching window. + - `ssm.token_path`: Parameter Store path used for registration tokens. + - `ssm.token_path_arn`: ARN of the Parameter Store path used for registration tokens. + - `ssm.config_path`: Parameter Store path used for persistent runner configuration. + - `ssm.config_path_arn`: ARN of the persistent runner configuration path. + - `ssm.kms_key_id`: Optional KMS key ARN used to decrypt shared parameters. Its value may be unknown until apply. + - `ssm.parameter_store_tags`: JSON-encoded tags applied to parameters created at runtime. + - `observability.logs`: Shared logging level, retention, encryption, and log-class configuration. + - `observability.tracing`: Lambda X-Ray and tracing-helper configuration. + - `observability.metrics`: Metrics enablement, namespace, and GitHub rate-limit metric configuration. + - `scale_up`: Scale-up Lambda sizing, concurrency, queued-job behavior, and resolved resource tag maps. + - `scale_up.tags.resources`: Tags for the scale-up IAM role and other component resources. + - `scale_up.tags.lambda`: Tags for the scale-up Lambda function. + - `scale_up.tags.log_group`: Tags for the scale-up log group. + - `scale_up.tags.event_source_mapping`: Tags for the build-queue event-source mapping. + - `scale_down`: Scale-down Lambda sizing, schedule, idle configuration, minimum runtime, and resolved resource tag maps. + - `scale_down.tags.resources`: Tags for the scale-down IAM role and EventBridge rule. + - `scale_down.tags.lambda`: Tags for the scale-down Lambda function. + - `scale_down.tags.log_group`: Tags for the scale-down log group. + - `job_retry.enabled`: Enables publishing retry checks from scale-up. + - `job_retry.queue`: Retry queue ARN and URL. Required when job retry is enabled. + - `job_retry.max_attempts`: Maximum queued-job retry attempts. + - `job_retry.delay_in_seconds`: Initial delay before checking the queued job. + - `job_retry.delay_backoff`: Multiplier applied to subsequent delays. + EOT + + type = object({ + prefix = string + lambda = object({ + artifact = object({ + zip = string + s3 = object({ + bucket = optional(string, null) + key = optional(string, null) + object_version = optional(string, null) + }) + }) + runtime = string + architecture = string + vpc = object({ + subnet_ids = list(string) + security_group_ids = list(string) + }) + role = object({ + path = string + permissions_boundary = optional(string, null) + principals = optional(list(object({ + type = string + identifiers = list(string) + })), []) + }) + }) + runner = object({ + os = string + auto_update_disabled = bool + ephemeral = bool + jit_config_enabled = optional(bool, null) + labels = list(string) + group_name = string + name_prefix = string + boot_time_in_minutes = number + maximum_count = number + }) + github = object({ + organization_runners = bool + enterprise_server = object({ + url = optional(string, null) + ssl_verify = bool + }) + user_agent = optional(string, null) + app_parameters = object({ + key_base64 = list(map(string)) + id = list(map(string)) + installation_id = list(object({ name = string, arn = string })) + }) + }) + queue = object({ + build = object({ + arn = string + }) + kms_key_id = optional(string, null) + event_source_mapping = object({ + batch_size = number + maximum_batching_window_in_seconds = number + }) + }) + ssm = object({ + token_path = string + token_path_arn = string + config_path = string + config_path_arn = string + parameter_store_tags = string + kms_key_id = optional(string, null) + }) + observability = object({ + logs = object({ + level = string + retention_in_days = number + kms_key_id = optional(string, null) + class = string + }) + tracing = object({ + mode = optional(string, null) + capture_http_requests = bool + capture_error = bool + }) + metrics = object({ + enabled = bool + namespace = string + metric = object({ + github_app_rate_limit = object({ + enabled = bool + }) + }) + }) + }) + scale_up = object({ + memory_size = number + timeout = number + reserved_concurrent_executions = number + job_queued_check_enabled = bool + tags = object({ + resources = map(string) + lambda = map(string) + log_group = map(string) + event_source_mapping = map(string) + }) + }) + scale_down = object({ + memory_size = number + timeout = number + schedule_expression = string + minimum_running_time_in_minutes = optional(number, null) + idle_config = list(object({ + cron = string + timeZone = string + idleCount = number + evictionStrategy = string + })) + tags = object({ + resources = map(string) + lambda = map(string) + log_group = map(string) + }) + }) + job_retry = object({ + enabled = bool + max_attempts = number + delay_in_seconds = number + delay_backoff = number + queue = optional(object({ + arn = string + url = string + }), null) + }) + }) + + nullable = false +} + +variable "runner_provider" { + description = <<-EOT + Selected compute-provider integration for the scaling control plane. + + - `type`: Compute-provider discriminator supplied to both Lambdas. + - `scale_up.environment_variables`: Provider-specific scale-up environment variables. + - `scale_up.iam_policy_json`: Provider-specific IAM policy merged into the common scale-up policy. + - `scale_up.additional_iam_policy_json`: Optional additional provider policy attached separately to the scale-up role. + - `scale_up.managed_policy`: Optional provider-managed policy attachment. Object presence controls attachment creation. + - `scale_up.managed_policy.arn`: ARN of the provider-managed policy. The ARN may remain unknown until apply. + - `scale_down.environment_variables`: Provider-specific scale-down environment variables. + - `scale_down.iam_policy_json`: Provider-specific IAM policy merged into the common scale-down policy. + EOT + + type = object({ + type = string + scale_up = object({ + environment_variables = map(string) + iam_policy_json = string + additional_iam_policy_json = optional(string, null) + managed_policy = optional(object({ + arn = string + }), null) + }) + scale_down = object({ + environment_variables = map(string) + iam_policy_json = string + }) + }) + + nullable = false +} diff --git a/modules/orchestration-providers/webhook/scale-runners/versions.tf b/modules/orchestration-providers/webhook/scale-runners/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/orchestration-providers/webhook/scale-runners/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/orchestration-providers/webhook/tests/webhook.tftest.hcl b/modules/orchestration-providers/webhook/tests/webhook.tftest.hcl new file mode 100644 index 0000000000..ac92fb907b --- /dev/null +++ b/modules/orchestration-providers/webhook/tests/webhook.tftest.hcl @@ -0,0 +1,315 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/webhook-orchestration-test" + } + } +} + +variables { + prefix = "webhook-test" + + tags = { + Scope = "common" + Precedence = "common" + } + + runner = { + os = "linux" + auto_update_disabled = false + labels = ["self-hosted", "linux"] + group_name = "default" + name_prefix = "webhook-test-" + } + + github = { + app_parameters = { + key_base64 = [{ + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + }] + id = [{ + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + }] + installation_id = [null] + } + enterprise_server = { + url = null + ssl_verify = true + } + user_agent = "webhook-orchestration-test" + } + + lambda = { + artifact = { + s3 = { + bucket = "lambda-artifacts" + } + } + runtime = "nodejs24.x" + architecture = "arm64" + subnet_ids = [] + security_group_ids = [] + tags = { + Lambda = "yes" + Precedence = "lambda" + } + role = { + path = "/webhook-test/" + } + } + + ssm = { + token_path = "/github-runner/tokens" + token_path_arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/tokens" + config_path = "/github-runner/config" + config_path_arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/config" + kms_key_id = "arn:aws:kms:eu-west-1:123456789012:key/webhook-test" + parameter_store_tags = "[]" + } + + observability = { + logs = { + level = "info" + retention_in_days = 14 + kms_key_id = null + class = "STANDARD" + } + tracing = { + mode = null + capture_http_requests = false + capture_error = false + } + metrics = { + enabled = true + namespace = "WebhookTest" + metric = { + github_app_rate_limit = { + enabled = true + } + job_retry = { + enabled = true + } + } + } + } + + config = { + runner = { + boot_time_in_minutes = 11 + ephemeral = true + jit_config_enabled = null + maximum_count = 10 + } + github = { + organization_runners = true + } + queue = { + build = { + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + } + kms_key_id = "arn:aws:kms:eu-west-1:123456789012:key/build-queue-test" + tags = { + Queue = "yes" + } + } + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + scale = { + up = { + memory_size = 512 + timeout = 60 + reserved_concurrent_executions = 1 + job_queued_check_enabled = null + event_source_mapping = { + batch_size = 10 + maximum_batching_window_in_seconds = 0 + } + tags = { + ScaleUp = "yes" + Precedence = "scale-up" + } + } + down = { + memory_size = 512 + timeout = 60 + schedule_expression = "cron(*/5 * * * ? *)" + minimum_running_time_in_minutes = null + idle_config = [] + tags = { + ScaleDown = "yes" + } + } + } + pool = { + memory_size = 512 + timeout = 60 + reserved_concurrent_executions = 1 + config = [{ + schedule_expression = "cron(0 8 * * ? *)" + schedule_expression_timezone = "UTC" + size = 1 + }] + include_busy_runners = false + runner_owner = "example" + tags = { + Pool = "yes" + } + } + } + job_retry = { + enabled = true + delay_in_seconds = 300 + delay_backoff = 2 + max_attempts = 2 + tags = { + JobRetry = "yes" + } + lambda = { + memory_size = 256 + reserved_concurrent_executions = 1 + timeout = 30 + } + } + } + + runner_provider = { + type = "test-provider" + scale_up = { + environment_variables = { + TEST_SCALE_UP = "yes" + } + iam_policy_json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + additional_iam_policy_json = null + managed_policy = null + } + scale_down = { + environment_variables = { + TEST_SCALE_DOWN = "yes" + } + iam_policy_json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + pool = { + environment_variables = { + TEST_POOL = "yes" + } + iam_policy_json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + managed_policy_enabled = false + managed_policy_arn = null + } + } +} + +run "owns_webhook_control_plane" { + command = plan + + assert { + condition = ( + toset(keys(output.scale_up)) == toset(["lambda", "log_group", "role"]) + && toset(keys(output.scale_down)) == toset(["lambda", "log_group", "role"]) + ) + error_message = "The webhook provider must own and expose both scaling functions." + } + + assert { + condition = ( + output.pool != null + && output.job_retry != null + && output.job_retry.lambda != null + && output.job_retry.queue != null + ) + error_message = "The webhook provider must own the optional pool and job-retry resources when enabled." + } + + assert { + condition = ( + output.scale_up.lambda.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "test-provider" + && output.scale_up.lambda.environment[0].variables["TEST_SCALE_UP"] == "yes" + && output.scale_down.lambda.environment[0].variables["TEST_SCALE_DOWN"] == "yes" + && output.pool.lambda.environment[0].variables["TEST_POOL"] == "yes" + ) + error_message = "The webhook provider must forward each compute-provider capability to the matching leaf." + } + + assert { + condition = ( + output.scale_up.lambda.environment[0].variables["RUNNERS_MAXIMUM_COUNT"] == "10" + && output.pool.lambda.environment[0].variables["RUNNERS_MAXIMUM_COUNT"] == "10" + && output.scale_down.lambda.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "11" + && output.pool.lambda.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "11" + && output.scale_up.lambda.environment[0].variables["ENABLE_JIT_CONFIG"] == "true" + && output.pool.lambda.environment[0].variables["ENABLE_JIT_CONFIG"] == "true" + ) + error_message = "The webhook provider must route its provider-owned runner lifecycle, capacity, and boot-time values without reading them from common runner values." + } + + assert { + condition = ( + output.runner_lifecycle.ephemeral + && output.runner_lifecycle.jit_config_enabled + ) + error_message = "The webhook provider must expose its resolved lifecycle contract and default JIT configuration to the effective ephemeral mode." + } + + assert { + condition = ( + output.scale_up.lambda.s3_bucket == "lambda-artifacts" + && output.scale_up.lambda.s3_key == "runners.zip" + && output.scale_down.lambda.s3_bucket == "lambda-artifacts" + && output.pool.lambda.s3_key == "runners.zip" + ) + error_message = "The webhook provider must combine the common artifact bucket with its shared runner-control artifact key for scale, pool, and job retry." + } + + assert { + condition = ( + output.scale_up.lambda.tags["Scope"] == "common" + && output.scale_up.lambda.tags["Lambda"] == "yes" + && output.scale_up.lambda.tags["ScaleUp"] == "yes" + && output.scale_up.lambda.tags["Precedence"] == "scale-up" + && output.job_retry.queue.tags["JobRetry"] == "yes" + && output.job_retry.queue.tags["Queue"] == "yes" + ) + error_message = "Provider-owned normalization must preserve common, substrate, and webhook component tag precedence." + } + + assert { + condition = ( + length(module.pool) == 1 + && length(module.job_retry) == 1 + ) + error_message = "Pool and job-retry leaf ownership must remain inside the webhook provider." + } +} + +run "rejects_conflicting_artifact_sources" { + command = plan + + plan_options { + target = [terraform_data.validate_config] + } + + variables { + config = merge(var.config, { + lambda = merge(var.config.lambda, { + artifact = merge(var.config.lambda.artifact, { + zip = "runners.zip" + }) + }) + }) + } + + expect_failures = [terraform_data.validate_config] +} diff --git a/modules/orchestration-providers/webhook/validations.tf b/modules/orchestration-providers/webhook/validations.tf new file mode 100644 index 0000000000..0f4d5213f9 --- /dev/null +++ b/modules/orchestration-providers/webhook/validations.tf @@ -0,0 +1,11 @@ +resource "terraform_data" "validate_config" { + lifecycle { + precondition { + condition = !( + var.config.lambda.artifact.zip != null && + var.config.lambda.artifact.s3 != null + ) + error_message = "config.lambda.artifact must select at most one of zip or s3." + } + } +} diff --git a/modules/orchestration-providers/webhook/variables.tf b/modules/orchestration-providers/webhook/variables.tf new file mode 100644 index 0000000000..5dfecdbd6c --- /dev/null +++ b/modules/orchestration-providers/webhook/variables.tf @@ -0,0 +1,272 @@ +variable "aws_partition" { + description = "AWS partition used to construct ARNs." + type = string + default = "aws" +} + +variable "prefix" { + description = "Prefix used to identify resources created for this webhook orchestration provider." + type = string +} + +variable "tags" { + description = "Base tags available to webhook-provider resources. Component-specific tags override this map within their documented scopes." + type = map(string) + default = {} +} + +variable "config" { + description = <<-EOT + Provider-owned webhook values supplied from `orchestration_provider.webhook`. The parent resolves inherited input values before calling this module; this provider still resolves the documented JIT, artifact, and tag-precedence fallbacks. + + - `runner`: Runner lifecycle, boot timeout, and capacity settings owned by webhook orchestration. + - `runner.boot_time_in_minutes`: Expected runner boot duration used by scale-down and pool controls. + - `runner.ephemeral`: Registers runners in ephemeral mode. + - `runner.jit_config_enabled`: Explicitly enables or disables just-in-time configuration. Null follows `runner.ephemeral`. + - `runner.maximum_count`: Maximum number of runners managed for this runner configuration. + - `github.organization_runners`: Registers runners at organization scope when true; otherwise registration is repository-scoped. + - `queue.build.arn`: ARN of the runner configuration's build queue. + - `queue.build.url`: URL of the runner configuration's build queue. + - `queue.kms_key_id`: Optional KMS key ARN encrypting the build queue. This is independent from the Parameter Store KMS key. + - `queue.tags`: Tags inherited by queue-related provider resources before component-specific overrides. + - `lambda.artifact`: Runner-control artifact shared by scale, pool, and job-retry components. At most one of `zip` or `s3` may be selected; no selection uses the packaged runner archive. + - `lambda.artifact.zip`: Optional local path to the runner-control Lambda archive. + - `lambda.artifact.s3`: Optional S3 object selector in the common `lambda.artifact.s3.bucket`. Wrapper presence must be known during planning and selecting it requires a non-null common bucket. + - `lambda.artifact.s3.key`: Object key of the runner-control Lambda archive. + - `lambda.artifact.s3.object_version`: Optional object version of the runner-control Lambda archive. + - `lambda.scale.up.memory_size`: Memory allocated to the scale-up Lambda in MB. + - `lambda.scale.up.timeout`: Scale-up Lambda timeout in seconds. + - `lambda.scale.up.reserved_concurrent_executions`: Reserved concurrency for scale-up. Use `-1` for unreserved concurrency. + - `lambda.scale.up.job_queued_check_enabled`: Enables queued-job verification before scaling. Null follows the resolved runner mode. + - `lambda.scale.up.event_source_mapping.batch_size`: Maximum build-queue records delivered per scale-up invocation. + - `lambda.scale.up.event_source_mapping.maximum_batching_window_in_seconds`: Maximum batching window for build-queue records. + - `lambda.scale.up.tags`: Tags applied within scale-up resource scopes after common provider tags. + - `lambda.scale.down.memory_size`: Memory allocated to the scale-down Lambda in MB. + - `lambda.scale.down.timeout`: Scale-down Lambda timeout in seconds. + - `lambda.scale.down.schedule_expression`: EventBridge schedule expression that invokes scale-down. + - `lambda.scale.down.minimum_running_time_in_minutes`: Optional minimum runner age before scale-down may terminate it. Null selects the operating-system default. + - `lambda.scale.down.idle_config`: Time-based desired idle-runner configurations. + - `lambda.scale.down.idle_config[].cron`: Cron expression identifying when the idle configuration applies. + - `lambda.scale.down.idle_config[].timeZone`: IANA time zone used to evaluate the cron expression. + - `lambda.scale.down.idle_config[].idleCount`: Number of idle runners retained during the matching period. + - `lambda.scale.down.idle_config[].evictionStrategy`: Selection strategy used when excess idle runners are removed. + - `lambda.scale.down.tags`: Tags applied within scale-down resource scopes after common provider tags. + - `lambda.pool.memory_size`: Memory allocated to the pool Lambda in MB. + - `lambda.pool.timeout`: Pool Lambda timeout in seconds. + - `lambda.pool.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. Use `-1` for unreserved concurrency. + - `lambda.pool.config`: Scheduled target pool sizes. An empty list disables the pool component. + - `lambda.pool.config[].schedule_expression`: Scheduler expression that activates the target size. + - `lambda.pool.config[].schedule_expression_timezone`: Optional IANA time zone used to evaluate the schedule. + - `lambda.pool.config[].size`: Desired number of runners for the schedule. + - `lambda.pool.include_busy_runners`: Includes busy runners when reconciling scheduled pool capacity. + - `lambda.pool.runner_owner`: Optional GitHub organization or repository owner used for pooled runners. + - `lambda.pool.tags`: Tags applied within pool resource scopes after common provider tags. + - `job_retry.enabled`: Creates the retry queue, Lambda function, event-source mapping, and related IAM resources. + - `job_retry.delay_in_seconds`: Initial delay before a queued-job retry check. + - `job_retry.delay_backoff`: Multiplier applied to the delay after each unsuccessful check. + - `job_retry.max_attempts`: Maximum retry-check attempts before the message is no longer republished. + - `job_retry.tags`: Tags applied within job-retry resource scopes after common provider tags. + - `job_retry.lambda.memory_size`: Memory allocated to the job-retry Lambda in MB. + - `job_retry.lambda.reserved_concurrent_executions`: Reserved concurrency for job retry. Use `-1` for unreserved concurrency. + - `job_retry.lambda.timeout`: Job-retry Lambda timeout in seconds and visibility timeout for its retry queue. + EOT + type = object({ + runner = object({ + boot_time_in_minutes = number + ephemeral = bool + jit_config_enabled = optional(bool, null) + maximum_count = number + }) + github = object({ + organization_runners = bool + }) + queue = object({ + build = object({ + arn = string + url = string + }) + kms_key_id = optional(string, null) + tags = optional(map(string), {}) + }) + lambda = object({ + artifact = object({ + zip = optional(string, null) + s3 = optional(object({ + key = string + object_version = optional(string, null) + }), null) + }) + scale = object({ + up = object({ + memory_size = number + timeout = number + reserved_concurrent_executions = number + job_queued_check_enabled = optional(bool, null) + event_source_mapping = object({ + batch_size = number + maximum_batching_window_in_seconds = number + }) + tags = optional(map(string), {}) + }) + down = object({ + memory_size = number + timeout = number + schedule_expression = string + minimum_running_time_in_minutes = optional(number, null) + idle_config = list(object({ + cron = string + timeZone = string + idleCount = number + evictionStrategy = string + })) + tags = optional(map(string), {}) + }) + }) + pool = object({ + memory_size = number + timeout = number + reserved_concurrent_executions = number + config = list(object({ + schedule_expression = string + schedule_expression_timezone = optional(string) + size = number + })) + include_busy_runners = bool + runner_owner = optional(string, null) + tags = optional(map(string), {}) + }) + }) + job_retry = object({ + enabled = bool + delay_in_seconds = number + delay_backoff = number + max_attempts = number + tags = optional(map(string), {}) + lambda = object({ + memory_size = number + reserved_concurrent_executions = number + timeout = number + }) + }) + }) + nullable = false +} + +variable "runner" { + description = "Common runner registration values consumed by webhook demand controls. Lifecycle, boot timeout, and capacity remain provider-owned under config.runner." + type = object({ + os = string + auto_update_disabled = bool + labels = list(string) + group_name = string + name_prefix = string + }) +} + +variable "github" { + description = "Common GitHub API client and GitHub App Parameter Store references." + type = object({ + app_parameters = object({ + key_base64 = list(map(string)) + id = list(map(string)) + installation_id = list(object({ name = string, arn = string })) + }) + enterprise_server = object({ + url = optional(string, null) + ssl_verify = bool + }) + user_agent = optional(string, null) + }) +} + +variable "lambda" { + description = "Common Lambda substrate. Only the shared artifact bucket crosses this boundary; the webhook provider owns its archive key, version, and local zip selection." + type = object({ + artifact = object({ + s3 = object({ + bucket = optional(string, null) + }) + }) + runtime = string + architecture = string + subnet_ids = list(string) + security_group_ids = list(string) + tags = optional(map(string), {}) + role = object({ + path = string + permissions_boundary = optional(string, null) + principals = optional(list(object({ + type = string + identifiers = list(string) + })), []) + }) + }) +} + +variable "ssm" { + description = "Resolved Parameter Store paths, optional decrypt key, and runtime parameter tags." + type = object({ + token_path = string + token_path_arn = string + config_path = string + config_path_arn = string + kms_key_id = optional(string, null) + parameter_store_tags = string + }) +} + +variable "observability" { + description = "Common logging, tracing, and metrics configuration consumed by webhook controls." + type = object({ + logs = object({ + level = string + retention_in_days = number + kms_key_id = optional(string, null) + class = string + tags = optional(map(string), {}) + }) + tracing = object({ + mode = optional(string, null) + capture_http_requests = bool + capture_error = bool + }) + metrics = object({ + enabled = bool + namespace = string + metric = object({ + github_app_rate_limit = object({ + enabled = bool + }) + job_retry = object({ + enabled = bool + }) + }) + }) + }) +} + +variable "runner_provider" { + description = "Selected compute-provider capabilities consumed by webhook scale-up, scale-down, and pool controls." + type = object({ + type = string + scale_up = object({ + environment_variables = map(string) + iam_policy_json = string + additional_iam_policy_json = optional(string, null) + managed_policy = optional(object({ + arn = string + }), null) + }) + scale_down = object({ + environment_variables = map(string) + iam_policy_json = string + }) + pool = object({ + environment_variables = map(string) + iam_policy_json = string + managed_policy_enabled = bool + managed_policy_arn = optional(string, null) + }) + }) + nullable = false +} diff --git a/modules/orchestration-providers/webhook/versions.tf b/modules/orchestration-providers/webhook/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/orchestration-providers/webhook/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/runner-config/README.md b/modules/runner-config/README.md new file mode 100644 index 0000000000..c918a79fa0 --- /dev/null +++ b/modules/runner-config/README.md @@ -0,0 +1,133 @@ +# Runner configuration module + +> This module is treated as an internal module; breaking changes do not trigger a major release bump. + +This internal module implements the experimental provider-neutral runner configuration selected by `experimental.multi_runner_config`. It is composed by `multi-runner` and is not intended as a standalone public entry point. Its direct contract may change while v2 remains experimental. + +The module selects the [`webhook` orchestration provider](../orchestration-providers/webhook), which owns its [`scale-runners`](../orchestration-providers/webhook/scale-runners), [`pool`](../orchestration-providers/webhook/pool), and [`job-retry`](../orchestration-providers/webhook/job-retry) leaves. The configuration module retains the common [`ssm-housekeeper`](./ssm-housekeeper), creates or selects the runner IAM role, manages shared runner configuration in SSM, and dispatches the selected compute provider. + +Runner demand orchestration is selected independently through `orchestration_provider`. `orchestration_provider.webhook` is the currently supported provider and owns the build queue reference; runner lifecycle, boot time, and capacity under `orchestration_provider.webhook.runner`; runner registration scope; scaling controls; scheduled pool; and job retry. Common `runner` contains no webhook lifecycle or capacity settings. The provider resolves its lifecycle contract before runner-config serializes the existing bootstrap parameters. The provider wrapper is nullable so a future sibling provider can be added without moving this webhook contract again, while validation requires exactly one provider to be selected. + +Common `lambda` contains only shared execution substrate and the optional shared artifact bucket. The webhook provider owns the runner-control archive shared by scale, pool, and job-retry at `orchestration_provider.webhook.lambda.artifact` and combines its zip or S3 key/version with that common substrate. The common SSM housekeeper independently owns `ssm.housekeeper.lambda.artifact`: an S3 selection combines its component key/version with the common bucket, a local zip is used otherwise when configured, and the packaged runner control-plane archive is the final fallback. It never inherits the webhook runner-control archive. + +Provider-owned settings remain nested under a typed namespace and provider leaf. For example, AMI, VPC, instance-profile, capacity, userdata, and runner-host logging settings live under `compute_provider.aws.ec2`. `multi-runner` resolves experimental globals and runner-configuration overrides first, then its final forwarding adapter preserves the wrapped `{ aws = { ec2 = ... } }` object expected by this module. Exactly one provider leaf must be non-null. The configuration module flattens the selected namespace and type to the Terraform dispatch key `aws_ec2`, while the webhook runtime registry continues to receive the provider type `ec2`. + +The EC2 leaf reaches runner-config with `compute_provider.aws.ec2.binaries_syncer = { enabled, s3 }`; the S3 object is null when synchronization is disabled. Binary discovery and this shape adaptation happen in `multi-runner`, not inside runner-config. Before creating the common runner role, the configuration module calls [`compute-providers/aws/ec2/trust-policy`](../compute-providers/aws/ec2/trust-policy) as `module.compute_aws_ec2_trust_policy[0]` to combine its default trust with `runner.iam.additional_trust_policy_json`. The resulting assume-role policy does not depend on the full [`compute-providers/aws/ec2`](../compute-providers/aws/ec2) module, dispatched at `module.compute_aws_ec2[0]`, which receives the resolved runner role only after it is created. Declarative moved blocks preserve state from the earlier experimental `module.compute_ec2_trust_policy[0]` and `module.compute_ec2[0]` labels. EC2 owns the instance profile, launch template, EC2 bootstrap parameters, runner log groups, and its provider policies and Lambda environment variables. The common configuration module attaches each returned policy group to its runner or webhook-provider role. Provider-specific outputs remain grouped under the matching namespace and provider path, currently `provider.aws.ec2`. Moved blocks do not rewrite output references, so consumers of the former experimental `provider.ec2` path must update their expressions. EC2 is the only implemented Terraform compute provider in this phase. + +## Tagging + +`tags` supplies module-wide defaults. Shared resource tags are set with `lambda.tags`, `orchestration_provider.webhook.queue.tags`, and `observability.logs.tags`. Component tags under `runner`, `orchestration_provider.webhook.lambda.scale.up`, `orchestration_provider.webhook.lambda.scale.down`, `orchestration_provider.webhook.lambda.pool`, `orchestration_provider.webhook.job_retry`, and `ssm` apply to the taggable resources owned by that component. `ssm.parameters.tags` and `ssm.housekeeper.tags` provide narrower SSM scopes. + +Tags are merged from broadest to narrowest: module tags, shared resource tags, component tags, and then subcomponent tags. The narrowest value wins when a key is repeated. For example, a scale-up Lambda receives `tags`, `lambda.tags`, and `orchestration_provider.webhook.lambda.scale.up.tags`, while its log group receives `tags`, `observability.logs.tags`, and `orchestration_provider.webhook.lambda.scale.up.tags`. + +Provider-specific runner tags remain inside the provider boundary. `compute_provider.aws.ec2.tags` applies to runtime EC2 instance, volume, network-interface, and spot-request tag specifications. `multi-runner` derives that map from global and runner-configuration `compute_provider.aws.ec2.tags` values. The EC2 provider applies the bootstrap tags `ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` last so they cannot be overridden; those tags are not added to common Lambda, IAM, queue, log-group, or SSM resources. + +## Overview + +### Action runners on EC2 + +The action runners are created via a launch template; in the launch template only the subnet needs to be provided. During launch the installation is handled via a user data script. The configuration is fetched from SSM parameter store. + +### Lambda scale up + +The scale up lambda is triggered by events on a SQS queue. Events on this queue are delayed, which will give the workflow some time to start running on available runners. For each event the lambda will check if the workflow is still queued and no other limits are reached. In that case the lambda will create a new EC2 instance. The lambda only needs to know which launch template to use and which subnets are available. From the available subnets a random one will be chosen. Once the instance is created the event is assumed as handled, and we assume the workflow wil start at some moment once the created instance is ready. + +### Lambda scale down + +The scale down lambda is triggered via a CloudWatch event. The event is triggered by a cron expression defined in `orchestration_provider.webhook.lambda.scale.down.schedule_expression` (https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html). For scaling down GitHub does not provide a good API yet, therefore we run the scaling down based on this event every x minutes. Each time the lambda is triggered it tries to remove all runners older than x minutes (configurable) managed in this deployment. In case the runner can be removed from GitHub, which means it is not executing a workflow, the lambda will terminate the EC2 instance. + +--8<-- "modules/orchestration-providers/webhook/scale-down-state-diagram.md:mkdocs_scale_down_state_diagram" + +## Lambda Function + +The Lambda function is written in [TypeScript](https://www.typescriptlang.org/) and requires Node 12.x and yarn. Sources are located in [./lambdas/runners]. Two lambda functions share the same sources, there is one entry point for `scaleDown` and another one for `scaleUp`. + +### Install + +```bash +cd lambdas/runners +yarn install +``` + +### Test + +Test are implemented with [vitest][https://vitest.dev/]), calls to AWS and GitHub are mocked. + +```bash +yarn run test +``` + +### Package + +To compile all TypeScript/JavaScript sources in a single file [ncc](https://github.com/zeit/ncc) is used. + +```bash +yarn run dist +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [compute\_aws\_ec2](#module\_compute\_aws\_ec2) | ../compute-providers/aws/ec2 | n/a | +| [compute\_aws\_ec2\_trust\_policy](#module\_compute\_aws\_ec2\_trust\_policy) | ../compute-providers/aws/ec2/trust-policy | n/a | +| [orchestration\_webhook](#module\_orchestration\_webhook) | ../orchestration-providers/webhook | n/a | +| [ssm\_housekeeper](#module\_ssm\_housekeeper) | ./ssm-housekeeper | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.runner_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_ssm_parameter.disable_default_labels](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.jit_config_enabled](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_agent_mode](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.token_path](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [terraform_data.validate_config](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | AWS partition used to construct ARNs. | `string` | `"aws"` | no | +| [aws\_region](#input\_aws\_region) | AWS region. | `string` | n/a | yes | +| [compute\_provider](#input\_compute\_provider) | Typed compute-provider configuration. Provider-owned settings remain inside the selected compute-provider block.

Exactly one compute-provider block must be non-null. The populated block selects the provider, and its presence must be known during planning. Values inside the selected block may remain unknown until apply.

- `aws`: AWS compute-provider configurations.
- `aws.ec2`: EC2 compute-provider configuration.
- `aws.ec2.ami`: Optional AMI discovery or external AMI-parameter configuration. Null uses the operating-system and architecture defaults.
- `aws.ec2.ami.filter`: EC2 AMI filters combined with the provider's default AMI-name filter.
- `aws.ec2.ami.owners`: AWS account IDs or aliases allowed to own the selected AMI.
- `aws.ec2.ami.id_ssm_parameter`: Optional externally managed SSM parameter containing the AMI ID. Null creates a provider-managed AMI-ID parameter. The wrapper's presence is the plan-time ownership discriminator, so keep the object literal even when its ARN comes from another resource.
- `aws.ec2.ami.id_ssm_parameter.arn`: ARN of the externally managed SSM parameter. The ARN may be unknown until apply.
- `aws.ec2.ami.kms_key`: Optional KMS key required to launch encrypted AMIs or snapshots. The wrapper's presence is the plan-time policy discriminator.
- `aws.ec2.ami.kms_key.arn`: ARN of the KMS key. The ARN may be unknown until apply.
- `aws.ec2.vpc_id`: VPC in which runner networking resources are created.
- `aws.ec2.subnet_ids`: Subnets from which scale-up may launch runner instances.
- `aws.ec2.overrides`: Optional resource-name overrides.
- `aws.ec2.overrides.name_runner`: Name tag used for runner compute resources. An empty value uses the generated provider name.
- `aws.ec2.overrides.name_sg`: Name tag used for the managed runner security group. An empty value uses the generated provider name.
- `aws.ec2.instance_profile`: Optional externally managed instance profile used by the launch template.
- `aws.ec2.instance_profile.name`: Name of the externally managed instance profile.
- `aws.ec2.instance_profile_path`: IAM path for the provider-managed instance profile. Null uses a path derived from the runner-configuration prefix.
- `aws.ec2.binaries_syncer`: Runner-distribution synchronization configuration.
- `aws.ec2.binaries_syncer.enabled`: Enables use of a synchronized runner distribution from S3.
- `aws.ec2.binaries_syncer.s3`: S3 object containing the synchronized runner distribution. Required when synchronization is enabled.
- `aws.ec2.binaries_syncer.s3.arn`: ARN of the runner-distribution bucket, used by IAM policies.
- `aws.ec2.binaries_syncer.s3.id`: Bucket name used to construct the runner-distribution S3 URI.
- `aws.ec2.binaries_syncer.s3.key`: Object key of the runner distribution.
- `aws.ec2.block_device_mappings`: EBS mappings added to the runner launch template.
- `aws.ec2.block_device_mappings[].delete_on_termination`: Deletes the volume when its runner instance terminates.
- `aws.ec2.block_device_mappings[].device_name`: Device name exposed to the runner instance.
- `aws.ec2.block_device_mappings[].encrypted`: Enables EBS encryption.
- `aws.ec2.block_device_mappings[].iops`: Provisioned IOPS for volume types that support it.
- `aws.ec2.block_device_mappings[].kms_key_id`: KMS key ID or ARN used to encrypt the volume.
- `aws.ec2.block_device_mappings[].snapshot_id`: Snapshot used to initialize the volume.
- `aws.ec2.block_device_mappings[].throughput`: Provisioned throughput for volume types that support it.
- `aws.ec2.block_device_mappings[].volume_initialization_rate`: Fixed initialization rate in MiB/s for supported snapshot-backed volumes.
- `aws.ec2.block_device_mappings[].volume_size`: Volume size in GiB.
- `aws.ec2.block_device_mappings[].volume_type`: EBS volume type.
- `aws.ec2.ebs_optimized`: Requests EBS-optimized runner instances.
- `aws.ec2.instance_target_capacity_type`: Primary capacity type, either `spot` or `on-demand`.
- `aws.ec2.instance_allocation_strategy`: EC2 Fleet allocation strategy used to select instance capacity.
- `aws.ec2.instance_type_priorities`: Optional numeric priorities keyed by instance type.
- `aws.ec2.instance_max_spot_price`: Optional maximum hourly Spot price.
- `aws.ec2.instance_types`: EC2 instance types available to the scale-up and pool functions.
- `aws.ec2.user_data`: Runner bootstrap user-data configuration.
- `aws.ec2.user_data.enabled`: Enables launch-template user data.
- `aws.ec2.user_data.template`: Optional path to a custom user-data template.
- `aws.ec2.user_data.content`: Optional complete user-data content. When set, it is used instead of rendering a template.
- `aws.ec2.user_data.pre_install`: Script content inserted before runner installation in the default template.
- `aws.ec2.user_data.post_install`: Script content inserted after runner installation in the default template.
- `aws.ec2.user_data.debug_logging_enabled`: Enables verbose user-data tracing, which can expose secrets in logs.
- `aws.ec2.ssm_enabled`: Attaches runner permissions and policies required for AWS Systems Manager access.
- `aws.ec2.create_service_linked_role_spot`: Allows scale-up to create the EC2 Spot service-linked role.
- `aws.ec2.cloudwatch_agent`: CloudWatch agent configuration for runner instances.
- `aws.ec2.cloudwatch_agent.enabled`: Installs and configures the CloudWatch agent through the default bootstrap flow.
- `aws.ec2.cloudwatch_agent.config`: Optional complete CloudWatch agent configuration. Null renders the provider default from `log_files`.
- `aws.ec2.managed_security_group_enabled`: Creates and attaches the provider-managed runner security group.
- `aws.ec2.log_files`: Optional log files collected by the CloudWatch agent. Null uses the provider defaults.
- `aws.ec2.log_files[].log_group_name`: CloudWatch log-group name, before optional prefixing.
- `aws.ec2.log_files[].prefix_log_group`: Prefixes the log-group name with the runner configuration path when true.
- `aws.ec2.log_files[].file_path`: File or glob read by the CloudWatch agent.
- `aws.ec2.log_files[].log_stream_name`: CloudWatch log-stream name template.
- `aws.ec2.log_files[].log_class`: CloudWatch log-group class for the collected file.
- `aws.ec2.key_name`: Optional EC2 key-pair name added to the launch template.
- `aws.ec2.additional_security_group_ids`: Existing security groups attached in addition to the managed security group.
- `aws.ec2.detailed_monitoring_enabled`: Enables detailed EC2 monitoring for runner instances.
- `aws.ec2.egress_rules`: Egress rules created on the managed runner security group.
- `aws.ec2.egress_rules[].cidr_blocks`: IPv4 CIDR destinations.
- `aws.ec2.egress_rules[].ipv6_cidr_blocks`: IPv6 CIDR destinations.
- `aws.ec2.egress_rules[].prefix_list_ids`: AWS prefix-list destinations.
- `aws.ec2.egress_rules[].from_port`: First destination port in the permitted range.
- `aws.ec2.egress_rules[].protocol`: IP protocol name or number. Use `-1` for all protocols.
- `aws.ec2.egress_rules[].security_groups`: Destination security-group IDs.
- `aws.ec2.egress_rules[].self`: Allows traffic to the managed security group itself when true.
- `aws.ec2.egress_rules[].to_port`: Last destination port in the permitted range.
- `aws.ec2.egress_rules[].description`: Optional rule description.
- `aws.ec2.tags`: Additional tags for runner instances, EBS volumes, network interfaces, and eligible Spot instance requests created from the launch template. They override module-level tags and the generated runner `Name`; the provider-managed `ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` bootstrap tags take final precedence. These tags do not apply to static provider resources such as the launch template, security group, IAM resources, SSM parameters, or log groups.
- `aws.ec2.metadata_options`: Instance Metadata Service configuration in the launch template.
- `aws.ec2.metadata_options.instance_metadata_tags`: Exposes instance tags through Instance Metadata Service when `enabled`.
- `aws.ec2.metadata_options.http_endpoint`: Enables or disables the Instance Metadata Service endpoint.
- `aws.ec2.metadata_options.http_tokens`: Controls whether IMDSv2 session tokens are optional or required.
- `aws.ec2.metadata_options.http_put_response_hop_limit`: Network hop limit for Instance Metadata Service token responses.
- `aws.ec2.credit_specification`: CPU credit mode for burstable instance types, either `standard` or `unlimited`.
- `aws.ec2.cpu_options`: CPU topology and processor-feature configuration.
- `aws.ec2.cpu_options.core_count`: Number of CPU cores exposed to the runner instance.
- `aws.ec2.cpu_options.threads_per_core`: Number of hardware threads exposed per CPU core.
- `aws.ec2.cpu_options.amd_sev_snp`: Enables or disables AMD SEV-SNP on supported instance types.
- `aws.ec2.cpu_options.nested_virtualization`: Enables or disables nested virtualization on supported instance types.
- `aws.ec2.placement`: EC2 placement configuration for runner instances.
- `aws.ec2.placement.affinity`: Host affinity setting.
- `aws.ec2.placement.availability_zone`: Availability Zone in which the instance is placed.
- `aws.ec2.placement.group_id`: Placement-group ID.
- `aws.ec2.placement.group_name`: Placement-group name.
- `aws.ec2.placement.host_id`: Dedicated Host ID.
- `aws.ec2.placement.host_resource_group_arn`: ARN of the host resource group used for placement.
- `aws.ec2.placement.spread_domain`: Spread-domain placement value.
- `aws.ec2.placement.tenancy`: Instance tenancy, such as `default`, `dedicated`, or `host`.
- `aws.ec2.placement.partition_number`: Placement-group partition number.
- `aws.ec2.license_specifications`: License Manager configurations added to the launch template.
- `aws.ec2.license_specifications[].license_configuration_arn`: ARN of a License Manager license configuration.
- `aws.ec2.associate_public_ipv4_address`: Associates a public IPv4 address with runner network interfaces.
- `aws.ec2.on_demand_failover_for_errors`: EC2 error codes that trigger an on-demand fallback after a Spot launch failure.
- `aws.ec2.scale_errors`: EC2 error codes treated as retryable scale-up failures.
- `aws.ec2.use_dedicated_host`: Enables the dedicated-host launch path, required for macOS runners. |
object({
aws = optional(object({
ec2 = optional(object({
ami = optional(object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter = optional(object({
arn = string
}), null)
kms_key = optional(object({
arn = string
}), null)
}), null)
vpc_id = string
subnet_ids = list(string)
overrides = optional(object({
name_runner = optional(string, "")
name_sg = optional(string, "")
}), {})
instance_profile = optional(object({
name = string
}), null)
instance_profile_path = optional(string, null)
binaries_syncer = optional(object({
enabled = optional(bool, true)
s3 = optional(object({
arn = string
id = string
key = string
}), null)
}), {})
block_device_mappings = optional(list(object({
delete_on_termination = optional(bool, true)
device_name = optional(string, "/dev/xvda")
encrypted = optional(bool, true)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = number
volume_type = optional(string, "gp3")
})), [{ volume_size = 30 }])
ebs_optimized = optional(bool, false)
instance_target_capacity_type = optional(string, "spot")
instance_allocation_strategy = optional(string, "lowest-price")
instance_type_priorities = optional(map(number), null)
instance_max_spot_price = optional(string, null)
instance_types = list(string)
user_data = optional(object({
enabled = optional(bool, true)
template = optional(string, null)
content = optional(string, null)
pre_install = optional(string, "")
post_install = optional(string, "")
debug_logging_enabled = optional(bool, false)
}), {})
ssm_enabled = optional(bool, false)
create_service_linked_role_spot = optional(bool, false)
cloudwatch_agent = optional(object({
enabled = optional(bool, true)
config = optional(string, null)
}), {})
managed_security_group_enabled = optional(bool, true)
log_files = optional(list(object({
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
log_class = optional(string, "STANDARD")
})), null)
key_name = optional(string, null)
additional_security_group_ids = optional(list(string), [])
detailed_monitoring_enabled = optional(bool, false)
egress_rules = optional(list(object({
cidr_blocks = list(string)
ipv6_cidr_blocks = list(string)
prefix_list_ids = list(string)
from_port = number
protocol = string
security_groups = list(string)
self = bool
to_port = number
description = string
})), [{
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
prefix_list_ids = null
from_port = 0
protocol = "-1"
security_groups = null
self = null
to_port = 0
description = null
}])
tags = optional(map(string), {})
metadata_options = optional(object({
instance_metadata_tags = optional(string, "enabled")
http_endpoint = optional(string, "enabled")
http_tokens = optional(string, "required")
http_put_response_hop_limit = optional(number, 1)
}), {})
credit_specification = optional(string, null)
cpu_options = optional(object({
core_count = optional(number)
threads_per_core = optional(number)
amd_sev_snp = optional(string)
nested_virtualization = optional(string)
}), null)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_id = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
spread_domain = optional(string)
tenancy = optional(string)
partition_number = optional(number)
}), null)
license_specifications = optional(list(object({
license_configuration_arn = string
})), [])
associate_public_ipv4_address = optional(bool, false)
on_demand_failover_for_errors = optional(list(string), [])
scale_errors = optional(list(string), [
"UnfulfillableCapacity",
"MaxSpotInstanceCountExceeded",
"TargetCapacityLimitExceededException",
"RequestLimitExceeded",
"ResourceLimitExceeded",
"MaxSpotInstanceCountExceeded",
"MaxSpotFleetRequestCountExceeded",
"InsufficientInstanceCapacity",
"InsufficientCapacityOnHost",
])
use_dedicated_host = optional(bool, false)
}), null)
}), {})
})
| n/a | yes | +| [compute\_provider\_key](#input\_compute\_provider\_key) | Optional plan-known compute-provider dispatch key. Null discovers the key from the exactly one populated compute\_provider block. | `string` | `null` | no | +| [github](#input\_github) | GitHub API and runner-registration configuration.

- `app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys.
- `app_parameters.id`: Ordered Parameter Store references for GitHub App IDs.
- `app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs.
- `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com.
- `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server requests.
- `user_agent`: Optional User-Agent value added to GitHub API requests. |
object({
app_parameters = object({
key_base64 = list(map(string))
id = list(map(string))
installation_id = list(object({ name = string, arn = string }))
})
enterprise_server = optional(object({
url = optional(string, null)
ssl_verify = optional(bool, true)
}), {})
user_agent = optional(string, null)
})
| n/a | yes | +| [lambda](#input\_lambda) | Common Lambda substrate independent of the selected runner orchestration provider.

- `artifact.s3.bucket`: Optional shared S3 bucket containing component-owned Lambda artifacts. An orchestration provider selects its own object key and version; the bucket alone selects no artifact.
- `runtime`: Runtime used by the control-plane Lambda functions.
- `architecture`: Instruction-set architecture used by the control-plane Lambda functions. Supported values are `arm64` and `x86_64`.
- `subnet_ids`: Subnets used for Lambda VPC configuration.
- `security_group_ids`: Security groups used for Lambda VPC configuration.
- `tags`: Shared tags applied to Lambda function resources only. These override module-level `tags`; component `tags` override this map when keys conflict.
- `principals`: Additional principals allowed to assume the control-plane Lambda roles.
- `role.path`: IAM path for module-managed Lambda execution roles. Defaults to a path derived from `prefix`.
- `role.permissions_boundary`: Permissions-boundary ARN applied to module-managed Lambda execution roles. |
object({
artifact = optional(object({
s3 = optional(object({
bucket = optional(string, null)
}), {})
}), {})
runtime = optional(string, "nodejs24.x")
architecture = optional(string, "arm64")
subnet_ids = optional(list(string), [])
security_group_ids = optional(list(string), [])
tags = optional(map(string), {})
principals = optional(list(object({
type = string
identifiers = list(string)
})), [])
role = optional(object({
path = optional(string, null)
permissions_boundary = optional(string, null)
}), {})
})
| `{}` | no | +| [observability](#input\_observability) | Logging, tracing, and metrics configuration for control-plane and provider resources.

- `logs.level`: Application log level supplied to the control-plane functions.
- `logs.retention_in_days`: CloudWatch Logs retention period.
- `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt CloudWatch log groups.
- `logs.class`: CloudWatch log-group class. Supported values are `STANDARD` and `INFREQUENT_ACCESS`.
- `logs.tags`: Shared tags for CloudWatch log groups. These override module-level `tags`; component `tags` override this map when keys conflict.
- `tracing.mode`: Optional Lambda active-tracing mode. Null disables X-Ray tracing configuration.
- `tracing.capture_http_requests`: Enables HTTP request capture in the tracing helper.
- `tracing.capture_error`: Enables error capture in the tracing helper.
- `metrics.enabled`: Enables module-emitted metrics.
- `metrics.namespace`: CloudWatch namespace used for emitted metrics.
- `metrics.metric.github_app_rate_limit.enabled`: Emits GitHub App rate-limit metrics.
- `metrics.metric.job_retry.enabled`: Emits job-retry metrics.
- `metrics.metric.spot_termination_warning.enabled`: Emits spot-termination warning metrics where supported. |
object({
logs = optional(object({
level = optional(string, "info")
retention_in_days = optional(number, 180)
kms_key_id = optional(string, null)
class = optional(string, "STANDARD")
tags = optional(map(string), {})
}), {})
tracing = optional(object({
mode = optional(string, null)
capture_http_requests = optional(bool, false)
capture_error = optional(bool, false)
}), {})
metrics = optional(object({
enabled = optional(bool, false)
namespace = optional(string, "GitHub Runners")
metric = optional(object({
github_app_rate_limit = optional(object({
enabled = optional(bool, true)
}), {})
job_retry = optional(object({
enabled = optional(bool, true)
}), {})
spot_termination_warning = optional(object({
enabled = optional(bool, true)
}), {})
}), {})
}), {})
})
| `{}` | no | +| [orchestration\_provider](#input\_orchestration\_provider) | Runner demand-orchestration provider configuration. Exactly one provider block must be non-null. Wrapper presence selects the provider and must therefore be known during planning; values inside the selected provider may remain unknown until apply.

- `webhook`: Selects the workflow-job webhook control plane. It owns runner lifecycle and capacity, the build queue reference, the runner-control artifact, scale-up, scale-down, scheduled pool, and optional job-retry controls. Future providers can be added as sibling blocks without moving this contract.
- `webhook.runner`: Runner lifecycle, boot timeout, and capacity settings owned by webhook orchestration.
- `webhook.runner.boot_time_in_minutes`: Expected runner boot duration used by scale-down and pool controls. The default is `5`.
- `webhook.runner.ephemeral`: Registers runners in ephemeral mode. The default is `false`.
- `webhook.runner.jit_config_enabled`: Explicitly enables or disables just-in-time configuration. The default is null, which follows `runner.ephemeral`.
- `webhook.runner.maximum_count`: Maximum number of runners managed for this runner configuration. The default is `3`.
- `webhook.github.organization_runners`: Registers runners at organization scope when true; otherwise registration is repository-scoped.
- `webhook.queue.build.arn`: ARN of the runner configuration's build queue.
- `webhook.queue.build.url`: URL of the runner configuration's build queue.
- `webhook.queue.kms_key_id`: Optional KMS key ARN encrypting the build queue. The default is null and is independent from the Parameter Store KMS key.
- `webhook.queue.tags`: Tags inherited by queue-related provider resources before component-specific overrides. The default is `{}`.
- `webhook.lambda.artifact`: Runner-control artifact shared by scale, pool, and job-retry components. Set at most one of `zip` or `s3`; no selection uses the packaged runner archive.
- `webhook.lambda.artifact.zip`: Optional local path to the runner-control Lambda archive. The default is null.
- `webhook.lambda.artifact.s3`: Optional S3 object selector in the common `lambda.artifact.s3.bucket`. Wrapper presence must be known during planning, selecting it requires a non-null common bucket, and the default is null.
- `webhook.lambda.artifact.s3.key`: Object key of the runner-control Lambda archive.
- `webhook.lambda.artifact.s3.object_version`: Optional object version of the runner-control Lambda archive. The default is null.
- `webhook.lambda.scale.up.memory_size`: Memory allocated to the scale-up Lambda in MB. The default is `512`.
- `webhook.lambda.scale.up.timeout`: Scale-up Lambda timeout in seconds. The default is `60`.
- `webhook.lambda.scale.up.reserved_concurrent_executions`: Reserved concurrency for scale-up. The default is `1`; use `-1` for unreserved concurrency.
- `webhook.lambda.scale.up.job_queued_check_enabled`: Enables queued-job verification before scaling. The default is null, which follows the resolved runner mode.
- `webhook.lambda.scale.up.event_source_mapping.batch_size`: Maximum build-queue records delivered per scale-up invocation. The default is `10`.
- `webhook.lambda.scale.up.event_source_mapping.maximum_batching_window_in_seconds`: Maximum batching window for build-queue records. The default is `0`.
- `webhook.lambda.scale.up.tags`: Tags applied within scale-up resource scopes after common provider tags. The default is `{}`.
- `webhook.lambda.scale.down.memory_size`: Memory allocated to the scale-down Lambda in MB. The default is `512`.
- `webhook.lambda.scale.down.timeout`: Scale-down Lambda timeout in seconds. The default is `60`.
- `webhook.lambda.scale.down.schedule_expression`: EventBridge schedule expression that invokes scale-down. The default is `cron(*/5 * * * ? *)`.
- `webhook.lambda.scale.down.minimum_running_time_in_minutes`: Optional minimum runner age before scale-down may terminate it. The default is null, which selects the operating-system default.
- `webhook.lambda.scale.down.idle_config`: Time-based desired idle-runner configurations. The default is `[]`.
- `webhook.lambda.scale.down.idle_config[].cron`: Cron expression identifying when the idle configuration applies.
- `webhook.lambda.scale.down.idle_config[].timeZone`: IANA time zone used to evaluate the cron expression.
- `webhook.lambda.scale.down.idle_config[].idleCount`: Number of idle runners retained during the matching period.
- `webhook.lambda.scale.down.idle_config[].evictionStrategy`: Selection strategy used when excess idle runners are removed. The default is `oldest_first`.
- `webhook.lambda.scale.down.tags`: Tags applied within scale-down resource scopes after common provider tags. The default is `{}`.
- `webhook.lambda.pool.memory_size`: Memory allocated to the pool Lambda in MB. The default is `512`.
- `webhook.lambda.pool.timeout`: Pool Lambda timeout in seconds. The default is `60`.
- `webhook.lambda.pool.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. The default is `1`; use `-1` for unreserved concurrency.
- `webhook.lambda.pool.config`: Scheduled target pool sizes. The default is `[]`, which disables the pool component.
- `webhook.lambda.pool.config[].schedule_expression`: Scheduler expression that activates the target size.
- `webhook.lambda.pool.config[].schedule_expression_timezone`: Optional IANA time zone used to evaluate the schedule.
- `webhook.lambda.pool.config[].size`: Desired number of runners for the schedule.
- `webhook.lambda.pool.include_busy_runners`: Includes busy runners when reconciling scheduled pool capacity. The default is `false`.
- `webhook.lambda.pool.runner_owner`: Optional GitHub organization or repository owner used for pooled runners. The default is null.
- `webhook.lambda.pool.tags`: Tags applied within pool resource scopes after common provider tags. The default is `{}`.
- `webhook.job_retry.enabled`: Creates the retry queue, Lambda function, event-source mapping, and related IAM resources. The default is `false`.
- `webhook.job_retry.delay_in_seconds`: Initial delay before a queued-job retry check. The default is `300`.
- `webhook.job_retry.delay_backoff`: Multiplier applied to the delay after each unsuccessful check. The default is `2`.
- `webhook.job_retry.max_attempts`: Maximum retry-check attempts before the message is no longer republished. The default is `1`.
- `webhook.job_retry.tags`: Tags applied within job-retry resource scopes after common provider tags. The default is `{}`.
- `webhook.job_retry.lambda.memory_size`: Memory allocated to the job-retry Lambda in MB. The default is `256`.
- `webhook.job_retry.lambda.reserved_concurrent_executions`: Reserved concurrency for job retry. The default is `1`; use `-1` for unreserved concurrency.
- `webhook.job_retry.lambda.timeout`: Job-retry Lambda timeout in seconds and visibility timeout for its retry queue. The default is `30`. |
object({
webhook = optional(object({
runner = optional(object({
boot_time_in_minutes = optional(number, 5)
ephemeral = optional(bool, false)
jit_config_enabled = optional(bool, null)
maximum_count = optional(number, 3)
}), {})
github = object({
organization_runners = bool
})
queue = object({
build = object({
arn = string
url = string
})
kms_key_id = optional(string, null)
tags = optional(map(string), {})
})
lambda = optional(object({
artifact = optional(object({
zip = optional(string, null)
s3 = optional(object({
key = string
object_version = optional(string, null)
}), null)
}), {})
scale = optional(object({
up = optional(object({
memory_size = optional(number, 512)
timeout = optional(number, 60)
reserved_concurrent_executions = optional(number, 1)
job_queued_check_enabled = optional(bool, null)
event_source_mapping = optional(object({
batch_size = optional(number, 10)
maximum_batching_window_in_seconds = optional(number, 0)
}), {})
tags = optional(map(string), {})
}), {})
down = optional(object({
memory_size = optional(number, 512)
timeout = optional(number, 60)
schedule_expression = optional(string, "cron(*/5 * * * ? *)")
minimum_running_time_in_minutes = optional(number, null)
idle_config = optional(list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = optional(string, "oldest_first")
})), [])
tags = optional(map(string), {})
}), {})
}), {})
pool = optional(object({
memory_size = optional(number, 512)
timeout = optional(number, 60)
reserved_concurrent_executions = optional(number, 1)
config = optional(list(object({
schedule_expression = string
schedule_expression_timezone = optional(string)
size = number
})), [])
include_busy_runners = optional(bool, false)
runner_owner = optional(string, null)
tags = optional(map(string), {})
}), {})
}), {})
job_retry = optional(object({
enabled = optional(bool, false)
delay_in_seconds = optional(number, 300)
delay_backoff = optional(number, 2)
max_attempts = optional(number, 1)
tags = optional(map(string), {})
lambda = optional(object({
memory_size = optional(number, 256)
reserved_concurrent_executions = optional(number, 1)
timeout = optional(number, 30)
}), {})
}), {})
}), null)
})
| n/a | yes | +| [prefix](#input\_prefix) | The prefix used for naming resources. | `string` | `"github-actions"` | no | +| [runner](#input\_runner) | Provider-neutral GitHub runner configuration.

- `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`.
- `architecture`: Runner distribution architecture, such as `x64` or `arm64`.
- `disable_default_labels`: Prevents GitHub's default self-hosted, operating-system, and architecture labels from being registered.
- `labels`: Complete set of labels supplied to the control-plane functions.
- `group_name`: GitHub runner group used during registration.
- `name_prefix`: Prefix added to registered runner names.
- `run_as_root`: Runs the runner service as root when supported by the compute provider.
- `run_as`: Operating-system user used when `run_as_root` is false.
- `auto_update_disabled`: Disables the GitHub runner application's built-in updater.
- `tags`: Additional tags for common runner resources, currently the managed runner IAM role. These override module-level `tags` with the same key.
- `hooks.job_started`: Script content installed as the runner job-started hook.
- `hooks.job_completed`: Script content installed as the runner job-completed hook.
- `iam.role.arn`: ARN of an externally managed runner role. When set, this module does not create or modify that role.
- `iam.managed_policy_arns`: Named managed-policy ARNs attached to the module-managed runner role.
- `iam.additional_trust_policy_json`: Optional IAM policy document merged with the selected compute provider's default runner-role trust policy.
- `iam.path`: IAM path for the module-managed runner role. Defaults to a path derived from `prefix`.
- `iam.permissions_boundary`: Permissions-boundary ARN for the module-managed runner role. |
object({
os = optional(string, "linux")
architecture = optional(string, "x64")
disable_default_labels = optional(bool, false)
labels = list(string)
group_name = optional(string, "Default")
name_prefix = optional(string, "")
run_as_root = optional(bool, false)
run_as = optional(string, "ec2-user")
auto_update_disabled = optional(bool, false)
tags = optional(map(string), {})
hooks = optional(object({
job_started = optional(string, "")
job_completed = optional(string, "")
}), {})
iam = optional(object({
role = optional(object({
arn = string
}), null)
managed_policy_arns = optional(map(string), {})
additional_trust_policy_json = optional(string, null)
path = optional(string, null)
permissions_boundary = optional(string, null)
}), {})
})
| n/a | yes | +| [ssm](#input\_ssm) | Parameter Store paths, encryption, tag scopes, and housekeeper configuration.

- `paths.root`: Root Parameter Store path for this runner configuration.
- `paths.tokens`: Path segment under `paths.root` used for registration tokens and just-in-time configuration.
- `paths.config`: Path segment under `paths.root` used for persistent runner configuration.
- `kms_key_id`: Optional customer-managed KMS key ARN used by control-plane IAM policies to decrypt shared GitHub App parameters. The ARN may be unknown until apply; null omits the provider-owned KMS statements. It does not select encryption for runtime-created runner parameters.
- `tags`: Shared tags for SSM-related resources. These override module-level `tags` and are inherited by parameter and housekeeper resources.
- `parameters.tags`: Tags for Terraform-managed runner configuration parameters and temporary parameters created by the scale-up and pool Lambdas. These override module-level and `ssm.tags` values with the same key.
- `housekeeper.schedule_expression`: EventBridge schedule expression that invokes the SSM housekeeper.
- `housekeeper.state`: EventBridge rule state, such as `ENABLED` or `DISABLED`.
- `housekeeper.tags`: Tags for housekeeper resources, including the Lambda function, log group, EventBridge rule, and IAM role. These override module-level, `ssm.tags`, shared Lambda, and shared log tags when keys conflict.
- `housekeeper.lambda.artifact`: Component-owned SSM-housekeeper artifact selection. Set at most one of `zip` or `s3`; when neither is selected, the module uses its packaged runner control-plane archive. This selector does not inherit an orchestration-provider artifact.
- `housekeeper.lambda.artifact.zip`: Optional local path to the SSM-housekeeper Lambda archive.
- `housekeeper.lambda.artifact.s3`: Optional object key and version in the shared `lambda.artifact.s3.bucket`. Selecting S3 requires that common bucket.
- `housekeeper.lambda.artifact.s3.key`: Object key of the SSM-housekeeper Lambda archive.
- `housekeeper.lambda.artifact.s3.object_version`: Optional object version of the SSM-housekeeper Lambda archive.
- `housekeeper.lambda.memory_size`: Memory allocated to the SSM housekeeper Lambda in MB.
- `housekeeper.lambda.timeout`: SSM housekeeper Lambda timeout in seconds.
- `housekeeper.config.tokenPath`: Parameter Store token path cleaned by the housekeeper. When omitted, the configured runner token path is used.
- `housekeeper.config.minimumDaysOld`: Minimum parameter age in days before deletion is allowed.
- `housekeeper.config.dryRun`: Reports eligible parameters without deleting them when true. |
object({
paths = object({
root = string
tokens = string
config = string
})
kms_key_id = optional(string, null)
tags = optional(map(string), {})
parameters = optional(object({
tags = optional(map(string), {})
}), {})
housekeeper = optional(object({
schedule_expression = optional(string, "rate(1 day)")
state = optional(string, "ENABLED")
tags = optional(map(string), {})
lambda = optional(object({
artifact = optional(object({
zip = optional(string, null)
s3 = optional(object({
key = string
object_version = optional(string, null)
}), null)
}), {})
memory_size = optional(number, 512)
timeout = optional(number, 60)
}), {})
config = optional(object({
tokenPath = optional(string)
minimumDaysOld = optional(number, 1)
dryRun = optional(bool, false)
}), {})
}), {})
})
| n/a | yes | +| [tags](#input\_tags) | Base tags added to taggable resources created by this runner configuration. Shared, component, and compute-provider tag maps override matching keys within their documented resource scopes. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [orchestration\_provider](#output\_orchestration\_provider) | Resources grouped under the selected runner orchestration provider. | +| [pool](#output\_pool) | Scheduled pool resources. Null when no pool configuration is supplied. | +| [provider](#output\_provider) | Provider-specific resources grouped under the selected provider namespace and type. | +| [runner](#output\_runner) | Common runner resources. The role is null when an external runner role is used. | +| [scale\_down](#output\_scale\_down) | Scale-down control-plane resources. Null when webhook orchestration is not configured. | +| [scale\_up](#output\_scale\_up) | Scale-up control-plane resources. Null when webhook orchestration is not configured. | + diff --git a/modules/runner-config/common-config.tf b/modules/runner-config/common-config.tf new file mode 100644 index 0000000000..660fcc60ab --- /dev/null +++ b/modules/runner-config/common-config.tf @@ -0,0 +1,44 @@ +# Shared control-plane configuration: naming, paths, tags, and normalized values. +locals { + common_tags = var.tags + runner_tags = merge(local.common_tags, var.runner.tags) + lambda_tags = merge(local.common_tags, var.lambda.tags) + observability_log_tags = merge(local.common_tags, var.observability.logs.tags) + + ssm_tags = merge(local.common_tags, var.ssm.tags) + ssm_parameter_tags = merge(local.ssm_tags, var.ssm.parameters.tags) + ssm_housekeeper_tags = merge(local.ssm_tags, var.ssm.housekeeper.tags) + ssm_housekeeper_lambda_tags = merge(local.lambda_tags, var.ssm.tags, var.ssm.housekeeper.tags) + ssm_housekeeper_log_tags = merge(local.observability_log_tags, var.ssm.tags, var.ssm.housekeeper.tags) + + lambda_role_path = var.lambda.role.path == null ? "/${var.prefix}/" : var.lambda.role.path + runner_role_path = var.runner.iam.path == null ? "/${var.prefix}/" : var.runner.iam.path + packaged_runners_lambda_zip = "${path.module}/../../lambdas/functions/control-plane/runners.zip" + ssm_housekeeper_artifact_s3_selected = ( + var.ssm.housekeeper.lambda.artifact.s3 != null + ) + ssm_housekeeper_artifact = { + zip = local.ssm_housekeeper_artifact_s3_selected ? null : coalesce( + var.ssm.housekeeper.lambda.artifact.zip, + local.packaged_runners_lambda_zip, + ) + s3 = { + bucket = local.ssm_housekeeper_artifact_s3_selected ? var.lambda.artifact.s3.bucket : null + key = try(var.ssm.housekeeper.lambda.artifact.s3.key, null) + object_version = try(var.ssm.housekeeper.lambda.artifact.s3.object_version, null) + } + } + kms_key_id = var.ssm.kms_key_id + token_path = "${var.ssm.paths.root}/${var.ssm.paths.tokens}" + arn_ssm_parameters_path_tokens = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.ssm.paths.root}/${var.ssm.paths.tokens}" + arn_ssm_parameters_path_config = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.ssm.paths.root}/${var.ssm.paths.config}" + + parameter_store_tags = jsonencode([ + for key, value in local.ssm_parameter_tags : { + Key = key + Value = value + } + ]) +} + +data "aws_caller_identity" "current" {} diff --git a/modules/runner-config/compute-provider.aws.ec2.tf b/modules/runner-config/compute-provider.aws.ec2.tf new file mode 100644 index 0000000000..5d053a73dc --- /dev/null +++ b/modules/runner-config/compute-provider.aws.ec2.tf @@ -0,0 +1,37 @@ +module "compute_aws_ec2_trust_policy" { + count = local.provider_key == "aws_ec2" ? 1 : 0 + source = "../compute-providers/aws/ec2/trust-policy" + + additional_trust_policy_json = var.runner.iam.additional_trust_policy_json +} + +module "compute_aws_ec2" { + count = local.provider_key == "aws_ec2" ? 1 : 0 + source = "../compute-providers/aws/ec2" + + aws_partition = var.aws_partition + aws_region = var.aws_region + prefix = var.prefix + tags = var.tags + + config = var.compute_provider.aws.ec2 + runner = merge(var.runner, { + iam = merge(var.runner.iam, { + role = local.runner_role + managed_policy_arns = local.common_runner_managed_policy_arns + }) + }) + github = var.github + ssm = var.ssm + observability = var.observability +} + +moved { + from = module.compute_ec2_trust_policy + to = module.compute_aws_ec2_trust_policy +} + +moved { + from = module.compute_ec2 + to = module.compute_aws_ec2 +} diff --git a/modules/runner-config/compute-provider.tf b/modules/runner-config/compute-provider.tf new file mode 100644 index 0000000000..bffc43b814 --- /dev/null +++ b/modules/runner-config/compute-provider.tf @@ -0,0 +1,29 @@ +locals { + compute_providers = { + aws_ec2 = var.compute_provider.aws.ec2 + } + + discovered_provider_key = one([ + for provider_key, provider_config in local.compute_providers : provider_key + if provider_config != null + ]) + provider_key = var.compute_provider_key != null ? var.compute_provider_key : local.discovered_provider_key + + provider_types = { + aws_ec2 = "ec2" + } + + provider_type = local.provider_types[local.provider_key] + + provider_assume_role_policies = { + aws_ec2 = try(module.compute_aws_ec2_trust_policy[0].assume_role_policy, null) + } + + provider_assume_role_policy = local.provider_assume_role_policies[local.provider_key] + + provider_contracts = { + aws_ec2 = one(module.compute_aws_ec2[*].provider) + } + + provider_contract = local.provider_contracts[local.provider_key] +} diff --git a/modules/runner-config/orchestration-provider.tf b/modules/runner-config/orchestration-provider.tf new file mode 100644 index 0000000000..25994beaba --- /dev/null +++ b/modules/runner-config/orchestration-provider.tf @@ -0,0 +1,73 @@ +locals { + orchestration_providers = { + for provider_type, provider_config in var.orchestration_provider : provider_type => provider_config + if provider_config != null + } + + orchestration_provider_type = one(keys(local.orchestration_providers)) + + orchestration_provider_enabled = { + webhook = local.orchestration_provider_type == "webhook" + } + + orchestration_provider_runner_lifecycle = { + webhook = one(module.orchestration_webhook[*].runner_lifecycle) + }[local.orchestration_provider_type] +} + +module "orchestration_webhook" { + source = "../orchestration-providers/webhook" + count = local.orchestration_provider_enabled.webhook ? 1 : 0 + + aws_partition = var.aws_partition + prefix = var.prefix + tags = var.tags + + config = var.orchestration_provider.webhook + runner = var.runner + github = var.github + lambda = { + artifact = var.lambda.artifact + runtime = var.lambda.runtime + architecture = var.lambda.architecture + subnet_ids = var.lambda.subnet_ids + security_group_ids = var.lambda.security_group_ids + tags = var.lambda.tags + role = { + path = local.lambda_role_path + permissions_boundary = var.lambda.role.permissions_boundary + principals = var.lambda.principals + } + } + ssm = { + token_path = local.token_path + token_path_arn = local.arn_ssm_parameters_path_tokens + config_path = "${var.ssm.paths.root}/${var.ssm.paths.config}" + config_path_arn = local.arn_ssm_parameters_path_config + kms_key_id = local.kms_key_id + parameter_store_tags = local.parameter_store_tags + } + observability = var.observability + + runner_provider = { + type = local.provider_type + scale_up = { + environment_variables = local.provider_contract.environment_variables.scale_up + iam_policy_json = local.provider_contract.policies.scale_up.iam_policy_json + additional_iam_policy_json = local.provider_contract.policies.scale_up.additional_iam_policy_json + managed_policy = local.provider_contract.policies.scale_up.managed_policy_enabled ? { + arn = local.provider_contract.policies.scale_up.managed_policy_arn + } : null + } + scale_down = { + environment_variables = local.provider_contract.environment_variables.scale_down + iam_policy_json = local.provider_contract.policies.scale_down.iam_policy_json + } + pool = { + environment_variables = local.provider_contract.environment_variables.pool + iam_policy_json = local.provider_contract.policies.pool.iam_policy_json + managed_policy_enabled = local.provider_contract.policies.pool.managed_policy_enabled + managed_policy_arn = local.provider_contract.policies.pool.managed_policy_arn + } + } +} diff --git a/modules/runner-config/outputs.tf b/modules/runner-config/outputs.tf new file mode 100644 index 0000000000..486e3261eb --- /dev/null +++ b/modules/runner-config/outputs.tf @@ -0,0 +1,42 @@ +output "runner" { + description = "Common runner resources. The role is null when an external runner role is used." + value = { + role = one(aws_iam_role.runner[*]) + } +} + +output "scale_up" { + description = "Scale-up control-plane resources. Null when webhook orchestration is not configured." + value = one(module.orchestration_webhook[*].scale_up) +} + +output "scale_down" { + description = "Scale-down control-plane resources. Null when webhook orchestration is not configured." + value = one(module.orchestration_webhook[*].scale_down) +} + +output "pool" { + description = "Scheduled pool resources. Null when no pool configuration is supplied." + value = one(module.orchestration_webhook[*].pool) +} + +output "orchestration_provider" { + description = "Resources grouped under the selected runner orchestration provider." + value = { + webhook = local.orchestration_provider_enabled.webhook ? { + scale_up = one(module.orchestration_webhook[*].scale_up) + scale_down = one(module.orchestration_webhook[*].scale_down) + pool = one(module.orchestration_webhook[*].pool) + job_retry = one(module.orchestration_webhook[*].job_retry) + } : null + } +} + +output "provider" { + description = "Provider-specific resources grouped under the selected provider namespace and type." + value = { + aws = { + ec2 = local.provider_key == "aws_ec2" ? local.provider_contract.resources : null + } + } +} diff --git a/modules/runner-config/runner-role.tf b/modules/runner-config/runner-role.tf new file mode 100644 index 0000000000..6baa1e4206 --- /dev/null +++ b/modules/runner-config/runner-role.tf @@ -0,0 +1,48 @@ +locals { + # Role ownership belongs to the common runner configuration. The selected trust-policy + # submodule supplies the assume-role document, while the full compute provider + # supplies permissions after the role has been resolved. + create_runner_role = var.runner.iam.role == null + + runner_role = { + arn = local.create_runner_role ? one(aws_iam_role.runner[*].arn) : var.runner.iam.role.arn + name = local.create_runner_role ? one(aws_iam_role.runner[*].name) : basename(var.runner.iam.role.arn) + managed = local.create_runner_role + } + + common_runner_managed_policy_arns = merge( + { + for policy_name, policy_arn in var.runner.iam.managed_policy_arns : + "user-${policy_name}" => policy_arn + }, + var.observability.tracing.mode != null ? { + xray = "arn:${var.aws_partition}:iam::aws:policy/AWSXRayDaemonWriteAccess" + } : {}, + ) + + provider_runner_policies = local.provider_contract.policies.runner +} + +resource "aws_iam_role" "runner" { + count = local.create_runner_role ? 1 : 0 + name = "${substr("${var.prefix}-runner", 0, 54)}-${substr(md5("${var.prefix}-runner"), 0, 8)}" + assume_role_policy = local.provider_assume_role_policy + path = local.runner_role_path + permissions_boundary = var.runner.iam.permissions_boundary + tags = local.runner_tags +} + +resource "aws_iam_role_policy" "runner_provider" { + for_each = local.create_runner_role ? local.provider_runner_policies.inline_policies : {} + + name = each.value.name + role = aws_iam_role.runner[0].name + policy = each.value.policy_json +} + +resource "aws_iam_role_policy_attachment" "runner" { + for_each = local.create_runner_role ? local.provider_runner_policies.managed_policy_arns : {} + + role = aws_iam_role.runner[0].name + policy_arn = each.value +} diff --git a/modules/runner-config/runner-ssm-parameters.tf b/modules/runner-config/runner-ssm-parameters.tf new file mode 100644 index 0000000000..1d97c908a8 --- /dev/null +++ b/modules/runner-config/runner-ssm-parameters.tf @@ -0,0 +1,28 @@ +# Shared runner configuration stored in SSM Parameter Store. +resource "aws_ssm_parameter" "runner_agent_mode" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/agent_mode" + type = "String" + value = local.orchestration_provider_runner_lifecycle.ephemeral ? "ephemeral" : "persistent" + tags = local.ssm_parameter_tags +} + +resource "aws_ssm_parameter" "disable_default_labels" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/disable_default_labels" + type = "String" + value = var.runner.disable_default_labels + tags = local.ssm_parameter_tags +} + +resource "aws_ssm_parameter" "jit_config_enabled" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/enable_jit_config" + type = "String" + value = local.orchestration_provider_runner_lifecycle.jit_config_enabled + tags = local.ssm_parameter_tags +} + +resource "aws_ssm_parameter" "token_path" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/token_path" + type = "String" + value = "${var.ssm.paths.root}/${var.ssm.paths.tokens}" + tags = local.ssm_parameter_tags +} diff --git a/modules/runner-config/ssm-housekeeper.tf b/modules/runner-config/ssm-housekeeper.tf new file mode 100644 index 0000000000..5bb31bb7b5 --- /dev/null +++ b/modules/runner-config/ssm-housekeeper.tf @@ -0,0 +1,57 @@ +locals { + ssm_housekeeper_token_path = coalesce(var.ssm.housekeeper.config.tokenPath, local.token_path) + ssm_housekeeper_parameter_path_arn = ( + "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${local.ssm_housekeeper_token_path}*" + ) +} + +module "ssm_housekeeper" { + source = "./ssm-housekeeper" + + config = { + prefix = var.prefix + aws_partition = var.aws_partition + schedule = { + expression = var.ssm.housekeeper.schedule_expression + state = var.ssm.housekeeper.state + } + cleanup = { + token_path = local.ssm_housekeeper_token_path + parameter_path_arn = local.ssm_housekeeper_parameter_path_arn + minimum_days_old = var.ssm.housekeeper.config.minimumDaysOld + dry_run = var.ssm.housekeeper.config.dryRun + } + lambda = { + # The housekeeper resolves only its component-owned selector and never + # inherits the selected orchestration provider's runner-control artifact. + artifact = local.ssm_housekeeper_artifact + runtime = var.lambda.runtime + architecture = var.lambda.architecture + memory_size = var.ssm.housekeeper.lambda.memory_size + timeout = var.ssm.housekeeper.lambda.timeout + vpc = { + subnet_ids = var.lambda.subnet_ids + security_group_ids = var.lambda.security_group_ids + } + role = { + path = local.lambda_role_path + permissions_boundary = var.lambda.role.permissions_boundary + principals = var.lambda.principals + } + } + observability = { + logs = { + level = var.observability.logs.level + retention_in_days = var.observability.logs.retention_in_days + kms_key_id = var.observability.logs.kms_key_id + class = var.observability.logs.class + } + tracing = var.observability.tracing + } + tags = { + resources = local.ssm_housekeeper_tags + lambda = local.ssm_housekeeper_lambda_tags + log_group = local.ssm_housekeeper_log_tags + } + } +} diff --git a/modules/runner-config/ssm-housekeeper/README.md b/modules/runner-config/ssm-housekeeper/README.md new file mode 100644 index 0000000000..5f5d1ad166 --- /dev/null +++ b/modules/runner-config/ssm-housekeeper/README.md @@ -0,0 +1,57 @@ +# SSM housekeeper module + +> This module is treated as an internal module; breaking changes do not trigger a major release bump. + +This provider-neutral child module owns the Lambda function, EventBridge schedule, IAM policies, and CloudWatch log group used to remove expired runner registration parameters from Parameter Store. + +The module is an implementation detail of the experimental runner configuration. It is composed by `runner-config` and is not intended to be called directly. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_event_rule.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_target.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_cloudwatch_log_group.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_role.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.ssm_housekeeper_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.ssm_housekeeper_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.ssm_housekeeper_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_lambda_function.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_lambda_permission.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [aws_iam_policy_document.lambda_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.ssm_housekeeper_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [config](#input\_config) | Provider-neutral SSM housekeeper configuration assembled by runner-config.

- `prefix`: Prefix used to name the housekeeper resources.
- `aws_partition`: AWS partition used to construct IAM policy ARNs.
- `schedule.expression`: EventBridge schedule expression that invokes the housekeeper.
- `schedule.state`: State of the EventBridge rule.
- `cleanup.token_path`: Parameter Store token path supplied to the Lambda.
- `cleanup.parameter_path_arn`: IAM resource ARN matching `cleanup.token_path`.
- `cleanup.minimum_days_old`: Minimum parameter age before deletion.
- `cleanup.dry_run`: Reports eligible parameters without deleting them when true.
- `lambda.artifact.zip`: Resolved local control-plane archive.
- `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive.
- `lambda.artifact.s3.key`: Object key of the Lambda archive.
- `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive.
- `lambda.runtime`: Runtime used by the housekeeper Lambda.
- `lambda.architecture`: Instruction-set architecture used by the housekeeper Lambda.
- `lambda.memory_size`: Memory allocated to the housekeeper Lambda.
- `lambda.timeout`: Housekeeper Lambda timeout in seconds.
- `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration.
- `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration.
- `lambda.role.path`: IAM path used for the housekeeper Lambda role.
- `lambda.role.permissions_boundary`: Optional permissions boundary for the housekeeper role.
- `lambda.role.principals`: Additional principals allowed to assume the housekeeper Lambda role.
- `observability.logs`: Logging level, retention, encryption, and log-class configuration.
- `observability.tracing`: Lambda X-Ray and tracing-helper configuration.
- `tags.resources`: Tags for the housekeeper role and EventBridge rule.
- `tags.lambda`: Tags for the housekeeper Lambda function.
- `tags.log_group`: Tags for the housekeeper log group. |
object({
prefix = string
aws_partition = string
schedule = object({
expression = string
state = string
})
cleanup = object({
token_path = string
parameter_path_arn = string
minimum_days_old = number
dry_run = bool
})
lambda = object({
artifact = object({
zip = string
s3 = object({
bucket = optional(string, null)
key = optional(string, null)
object_version = optional(string, null)
})
})
runtime = string
architecture = string
memory_size = number
timeout = number
vpc = object({
subnet_ids = list(string)
security_group_ids = list(string)
})
role = object({
path = string
permissions_boundary = optional(string, null)
principals = optional(list(object({
type = string
identifiers = list(string)
})), [])
})
})
observability = object({
logs = object({
level = string
retention_in_days = number
kms_key_id = optional(string, null)
class = string
})
tracing = object({
mode = optional(string, null)
capture_http_requests = bool
capture_error = bool
})
})
tags = object({
resources = map(string)
lambda = map(string)
log_group = map(string)
})
})
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [housekeeper](#output\_housekeeper) | SSM housekeeper Lambda resources. | + diff --git a/modules/runner-config/ssm-housekeeper/iam-policies.tf b/modules/runner-config/ssm-housekeeper/iam-policies.tf new file mode 100644 index 0000000000..8d3bab2865 --- /dev/null +++ b/modules/runner-config/ssm-housekeeper/iam-policies.tf @@ -0,0 +1,58 @@ +data "aws_iam_policy_document" "lambda_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + + dynamic "principals" { + for_each = var.config.lambda.role.principals + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + } +} + +data "aws_iam_policy_document" "lambda_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + + # AWS X-Ray trace APIs do not support resource-level permissions. + statement { + sid = "AllowXRay" + effect = "Allow" + actions = [ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments", + ] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "ssm_housekeeper" { + statement { + effect = "Allow" + actions = [ + "ssm:DeleteParameter", + "ssm:GetParametersByPath", + ] + resources = [var.config.cleanup.parameter_path_arn] + } +} + +data "aws_iam_policy_document" "ssm_housekeeper_logging" { + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = ["${aws_cloudwatch_log_group.ssm_housekeeper.arn}*"] + } +} diff --git a/modules/runner-config/ssm-housekeeper/outputs.tf b/modules/runner-config/ssm-housekeeper/outputs.tf new file mode 100644 index 0000000000..064f5a1ab1 --- /dev/null +++ b/modules/runner-config/ssm-housekeeper/outputs.tf @@ -0,0 +1,8 @@ +output "housekeeper" { + description = "SSM housekeeper Lambda resources." + value = { + lambda = aws_lambda_function.ssm_housekeeper + log_group = aws_cloudwatch_log_group.ssm_housekeeper + role = aws_iam_role.ssm_housekeeper + } +} diff --git a/modules/runner-config/ssm-housekeeper/ssm-housekeeper.tf b/modules/runner-config/ssm-housekeeper/ssm-housekeeper.tf new file mode 100644 index 0000000000..bcafed201a --- /dev/null +++ b/modules/runner-config/ssm-housekeeper/ssm-housekeeper.tf @@ -0,0 +1,119 @@ +locals { + vpc_enabled = ( + length(var.config.lambda.vpc.subnet_ids) > 0 && + length(var.config.lambda.vpc.security_group_ids) > 0 + ) + + cleanup_config = { + tokenPath = var.config.cleanup.token_path + minimumDaysOld = var.config.cleanup.minimum_days_old + dryRun = var.config.cleanup.dry_run + } +} + +resource "aws_lambda_function" "ssm_housekeeper" { + s3_bucket = var.config.lambda.artifact.s3.bucket + s3_key = var.config.lambda.artifact.s3.key + s3_object_version = var.config.lambda.artifact.s3.object_version + filename = var.config.lambda.artifact.s3.bucket == null ? var.config.lambda.artifact.zip : null + source_code_hash = var.config.lambda.artifact.s3.bucket == null ? filebase64sha256(var.config.lambda.artifact.zip) : null + function_name = "${var.config.prefix}-ssm-housekeeper" + role = aws_iam_role.ssm_housekeeper.arn + handler = "index.ssmHousekeeper" + runtime = var.config.lambda.runtime + timeout = var.config.lambda.timeout + tags = var.config.tags.lambda + memory_size = var.config.lambda.memory_size + architectures = [var.config.lambda.architecture] + + environment { + variables = { + ENVIRONMENT = var.config.prefix + LOG_LEVEL = upper(var.config.observability.logs.level) + SSM_CLEANUP_CONFIG = jsonencode(local.cleanup_config) + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-ssm-housekeeper" + POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error + } + } + + dynamic "vpc_config" { + for_each = local.vpc_enabled ? [true] : [] + + content { + security_group_ids = var.config.lambda.vpc.security_group_ids + subnet_ids = var.config.lambda.vpc.subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.config.observability.tracing.mode != null ? [true] : [] + + content { + mode = var.config.observability.tracing.mode + } + } +} + +resource "aws_cloudwatch_log_group" "ssm_housekeeper" { + name = "/aws/lambda/${aws_lambda_function.ssm_housekeeper.function_name}" + retention_in_days = var.config.observability.logs.retention_in_days + kms_key_id = var.config.observability.logs.kms_key_id + log_group_class = var.config.observability.logs.class + tags = var.config.tags.log_group +} + +resource "aws_cloudwatch_event_rule" "ssm_housekeeper" { + name = "${var.config.prefix}-ssm-housekeeper" + schedule_expression = var.config.schedule.expression + state = var.config.schedule.state + tags = var.config.tags.resources +} + +resource "aws_cloudwatch_event_target" "ssm_housekeeper" { + rule = aws_cloudwatch_event_rule.ssm_housekeeper.name + arn = aws_lambda_function.ssm_housekeeper.arn +} + +resource "aws_lambda_permission" "ssm_housekeeper" { + statement_id = "AllowExecutionFromCloudWatch" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.ssm_housekeeper.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.ssm_housekeeper.arn +} + +resource "aws_iam_role" "ssm_housekeeper" { + name = "${substr("${var.config.prefix}-ssm-hk-lambda", 0, 54)}-${substr(md5("${var.config.prefix}-ssm-hk-lambda"), 0, 8)}" + description = "Lambda role for SSM Housekeeper (${var.config.prefix})" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json + path = var.config.lambda.role.path + permissions_boundary = var.config.lambda.role.permissions_boundary + tags = var.config.tags.resources +} + +resource "aws_iam_role_policy" "ssm_housekeeper" { + name = "ssm-policy" + role = aws_iam_role.ssm_housekeeper.name + policy = data.aws_iam_policy_document.ssm_housekeeper.json +} + +resource "aws_iam_role_policy" "ssm_housekeeper_logging" { + name = "logging-policy" + role = aws_iam_role.ssm_housekeeper.name + policy = data.aws_iam_policy_document.ssm_housekeeper_logging.json +} + +resource "aws_iam_role_policy_attachment" "ssm_housekeeper_vpc_execution_role" { + count = local.vpc_enabled ? 1 : 0 + role = aws_iam_role.ssm_housekeeper.name + policy_arn = "arn:${var.config.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +resource "aws_iam_role_policy" "ssm_housekeeper_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + name = "xray-policy" + policy = data.aws_iam_policy_document.lambda_xray[0].json + role = aws_iam_role.ssm_housekeeper.name +} diff --git a/modules/runner-config/ssm-housekeeper/tests/ssm-housekeeper.tftest.hcl b/modules/runner-config/ssm-housekeeper/tests/ssm-housekeeper.tftest.hcl new file mode 100644 index 0000000000..bac30c6752 --- /dev/null +++ b/modules/runner-config/ssm-housekeeper/tests/ssm-housekeeper.tftest.hcl @@ -0,0 +1,263 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/ssm-housekeeper-test" + } + } + + mock_resource "aws_lambda_function" { + defaults = { + arn = "arn:aws:lambda:eu-west-1:123456789012:function:ssm-housekeeper-test" + } + } + + mock_resource "aws_cloudwatch_event_rule" { + defaults = { + arn = "arn:aws:events:eu-west-1:123456789012:rule/ssm-housekeeper-test" + } + } + + mock_resource "aws_cloudwatch_log_group" { + defaults = { + arn = "arn:aws:logs:eu-west-1:123456789012:log-group:/aws/lambda/ssm-housekeeper-test" + } + } +} + +variables { + config = { + prefix = "ssm-housekeeper-test" + aws_partition = "aws-us-gov" + schedule = { + expression = "rate(6 hours)" + state = "DISABLED" + } + cleanup = { + token_path = "/custom/runner/tokens" + parameter_path_arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/custom/runner/tokens*" + minimum_days_old = 7 + dry_run = true + } + lambda = { + artifact = { + zip = "unused-with-s3.zip" + s3 = { + bucket = "lambda-artifacts" + key = "control-plane/runners.zip" + object_version = "version-1" + } + } + runtime = "nodejs24.x" + architecture = "arm64" + memory_size = 384 + timeout = 45 + vpc = { + subnet_ids = [] + security_group_ids = [] + } + role = { + path = "/runner-config/" + permissions_boundary = null + principals = [{ + type = "AWS" + identifiers = ["arn:aws-us-gov:iam::123456789012:role/local-testing"] + }] + } + } + observability = { + logs = { + level = "debug" + retention_in_days = 30 + kms_key_id = null + class = "STANDARD" + } + tracing = { + mode = null + capture_http_requests = false + capture_error = false + } + } + tags = { + resources = { + Scope = "housekeeper" + } + lambda = { + Scope = "housekeeper" + Resource = "lambda" + } + log_group = { + Scope = "housekeeper" + Resource = "logs" + } + } + } +} + +run "configures_schedule_cleanup_and_outputs" { + command = plan + + assert { + condition = ( + length(data.aws_iam_policy_document.lambda_assume_role.statement[0].principals) == 2 && + contains(data.aws_iam_policy_document.lambda_assume_role.statement[0].principals[*].type, "AWS") + ) + error_message = "The housekeeper Lambda trust policy must include configured additional principals." + } + + assert { + condition = ( + aws_cloudwatch_event_rule.ssm_housekeeper.schedule_expression == "rate(6 hours)" && + aws_cloudwatch_event_rule.ssm_housekeeper.state == "DISABLED" + ) + error_message = "The housekeeper EventBridge rule must use the configured schedule and state." + } + + assert { + condition = ( + jsondecode(aws_lambda_function.ssm_housekeeper.environment[0].variables["SSM_CLEANUP_CONFIG"]).tokenPath == "/custom/runner/tokens" && + jsondecode(aws_lambda_function.ssm_housekeeper.environment[0].variables["SSM_CLEANUP_CONFIG"]).minimumDaysOld == 7 && + jsondecode(aws_lambda_function.ssm_housekeeper.environment[0].variables["SSM_CLEANUP_CONFIG"]).dryRun + ) + error_message = "The Lambda cleanup configuration must preserve the configured path override, age, and dry-run setting." + } + + assert { + condition = contains( + data.aws_iam_policy_document.ssm_housekeeper.statement[0].resources, + "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/custom/runner/tokens*", + ) + error_message = "The housekeeper IAM policy must authorize the same overridden Parameter Store path supplied to the Lambda." + } + + assert { + condition = toset(keys(output.housekeeper)) == toset(["lambda", "log_group", "role"]) + error_message = "The module must expose Lambda, log-group, and role resources through one nested housekeeper output." + } + + assert { + condition = ( + output.housekeeper.lambda.tags == tomap({ + Scope = "housekeeper" + Resource = "lambda" + }) && + output.housekeeper.log_group.tags == tomap({ + Scope = "housekeeper" + Resource = "logs" + }) && + output.housekeeper.role.tags == tomap({ + Scope = "housekeeper" + }) + ) + error_message = "Each nested output resource must retain its resolved component tags." + } + + assert { + condition = ( + length(aws_lambda_function.ssm_housekeeper.vpc_config) == 0 && + length(aws_iam_role_policy_attachment.ssm_housekeeper_vpc_execution_role) == 0 && + length(aws_lambda_function.ssm_housekeeper.tracing_config) == 0 && + length(aws_iam_role_policy.ssm_housekeeper_xray) == 0 + ) + error_message = "Empty VPC configuration and disabled tracing must not create their optional Lambda or IAM configuration." + } +} + +run "enables_vpc_and_xray_together" { + command = plan + + variables { + config = { + prefix = "ssm-housekeeper-vpc-test" + aws_partition = "aws-us-gov" + schedule = { + expression = "rate(1 day)" + state = "ENABLED" + } + cleanup = { + token_path = "/github-runner/tokens" + parameter_path_arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/tokens*" + minimum_days_old = 1 + dry_run = false + } + lambda = { + artifact = { + zip = "unused-with-s3.zip" + s3 = { + bucket = "lambda-artifacts" + key = "control-plane/runners.zip" + } + } + runtime = "nodejs24.x" + architecture = "arm64" + memory_size = 512 + timeout = 60 + vpc = { + subnet_ids = ["subnet-12345678"] + security_group_ids = ["sg-12345678"] + } + role = { + path = "/runner-config/" + permissions_boundary = null + } + } + observability = { + logs = { + level = "info" + retention_in_days = 14 + kms_key_id = null + class = "STANDARD" + } + tracing = { + mode = "Active" + capture_http_requests = true + capture_error = true + } + } + tags = { + resources = {} + lambda = {} + log_group = {} + } + } + } + + assert { + condition = ( + length(aws_lambda_function.ssm_housekeeper.vpc_config) == 1 && + aws_lambda_function.ssm_housekeeper.vpc_config[0].subnet_ids == toset(["subnet-12345678"]) && + aws_lambda_function.ssm_housekeeper.vpc_config[0].security_group_ids == toset(["sg-12345678"]) && + length(aws_iam_role_policy_attachment.ssm_housekeeper_vpc_execution_role) == 1 && + aws_iam_role_policy_attachment.ssm_housekeeper_vpc_execution_role[0].policy_arn == "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + ) + error_message = "A complete VPC configuration must configure the Lambda and attach the partition-aware VPC execution policy." + } + + assert { + condition = ( + length(aws_lambda_function.ssm_housekeeper.tracing_config) == 1 && + aws_lambda_function.ssm_housekeeper.tracing_config[0].mode == "Active" && + length(aws_iam_role_policy.ssm_housekeeper_xray) == 1 && + aws_lambda_function.ssm_housekeeper.environment[0].variables["POWERTOOLS_TRACE_ENABLED"] == "true" && + aws_lambda_function.ssm_housekeeper.environment[0].variables["POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS"] == "true" && + aws_lambda_function.ssm_housekeeper.environment[0].variables["POWERTOOLS_TRACER_CAPTURE_ERROR"] == "true" + ) + error_message = "Active tracing must configure Lambda tracing, X-Ray IAM permissions, and tracing-helper environment variables." + } + + assert { + condition = ( + data.aws_iam_policy_document.lambda_xray[0].statement[0].resources == toset(["*"]) + && alltrue([ + for action in data.aws_iam_policy_document.lambda_xray[0].statement[0].actions : + startswith(action, "xray:") + ]) + ) + error_message = "The housekeeper wildcard resource must be limited to X-Ray APIs, which do not support resource-level IAM permissions." + } +} diff --git a/modules/runner-config/ssm-housekeeper/variables.tf b/modules/runner-config/ssm-housekeeper/variables.tf new file mode 100644 index 0000000000..64848fc33c --- /dev/null +++ b/modules/runner-config/ssm-housekeeper/variables.tf @@ -0,0 +1,93 @@ +variable "config" { + description = <<-EOT + Provider-neutral SSM housekeeper configuration assembled by runner-config. + + - `prefix`: Prefix used to name the housekeeper resources. + - `aws_partition`: AWS partition used to construct IAM policy ARNs. + - `schedule.expression`: EventBridge schedule expression that invokes the housekeeper. + - `schedule.state`: State of the EventBridge rule. + - `cleanup.token_path`: Parameter Store token path supplied to the Lambda. + - `cleanup.parameter_path_arn`: IAM resource ARN matching `cleanup.token_path`. + - `cleanup.minimum_days_old`: Minimum parameter age before deletion. + - `cleanup.dry_run`: Reports eligible parameters without deleting them when true. + - `lambda.artifact.zip`: Resolved local control-plane archive. + - `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive. + - `lambda.artifact.s3.key`: Object key of the Lambda archive. + - `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive. + - `lambda.runtime`: Runtime used by the housekeeper Lambda. + - `lambda.architecture`: Instruction-set architecture used by the housekeeper Lambda. + - `lambda.memory_size`: Memory allocated to the housekeeper Lambda. + - `lambda.timeout`: Housekeeper Lambda timeout in seconds. + - `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration. + - `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration. + - `lambda.role.path`: IAM path used for the housekeeper Lambda role. + - `lambda.role.permissions_boundary`: Optional permissions boundary for the housekeeper role. + - `lambda.role.principals`: Additional principals allowed to assume the housekeeper Lambda role. + - `observability.logs`: Logging level, retention, encryption, and log-class configuration. + - `observability.tracing`: Lambda X-Ray and tracing-helper configuration. + - `tags.resources`: Tags for the housekeeper role and EventBridge rule. + - `tags.lambda`: Tags for the housekeeper Lambda function. + - `tags.log_group`: Tags for the housekeeper log group. + EOT + + type = object({ + prefix = string + aws_partition = string + schedule = object({ + expression = string + state = string + }) + cleanup = object({ + token_path = string + parameter_path_arn = string + minimum_days_old = number + dry_run = bool + }) + lambda = object({ + artifact = object({ + zip = string + s3 = object({ + bucket = optional(string, null) + key = optional(string, null) + object_version = optional(string, null) + }) + }) + runtime = string + architecture = string + memory_size = number + timeout = number + vpc = object({ + subnet_ids = list(string) + security_group_ids = list(string) + }) + role = object({ + path = string + permissions_boundary = optional(string, null) + principals = optional(list(object({ + type = string + identifiers = list(string) + })), []) + }) + }) + observability = object({ + logs = object({ + level = string + retention_in_days = number + kms_key_id = optional(string, null) + class = string + }) + tracing = object({ + mode = optional(string, null) + capture_http_requests = bool + capture_error = bool + }) + }) + tags = object({ + resources = map(string) + lambda = map(string) + log_group = map(string) + }) + }) + + nullable = false +} diff --git a/modules/runner-config/ssm-housekeeper/versions.tf b/modules/runner-config/ssm-housekeeper/versions.tf new file mode 100644 index 0000000000..da9769f550 --- /dev/null +++ b/modules/runner-config/ssm-housekeeper/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/runner-config/tests/README.md b/modules/runner-config/tests/README.md new file mode 100644 index 0000000000..fa55dfecd9 --- /dev/null +++ b/modules/runner-config/tests/README.md @@ -0,0 +1,72 @@ +# Terraform Tests + +This directory contains [Terraform test files](https://developer.hashicorp.com/terraform/language/tests) (`.tftest.hcl`) for the runners module. + +## Why `terraform test` instead of `terraform validate`? + +`terraform validate` only checks syntax and basic type correctness of the configuration. It **cannot** detect: + +- Conditional expressions with inconsistent result types (e.g., one branch returns an object with 1 attribute, the other returns 16) +- Runtime type mismatches that only surface during `plan` +- Invalid cross-module references that depend on resource attribute shapes + +`terraform test` with `mock_provider` runs a full plan without needing real cloud credentials, catching these classes of bugs in CI. + +## Requirements + +- Terraform >= 1.7 (for `mock_provider` and `mock_data` support) +- No AWS credentials required — all providers are mocked + +## Running locally + +```bash +cd modules/runners +terraform test -test-directory=tests +``` + +Expected output: + +``` +tests/pool.tftest.hcl... in progress + run "plan_with_pool_enabled"... pass +tests/pool.tftest.hcl... pass + +Success! 1 passed, 0 failed. +``` + +## Writing new tests + +1. Create a `.tftest.hcl` file in this directory +2. Use `mock_provider "aws" {}` to avoid needing credentials +3. Use `mock_data` blocks to provide realistic values for data sources that perform validation (e.g., `aws_iam_policy_document` validates JSON) +4. Set all required variables in a `variables {}` block +5. Use `run` blocks with `command = plan` and `assert` conditions + +### Example template + +```hcl +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + } + } +} + +variables { + # ... required variables ... +} + +run "descriptive_test_name" { + command = plan + + assert { + condition = + error_message = "Explanation of what failed" + } +} +``` + +## CI integration + +These tests run automatically in the `terraform_test` job of `.github/workflows/terraform.yml` on every PR that touches `*.tf` or `*.hcl` files. diff --git a/modules/runner-config/tests/computed-iam-inputs.tftest.hcl b/modules/runner-config/tests/computed-iam-inputs.tftest.hcl new file mode 100644 index 0000000000..9fcea735ed --- /dev/null +++ b/modules/runner-config/tests/computed-iam-inputs.tftest.hcl @@ -0,0 +1,35 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} + +run "computed_external_values_keep_plan_shape_known" { + command = plan + + module { + source = "./tests/fixtures/computed-iam-inputs" + } + + # The packaged runner archive is added by the release build, so the computed + # IAM fixture isolates the two common housekeeper children in a source checkout. + override_module { + target = module.external_iam.module.ssm_housekeeper + } + + override_module { + target = module.generated_policy.module.ssm_housekeeper + } + + assert { + condition = output.external_role_runner_count == 0 + error_message = "Computed external AMI parameter, KMS key, role, and profile values must not make resource or policy-block counts unknown." + } + + assert { + condition = output.generated_policy_role_runner_count == 1 + error_message = "A computed managed-policy ARN under a caller-known map key must keep attachment planning stable." + } +} diff --git a/modules/runner-config/tests/fixtures/computed-iam-inputs/README.md b/modules/runner-config/tests/fixtures/computed-iam-inputs/README.md new file mode 100644 index 0000000000..3bbf0f9027 --- /dev/null +++ b/modules/runner-config/tests/fixtures/computed-iam-inputs/README.md @@ -0,0 +1,39 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [external\_iam](#module\_external\_iam) | ../../.. | n/a | +| [generated\_policy](#module\_generated\_policy) | ../../.. | n/a | + +## Resources + +| Name | Type | +|------|------| +| [random_id.external](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [random_id.generated_policy](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [external\_role\_runner\_count](#output\_external\_role\_runner\_count) | n/a | +| [generated\_policy\_role\_runner\_count](#output\_generated\_policy\_role\_runner\_count) | n/a | + diff --git a/modules/runner-config/tests/fixtures/computed-iam-inputs/computed-iam-inputs.tf b/modules/runner-config/tests/fixtures/computed-iam-inputs/computed-iam-inputs.tf new file mode 100644 index 0000000000..e1b763f8bd --- /dev/null +++ b/modules/runner-config/tests/fixtures/computed-iam-inputs/computed-iam-inputs.tf @@ -0,0 +1,214 @@ +# A .tftest.hcl variable block supplies plan-known values. This wrapper uses +# random_id results to exercise caller inputs that remain unknown during plan, +# which catches invalid count, for_each, and dynamic-block expressions in the +# IAM boundary. +resource "random_id" "external" { + byte_length = 4 +} + +resource "random_id" "generated_policy" { + byte_length = 4 +} + +module "external_iam" { + source = "../../.." + + aws_region = "eu-west-1" + prefix = "computed-external" + + compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/external-ami-${random_id.external.hex}" + } + kms_key = { + arn = "arn:aws:kms:eu-west-1:123456789012:key/${random_id.external.hex}" + } + } + instance_profile = { + name = "external-runner-${random_id.external.hex}" + } + cloudwatch_agent = { + enabled = false + } + binaries_syncer = { + enabled = false + } + } + } + } + + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external-runner-${random_id.external.hex}" + } + } + } + + lambda = { + artifact = { + s3 = { + bucket = "lambda-artifacts" + } + } + } + + github = { + app_parameters = { + key_base64 = [{ + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + }] + id = [{ + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + }] + installation_id = [null] + } + } + + orchestration_provider = { + webhook = { + runner = { + maximum_count = 3 + } + github = { + organization_runners = true + } + queue = { + build = { + arn = "arn:aws:sqs:eu-west-1:123456789012:computed-external" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/computed-external" + } + kms_key_id = "arn:aws:kms:eu-west-1:123456789012:key/build-queue-${random_id.external.hex}" + } + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + pool = { + runner_owner = "example" + config = [{ + schedule_expression = "cron(0 8 * * ? *)" + size = 1 + }] + } + } + job_retry = { + enabled = true + } + } + } + + ssm = { + kms_key_id = "arn:aws:kms:eu-west-1:123456789012:key/${random_id.external.hex}" + paths = { + root = "/github-runner/computed-external" + tokens = "tokens" + config = "config" + } + } +} + +module "generated_policy" { + source = "../../.." + + aws_region = "eu-west-1" + prefix = "computed-policy" + + compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + cloudwatch_agent = { + enabled = false + } + binaries_syncer = { + enabled = false + } + } + } + } + + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + managed_policy_arns = { + generated = "arn:aws:iam::123456789012:policy/generated-runner-${random_id.generated_policy.hex}" + } + } + } + + lambda = { + artifact = { + s3 = { + bucket = "lambda-artifacts" + } + } + } + + github = { + app_parameters = { + key_base64 = [{ + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + }] + id = [{ + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + }] + installation_id = [null] + } + } + + orchestration_provider = { + webhook = { + runner = { + maximum_count = 3 + } + github = { + organization_runners = true + } + queue = { + build = { + arn = "arn:aws:sqs:eu-west-1:123456789012:computed-policy" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/computed-policy" + } + } + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + } + } + } + + ssm = { + paths = { + root = "/github-runner/computed-policy" + tokens = "tokens" + config = "config" + } + } +} + +output "external_role_runner_count" { + value = module.external_iam.runner.role == null ? 0 : 1 +} + +output "generated_policy_role_runner_count" { + value = module.generated_policy.runner.role == null ? 0 : 1 +} diff --git a/modules/runner-config/tests/fixtures/computed-iam-inputs/versions.tf b/modules/runner-config/tests/fixtures/computed-iam-inputs/versions.tf new file mode 100644 index 0000000000..9fd85fad8f --- /dev/null +++ b/modules/runner-config/tests/fixtures/computed-iam-inputs/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.3" + + required_providers { + aws = { + source = "hashicorp/aws" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/modules/runner-config/tests/pool.tftest.hcl b/modules/runner-config/tests/pool.tftest.hcl new file mode 100644 index 0000000000..12a81854c3 --- /dev/null +++ b/modules/runner-config/tests/pool.tftest.hcl @@ -0,0 +1,652 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/runner-test" + } + } + + mock_resource "aws_ssm_parameter" { + defaults = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/ami-id" + } + } +} + +# The runner archive is injected during packaging, so isolate the common +# housekeeper child in source-checkout tests where that build artifact is absent. +override_module { + target = module.ssm_housekeeper +} + +variables { + aws_region = "eu-west-1" + + compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/external-ami-id" + } + kms_key = null + } + binaries_syncer = { + s3 = { + arn = "arn:aws:s3:::my-bucket" + id = "my-bucket" + key = "runners/linux/actions-runner.tar.gz" + } + } + ssm_enabled = true + } + } + } + + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + managed_policy_arns = { + readonly = "arn:aws:iam::aws:policy/ReadOnlyAccess" + } + additional_trust_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "AdditionalTrustedAccount" + Effect = "Allow" + Action = "sts:AssumeRole" + Principal = { AWS = "arn:aws:iam::210987654321:root" } + }] + }) + } + } + + lambda = { + artifact = { + s3 = { + bucket = "my-lambda-bucket" + } + } + } + + github = { + app_parameters = { + key_base64 = [{ name = "/github-runner/key-base64", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" }] + id = [{ name = "/github-runner/app-id", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" }] + installation_id = [null] + } + } + + orchestration_provider = { + webhook = { + runner = { + boot_time_in_minutes = 8 + ephemeral = true + jit_config_enabled = null + maximum_count = 9 + } + github = { + organization_runners = true + } + queue = { + build = { + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + } + } + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + pool = { + config = [{ + schedule_expression = "cron(0 8 * * ? *)" + size = 1 + }] + } + } + } + } + + ssm = { + paths = { + root = "/github-runner" + tokens = "tokens" + config = "config" + } + } + +} + +run "plan_with_pool_enabled" { + command = plan + + assert { + condition = module.orchestration_webhook[0].pool != null + error_message = "Pool module should be enabled when pool.config is non-empty" + } + + assert { + condition = ( + !contains(keys(var.runner), "maximum_count") + && !contains(keys(var.runner), "boot_time_in_minutes") + && !contains(keys(var.runner), "ephemeral") + && !contains(keys(var.runner), "jit_config_enabled") + && var.orchestration_provider.webhook.runner.boot_time_in_minutes == 8 + && var.orchestration_provider.webhook.runner.ephemeral + && var.orchestration_provider.webhook.runner.jit_config_enabled == null + && var.orchestration_provider.webhook.runner.maximum_count == 9 + && module.orchestration_webhook[0].scale_up.lambda.environment[0].variables["RUNNERS_MAXIMUM_COUNT"] == "9" + && module.orchestration_webhook[0].scale_down.lambda.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "8" + && module.orchestration_webhook[0].pool.lambda.environment[0].variables["RUNNERS_MAXIMUM_COUNT"] == "9" + && module.orchestration_webhook[0].pool.lambda.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "8" + ) + error_message = "Runner capacity and boot time must be owned by orchestration_provider.webhook.runner and routed to webhook controls, not retained in the common runner contract." + } + + assert { + condition = ( + aws_ssm_parameter.runner_agent_mode.value == "ephemeral" + && aws_ssm_parameter.jit_config_enabled.value == "true" + ) + error_message = "Runner-config must serialize the webhook provider's resolved lifecycle contract without duplicating its JIT fallback." + } + + assert { + condition = ( + module.orchestration_webhook[0].scale_up.lambda.s3_bucket == "my-lambda-bucket" + && module.orchestration_webhook[0].scale_up.lambda.s3_key == "runners.zip" + && local.ssm_housekeeper_artifact.s3.bucket == null + && endswith(local.ssm_housekeeper_artifact.zip, "/lambdas/functions/control-plane/runners.zip") + ) + error_message = "The webhook provider must combine its artifact key with the shared bucket while the common SSM housekeeper remains on the packaged archive." + } + + assert { + condition = ( + toset(keys(output.provider)) == toset(["aws"]) + && toset(keys(output.provider.aws)) == toset(["ec2"]) + ) + error_message = "The runner configuration must expose resources under the selected provider namespace and type." + } + + assert { + condition = contains(keys(output.provider.aws.ec2), "launch_template") + error_message = "The runner configuration must expose EC2 resources only under provider.aws.ec2." + } + + assert { + condition = length(aws_iam_role.runner) == 1 && output.runner.role != null + error_message = "The common runner configuration must create and expose the runner role." + } + + assert { + condition = ( + length(module.compute_aws_ec2_trust_policy) == 1 + && aws_iam_role.runner[0].assume_role_policy == module.compute_aws_ec2_trust_policy[0].assume_role_policy + ) + error_message = "The common runner role must use the selected EC2 trust-policy submodule output." + } + + assert { + condition = ( + output.pool != null + && toset(keys(output.pool)) == toset(["lambda", "log_group", "role"]) + ) + error_message = "An enabled pool must expose its Lambda, log group, and role through the nested pool output." + } + + assert { + condition = ( + toset(keys(output.orchestration_provider)) == toset(["webhook"]) + && output.orchestration_provider.webhook != null + && output.orchestration_provider.webhook.scale_up != null + && output.orchestration_provider.webhook.scale_down != null + && output.orchestration_provider.webhook.pool != null + ) + error_message = "The canonical orchestration output must group the existing webhook control-plane resources while flat aliases remain available." + } + + assert { + condition = length(jsondecode(module.orchestration_webhook[0].scale_up.lambda.environment[0].variables["SSM_PARAMETER_STORE_TAGS"])) == 0 + error_message = "Runtime Parameter Store tags must remain empty when no module or SSM tags are configured; EC2 bootstrap tags must not leak into them." + } + + assert { + condition = !contains(keys(output.provider.aws.ec2), "role_runner") + error_message = "The common runner role must not be duplicated in the EC2 resource output." + } + + assert { + condition = toset(keys(aws_iam_role_policy.runner_provider)) == toset([ + "ssm_parameters", + "describe_tags", + "create_tags", + "terminate_self", + "session_manager", + "distribution_bucket", + "cloudwatch", + ]) + error_message = "The common runner configuration must attach every enabled EC2 runner policy by its stable provider key." + } + + assert { + condition = aws_iam_role_policy_attachment.runner["user-readonly"].policy_arn == "arn:aws:iam::aws:policy/ReadOnlyAccess" + error_message = "The selected EC2 provider contract must return common managed runner policies for one attachment path." + } + + assert { + condition = ( + module.orchestration_webhook[0].scale_up.lambda.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "ec2" + && module.orchestration_webhook[0].scale_down.lambda.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "ec2" + ) + error_message = "Scaling Lambdas must receive the provider type from the selected provider." + } + + assert { + condition = module.orchestration_webhook[0].scale_up.lambda.environment[0].variables["INSTANCE_TYPES"] == "m5.large" + error_message = "Scale-up must merge the EC2 environment fragment." + } + + assert { + condition = module.orchestration_webhook[0].scale_down.lambda.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "8" + error_message = "Scale-down must receive boot time from the webhook orchestration configuration." + } + + assert { + condition = ( + toset(keys(module.orchestration_webhook[0].scale_up)) == toset(["lambda", "log_group", "role"]) + && toset(keys(module.orchestration_webhook[0].scale_down)) == toset(["lambda", "log_group", "role"]) + ) + error_message = "The scale-runners child module must forward the nested scale-up and scale-down resource contracts." + } + +} + +run "housekeeper_uses_component_s3_artifact" { + command = plan + + variables { + ssm = { + paths = { + root = "/github-runner" + tokens = "tokens" + config = "config" + } + housekeeper = { + lambda = { + artifact = { + s3 = { + key = "housekeeper/runner-config.zip" + object_version = "housekeeper-version" + } + } + } + } + } + } + + assert { + condition = ( + local.ssm_housekeeper_artifact.zip == null + && local.ssm_housekeeper_artifact.s3.bucket == "my-lambda-bucket" + && local.ssm_housekeeper_artifact.s3.key == "housekeeper/runner-config.zip" + && local.ssm_housekeeper_artifact.s3.object_version == "housekeeper-version" + ) + error_message = "The SSM housekeeper must combine its component-owned S3 key and version with the common Lambda artifact bucket." + } +} + +run "housekeeper_uses_component_local_zip" { + command = plan + + variables { + ssm = { + paths = { + root = "/github-runner" + tokens = "tokens" + config = "config" + } + housekeeper = { + lambda = { + artifact = { + zip = "README.md" + } + } + } + } + } + + assert { + condition = ( + local.ssm_housekeeper_artifact.zip == "README.md" + && local.ssm_housekeeper_artifact.s3.bucket == null + && local.ssm_housekeeper_artifact.s3.key == null + && local.ssm_housekeeper_artifact.s3.object_version == null + ) + error_message = "The SSM housekeeper must use its component-owned local zip without inheriting the common bucket or webhook artifact." + } +} + +run "rejects_conflicting_housekeeper_artifacts" { + command = plan + + variables { + ssm = { + paths = { + root = "/github-runner" + tokens = "tokens" + config = "config" + } + housekeeper = { + lambda = { + artifact = { + zip = "README.md" + s3 = { + key = "housekeeper/runner-config.zip" + } + } + } + } + } + } + + plan_options { + target = [terraform_data.validate_config] + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_housekeeper_s3_without_common_bucket" { + command = plan + + variables { + lambda = { + artifact = { + s3 = { + bucket = null + } + } + } + ssm = { + paths = { + root = "/github-runner" + tokens = "tokens" + config = "config" + } + housekeeper = { + lambda = { + artifact = { + s3 = { + key = "housekeeper/runner-config.zip" + } + } + } + } + } + } + + plan_options { + target = [terraform_data.validate_config] + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_missing_orchestration_provider" { + command = plan + + variables { + orchestration_provider = { + webhook = null + } + } + + plan_options { + target = [terraform_data.validate_config] + } + + expect_failures = [terraform_data.validate_config] +} + +run "external_runner_role_is_not_managed_by_common" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external/runner-role" + } + } + } + } + + assert { + condition = length(aws_iam_role.runner) == 0 && length(aws_iam_role_policy.runner_provider) == 0 && length(aws_iam_role_policy_attachment.runner) == 0 + error_message = "An external runner role must remain unmanaged by the common runner configuration." + } + + assert { + condition = output.runner.role == null + error_message = "The nested runner role output must be null when an external role is selected." + } + + + assert { + condition = output.provider.aws.ec2.launch_template.iam_instance_profile[0].name == "github-actions-runner-profile" + error_message = "EC2 must create an instance profile around an externally supplied runner role when no profile override is provided." + } +} + +run "external_runner_role_and_profile_remain_external" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external/runner-role" + } + } + } + compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + instance_profile = { + name = "external-runner-profile" + } + binaries_syncer = { + enabled = false + } + } + } + } + } + + assert { + condition = length(aws_iam_role.runner) == 0 && length(aws_iam_role_policy.runner_provider) == 0 + error_message = "The common runner configuration must not manage an external role." + } + + assert { + condition = output.provider.aws.ec2.launch_template.iam_instance_profile[0].name == "external-runner-profile" + error_message = "The EC2 launch template must use the external instance profile." + } +} + +run "empty_runner_iam_uses_common_role" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = {} + } + } + + assert { + condition = length(aws_iam_role.runner) == 1 + error_message = "An empty runner.iam object must use common role ownership." + } +} + +run "external_role_rejects_managed_policy_attachments" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external/runner-role" + } + managed_policy_arns = { + readonly = "arn:aws:iam::aws:policy/ReadOnlyAccess" + } + } + } + } + + plan_options { + target = [terraform_data.validate_config] + } + + expect_failures = [terraform_data.validate_config] +} + +run "external_role_rejects_trust_policy_extension" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external/runner-role" + } + additional_trust_policy_json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + } + + plan_options { + target = [terraform_data.validate_config] + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_invalid_trust_policy_extension" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + additional_trust_policy_json = "{" + } + } + } + + plan_options { + target = [terraform_data.validate_config] + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_empty_compute_provider" { + command = plan + + variables { + compute_provider = {} + } + + plan_options { + target = [terraform_data.validate_config] + } + + expect_failures = [terraform_data.validate_config] +} + +run "rejects_empty_aws_compute_provider_namespace" { + command = plan + + variables { + compute_provider = { + aws = {} + } + } + + plan_options { + target = [terraform_data.validate_config] + } + + expect_failures = [terraform_data.validate_config] +} + +run "job_retry_uses_common_runner_configuration_identity" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + name_prefix = "provider-neutral-" + } + orchestration_provider = { + webhook = { + github = { + organization_runners = true + } + queue = { + build = { + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + } + } + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + } + job_retry = { + enabled = true + lambda = { + reserved_concurrent_executions = 2 + } + } + } + } + } + + assert { + condition = module.orchestration_webhook[0].job_retry.lambda.function.environment[0].variables["RUNNER_NAME_PREFIX"] == "provider-neutral-" + error_message = "Job retry must receive the common runner-configuration name prefix." + } + + assert { + condition = module.orchestration_webhook[0].job_retry.lambda.function.reserved_concurrent_executions == 2 + error_message = "Job retry must apply its configured Lambda reserved concurrency." + } +} diff --git a/modules/runner-config/tests/tags.tftest.hcl b/modules/runner-config/tests/tags.tftest.hcl new file mode 100644 index 0000000000..6c004879ba --- /dev/null +++ b/modules/runner-config/tests/tags.tftest.hcl @@ -0,0 +1,322 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/runner-test" + } + } + + mock_resource "aws_ssm_parameter" { + defaults = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/config" + } + } +} + +# The runner archive is injected during packaging, so model the common +# housekeeper output while testing parent-level tag composition from source. +override_module { + target = module.ssm_housekeeper +} + +variables { + aws_region = "eu-west-1" + + tags = { + precedence = "module" + module = "yes" + } + + compute_provider = { + aws = { + ec2 = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/external-ami-id" + } + kms_key = null + } + binaries_syncer = { + s3 = { + arn = "arn:aws:s3:::my-bucket" + id = "my-bucket" + key = "runners/linux/actions-runner.tar.gz" + } + } + } + } + } + + runner = { + labels = ["self-hosted", "linux", "x64"] + tags = { + precedence = "runner" + runner = "yes" + } + } + + lambda = { + artifact = { + s3 = { + bucket = "my-lambda-bucket" + } + } + tags = { + precedence = "lambda" + lambda = "yes" + } + } + + github = { + app_parameters = { + key_base64 = [{ + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + }] + id = [{ + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + }] + installation_id = [null] + } + } + + orchestration_provider = { + webhook = { + github = { + organization_runners = true + } + queue = { + build = { + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + } + tags = { + precedence = "queue" + queue = "yes" + } + } + lambda = { + artifact = { + s3 = { + key = "runners.zip" + } + } + scale = { + up = { + tags = { + precedence = "scale-up" + scale_up = "yes" + } + } + down = { + tags = { + precedence = "scale-down" + scale_down = "yes" + } + } + } + pool = { + config = [{ + schedule_expression = "cron(0 8 * * ? *)" + size = 1 + }] + tags = { + precedence = "pool" + pool = "yes" + } + } + } + job_retry = { + enabled = true + tags = { + precedence = "job-retry" + job_retry = "yes" + } + } + } + } + + ssm = { + paths = { + root = "/github-runner" + tokens = "tokens" + config = "config" + } + tags = { + precedence = "ssm" + ssm = "yes" + } + parameters = { + tags = { + precedence = "ssm-parameter" + parameter = "yes" + } + } + housekeeper = { + tags = { + precedence = "ssm-housekeeper" + housekeeper = "yes" + } + } + } + + observability = { + logs = { + level = "debug" + tags = { + precedence = "log" + log = "yes" + } + } + } +} + +run "layered_component_tags" { + command = plan + + assert { + condition = module.orchestration_webhook[0].scale_up.lambda.environment[0].variables["LOG_LEVEL"] == "DEBUG" + error_message = "The nested observability.logs.level value must configure the control-plane functions." + } + + assert { + condition = module.orchestration_webhook[0].scale_up.lambda.tags == tomap({ + precedence = "scale-up" + module = "yes" + lambda = "yes" + scale_up = "yes" + }) && module.orchestration_webhook[0].scale_up.log_group.tags == tomap({ + precedence = "scale-up" + module = "yes" + log = "yes" + scale_up = "yes" + }) && module.orchestration_webhook[0].scale_up.role.tags == tomap({ + precedence = "scale-up" + module = "yes" + scale_up = "yes" + }) + error_message = "Scale-up tags must layer module, shared resource, and component tags with the component taking precedence." + } + + assert { + condition = module.orchestration_webhook[0].scale_down.lambda.tags == tomap({ + precedence = "scale-down" + module = "yes" + lambda = "yes" + scale_down = "yes" + }) && module.orchestration_webhook[0].scale_down.log_group.tags == tomap({ + precedence = "scale-down" + module = "yes" + log = "yes" + scale_down = "yes" + }) && module.orchestration_webhook[0].scale_down.role.tags == tomap({ + precedence = "scale-down" + module = "yes" + scale_down = "yes" + }) + error_message = "Scale-down tags must layer module, shared resource, and component tags with the component taking precedence." + } + + assert { + condition = aws_iam_role.runner[0].tags == tomap({ + precedence = "runner" + module = "yes" + runner = "yes" + }) + error_message = "Runner tags must override module tags on the common runner role." + } + + assert { + condition = aws_ssm_parameter.runner_agent_mode.tags == tomap({ + precedence = "ssm-parameter" + module = "yes" + ssm = "yes" + parameter = "yes" + }) && tomap({ + for tag in jsondecode(module.orchestration_webhook[0].scale_up.lambda.environment[0].variables["SSM_PARAMETER_STORE_TAGS"]) : + tag.Key => tag.Value + }) == tomap({ + precedence = "ssm-parameter" + module = "yes" + ssm = "yes" + parameter = "yes" + }) + error_message = "Terraform-managed and runtime-created SSM parameters must use the same layered parameter tags." + } + + assert { + condition = local.ssm_housekeeper_lambda_tags == tomap({ + precedence = "ssm-housekeeper" + module = "yes" + lambda = "yes" + ssm = "yes" + housekeeper = "yes" + }) && local.ssm_housekeeper_log_tags == tomap({ + precedence = "ssm-housekeeper" + module = "yes" + log = "yes" + ssm = "yes" + housekeeper = "yes" + }) && local.ssm_housekeeper_tags == tomap({ + precedence = "ssm-housekeeper" + module = "yes" + ssm = "yes" + housekeeper = "yes" + }) + error_message = "SSM housekeeper tags must layer module, SSM, shared resource, and housekeeper tags." + } + + assert { + condition = module.orchestration_webhook[0].pool.lambda.tags == tomap({ + precedence = "pool" + module = "yes" + lambda = "yes" + pool = "yes" + }) && module.orchestration_webhook[0].pool.log_group.tags == tomap({ + precedence = "pool" + module = "yes" + log = "yes" + pool = "yes" + }) && module.orchestration_webhook[0].pool.role.tags == tomap({ + precedence = "pool" + module = "yes" + pool = "yes" + }) + error_message = "Pool tags must layer module, shared resource, and component tags with the component taking precedence." + } + + assert { + condition = module.orchestration_webhook[0].job_retry.lambda.function.tags == tomap({ + precedence = "job-retry" + module = "yes" + lambda = "yes" + job_retry = "yes" + }) && module.orchestration_webhook[0].job_retry.lambda.log_group.tags == tomap({ + precedence = "job-retry" + module = "yes" + log = "yes" + job_retry = "yes" + }) && module.orchestration_webhook[0].job_retry.lambda.role.tags == tomap({ + precedence = "job-retry" + module = "yes" + job_retry = "yes" + }) && module.orchestration_webhook[0].job_retry.queue.tags == tomap({ + precedence = "job-retry" + module = "yes" + queue = "yes" + job_retry = "yes" + }) + error_message = "Job-retry tags must layer module, shared resource, and component tags with the component taking precedence." + } +} diff --git a/modules/runner-config/validations.tf b/modules/runner-config/validations.tf new file mode 100644 index 0000000000..f510bf0422 --- /dev/null +++ b/modules/runner-config/validations.tf @@ -0,0 +1,119 @@ +resource "terraform_data" "validate_config" { + lifecycle { + precondition { + condition = contains(["linux", "osx", "windows"], var.runner.os) + error_message = "Valid values for runner.os are linux, osx, and windows." + } + + precondition { + condition = length(var.runner.name_prefix) <= 45 + error_message = "runner.name_prefix must be at most 45 characters." + } + + precondition { + condition = var.runner.iam.role == null ? true : trimspace(var.runner.iam.role.arn) != "" + error_message = "runner.iam.role.arn must be a non-empty ARN when set." + } + + precondition { + condition = var.runner.iam.role == null || length(var.runner.iam.managed_policy_arns) == 0 + error_message = "runner.iam.managed_policy_arns cannot be set with an external runner.iam.role because external roles are not managed by this module." + } + + precondition { + condition = var.runner.iam.additional_trust_policy_json == null ? true : can(jsondecode(var.runner.iam.additional_trust_policy_json)) + error_message = "runner.iam.additional_trust_policy_json must be valid JSON when set." + } + + precondition { + condition = var.runner.iam.role == null || var.runner.iam.additional_trust_policy_json == null + error_message = "runner.iam.additional_trust_policy_json cannot be set with an external runner.iam.role because external role trust is not managed by this module." + } + + precondition { + condition = contains(["arm64", "x86_64"], var.lambda.architecture) + error_message = "lambda.architecture must be arm64 or x86_64." + } + + precondition { + condition = !( + var.ssm.housekeeper.lambda.artifact.zip != null && + var.ssm.housekeeper.lambda.artifact.s3 != null + ) + error_message = "ssm.housekeeper.lambda.artifact must select at most one of zip or s3." + } + + precondition { + condition = ( + var.ssm.housekeeper.lambda.artifact.s3 == null || + var.lambda.artifact.s3.bucket != null + ) + error_message = "lambda.artifact.s3.bucket must be set when ssm.housekeeper.lambda.artifact.s3 is selected." + } + + precondition { + condition = contains(["STANDARD", "INFREQUENT_ACCESS"], var.observability.logs.class) + error_message = "observability.logs.class must be STANDARD or INFREQUENT_ACCESS." + } + + precondition { + condition = contains([ + "silly", + "trace", + "debug", + "info", + "warn", + "error", + "fatal", + ], var.observability.logs.level) + error_message = "observability.logs.level must be one of silly, trace, debug, info, warn, error, or fatal." + } + + precondition { + condition = length([ + for provider_key, provider_config in local.compute_providers : provider_key + if provider_config != null + ]) == 1 + error_message = "Exactly one compute-provider block must be set. Supported compute-provider blocks: aws.ec2." + } + + precondition { + condition = var.compute_provider_key == null || try( + local.compute_providers[var.compute_provider_key] != null, + false, + ) + error_message = "compute_provider_key must identify the non-null typed compute-provider block." + } + + precondition { + condition = length([ + for provider_name, provider_config in var.orchestration_provider : provider_name + if provider_config != null + ]) == 1 + error_message = "Exactly one orchestration provider must be configured. Supported providers: webhook." + } + + precondition { + condition = var.orchestration_provider.webhook == null ? true : ( + var.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.batch_size >= 1 && + var.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.batch_size <= 1000 && + var.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.maximum_batching_window_in_seconds >= 0 && + var.orchestration_provider.webhook.lambda.scale.up.event_source_mapping.maximum_batching_window_in_seconds <= 300 + ) + error_message = "orchestration_provider.webhook.lambda.scale.up.event_source_mapping batch size must be between 1 and 1000 and its batching window between 0 and 300 seconds." + } + + precondition { + condition = var.orchestration_provider.webhook == null ? true : !( + var.orchestration_provider.webhook.lambda.artifact.zip != null && + var.orchestration_provider.webhook.lambda.artifact.s3 != null + ) + error_message = "orchestration_provider.webhook.lambda.artifact must select at most one of zip or s3." + } + + precondition { + condition = var.orchestration_provider.webhook == null ? true : (!var.orchestration_provider.webhook.job_retry.enabled || var.orchestration_provider.webhook.job_retry.delay_in_seconds <= 900) + error_message = "orchestration_provider.webhook.job_retry.delay_in_seconds cannot exceed the SQS maximum of 900 seconds." + } + } +} diff --git a/modules/runner-config/variables.compute-provider.tf b/modules/runner-config/variables.compute-provider.tf new file mode 100644 index 0000000000..87c85b7c05 --- /dev/null +++ b/modules/runner-config/variables.compute-provider.tf @@ -0,0 +1,263 @@ +# Optional plan-known dispatch key supplied by the multi-runner topology layer. +variable "compute_provider_key" { + description = "Optional plan-known compute-provider dispatch key. Null discovers the key from the exactly one populated compute_provider block." + type = string + default = null + + validation { + condition = var.compute_provider_key == null ? true : contains(["aws_ec2"], var.compute_provider_key) + error_message = "compute_provider_key must be null or aws_ec2." + } +} + +# Typed compute-provider input boundary between the common control plane and compute implementations. +variable "compute_provider" { + description = <<-EOT + Typed compute-provider configuration. Provider-owned settings remain inside the selected compute-provider block. + + Exactly one compute-provider block must be non-null. The populated block selects the provider, and its presence must be known during planning. Values inside the selected block may remain unknown until apply. + + - `aws`: AWS compute-provider configurations. + - `aws.ec2`: EC2 compute-provider configuration. + - `aws.ec2.ami`: Optional AMI discovery or external AMI-parameter configuration. Null uses the operating-system and architecture defaults. + - `aws.ec2.ami.filter`: EC2 AMI filters combined with the provider's default AMI-name filter. + - `aws.ec2.ami.owners`: AWS account IDs or aliases allowed to own the selected AMI. + - `aws.ec2.ami.id_ssm_parameter`: Optional externally managed SSM parameter containing the AMI ID. Null creates a provider-managed AMI-ID parameter. The wrapper's presence is the plan-time ownership discriminator, so keep the object literal even when its ARN comes from another resource. + - `aws.ec2.ami.id_ssm_parameter.arn`: ARN of the externally managed SSM parameter. The ARN may be unknown until apply. + - `aws.ec2.ami.kms_key`: Optional KMS key required to launch encrypted AMIs or snapshots. The wrapper's presence is the plan-time policy discriminator. + - `aws.ec2.ami.kms_key.arn`: ARN of the KMS key. The ARN may be unknown until apply. + - `aws.ec2.vpc_id`: VPC in which runner networking resources are created. + - `aws.ec2.subnet_ids`: Subnets from which scale-up may launch runner instances. + - `aws.ec2.overrides`: Optional resource-name overrides. + - `aws.ec2.overrides.name_runner`: Name tag used for runner compute resources. An empty value uses the generated provider name. + - `aws.ec2.overrides.name_sg`: Name tag used for the managed runner security group. An empty value uses the generated provider name. + - `aws.ec2.instance_profile`: Optional externally managed instance profile used by the launch template. + - `aws.ec2.instance_profile.name`: Name of the externally managed instance profile. + - `aws.ec2.instance_profile_path`: IAM path for the provider-managed instance profile. Null uses a path derived from the runner-configuration prefix. + - `aws.ec2.binaries_syncer`: Runner-distribution synchronization configuration. + - `aws.ec2.binaries_syncer.enabled`: Enables use of a synchronized runner distribution from S3. + - `aws.ec2.binaries_syncer.s3`: S3 object containing the synchronized runner distribution. Required when synchronization is enabled. + - `aws.ec2.binaries_syncer.s3.arn`: ARN of the runner-distribution bucket, used by IAM policies. + - `aws.ec2.binaries_syncer.s3.id`: Bucket name used to construct the runner-distribution S3 URI. + - `aws.ec2.binaries_syncer.s3.key`: Object key of the runner distribution. + - `aws.ec2.block_device_mappings`: EBS mappings added to the runner launch template. + - `aws.ec2.block_device_mappings[].delete_on_termination`: Deletes the volume when its runner instance terminates. + - `aws.ec2.block_device_mappings[].device_name`: Device name exposed to the runner instance. + - `aws.ec2.block_device_mappings[].encrypted`: Enables EBS encryption. + - `aws.ec2.block_device_mappings[].iops`: Provisioned IOPS for volume types that support it. + - `aws.ec2.block_device_mappings[].kms_key_id`: KMS key ID or ARN used to encrypt the volume. + - `aws.ec2.block_device_mappings[].snapshot_id`: Snapshot used to initialize the volume. + - `aws.ec2.block_device_mappings[].throughput`: Provisioned throughput for volume types that support it. + - `aws.ec2.block_device_mappings[].volume_initialization_rate`: Fixed initialization rate in MiB/s for supported snapshot-backed volumes. + - `aws.ec2.block_device_mappings[].volume_size`: Volume size in GiB. + - `aws.ec2.block_device_mappings[].volume_type`: EBS volume type. + - `aws.ec2.ebs_optimized`: Requests EBS-optimized runner instances. + - `aws.ec2.instance_target_capacity_type`: Primary capacity type, either `spot` or `on-demand`. + - `aws.ec2.instance_allocation_strategy`: EC2 Fleet allocation strategy used to select instance capacity. + - `aws.ec2.instance_type_priorities`: Optional numeric priorities keyed by instance type. + - `aws.ec2.instance_max_spot_price`: Optional maximum hourly Spot price. + - `aws.ec2.instance_types`: EC2 instance types available to the scale-up and pool functions. + - `aws.ec2.user_data`: Runner bootstrap user-data configuration. + - `aws.ec2.user_data.enabled`: Enables launch-template user data. + - `aws.ec2.user_data.template`: Optional path to a custom user-data template. + - `aws.ec2.user_data.content`: Optional complete user-data content. When set, it is used instead of rendering a template. + - `aws.ec2.user_data.pre_install`: Script content inserted before runner installation in the default template. + - `aws.ec2.user_data.post_install`: Script content inserted after runner installation in the default template. + - `aws.ec2.user_data.debug_logging_enabled`: Enables verbose user-data tracing, which can expose secrets in logs. + - `aws.ec2.ssm_enabled`: Attaches runner permissions and policies required for AWS Systems Manager access. + - `aws.ec2.create_service_linked_role_spot`: Allows scale-up to create the EC2 Spot service-linked role. + - `aws.ec2.cloudwatch_agent`: CloudWatch agent configuration for runner instances. + - `aws.ec2.cloudwatch_agent.enabled`: Installs and configures the CloudWatch agent through the default bootstrap flow. + - `aws.ec2.cloudwatch_agent.config`: Optional complete CloudWatch agent configuration. Null renders the provider default from `log_files`. + - `aws.ec2.managed_security_group_enabled`: Creates and attaches the provider-managed runner security group. + - `aws.ec2.log_files`: Optional log files collected by the CloudWatch agent. Null uses the provider defaults. + - `aws.ec2.log_files[].log_group_name`: CloudWatch log-group name, before optional prefixing. + - `aws.ec2.log_files[].prefix_log_group`: Prefixes the log-group name with the runner configuration path when true. + - `aws.ec2.log_files[].file_path`: File or glob read by the CloudWatch agent. + - `aws.ec2.log_files[].log_stream_name`: CloudWatch log-stream name template. + - `aws.ec2.log_files[].log_class`: CloudWatch log-group class for the collected file. + - `aws.ec2.key_name`: Optional EC2 key-pair name added to the launch template. + - `aws.ec2.additional_security_group_ids`: Existing security groups attached in addition to the managed security group. + - `aws.ec2.detailed_monitoring_enabled`: Enables detailed EC2 monitoring for runner instances. + - `aws.ec2.egress_rules`: Egress rules created on the managed runner security group. + - `aws.ec2.egress_rules[].cidr_blocks`: IPv4 CIDR destinations. + - `aws.ec2.egress_rules[].ipv6_cidr_blocks`: IPv6 CIDR destinations. + - `aws.ec2.egress_rules[].prefix_list_ids`: AWS prefix-list destinations. + - `aws.ec2.egress_rules[].from_port`: First destination port in the permitted range. + - `aws.ec2.egress_rules[].protocol`: IP protocol name or number. Use `-1` for all protocols. + - `aws.ec2.egress_rules[].security_groups`: Destination security-group IDs. + - `aws.ec2.egress_rules[].self`: Allows traffic to the managed security group itself when true. + - `aws.ec2.egress_rules[].to_port`: Last destination port in the permitted range. + - `aws.ec2.egress_rules[].description`: Optional rule description. + - `aws.ec2.tags`: Additional tags for runner instances, EBS volumes, network interfaces, and eligible Spot instance requests created from the launch template. They override module-level tags and the generated runner `Name`; the provider-managed `ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` bootstrap tags take final precedence. These tags do not apply to static provider resources such as the launch template, security group, IAM resources, SSM parameters, or log groups. + - `aws.ec2.metadata_options`: Instance Metadata Service configuration in the launch template. + - `aws.ec2.metadata_options.instance_metadata_tags`: Exposes instance tags through Instance Metadata Service when `enabled`. + - `aws.ec2.metadata_options.http_endpoint`: Enables or disables the Instance Metadata Service endpoint. + - `aws.ec2.metadata_options.http_tokens`: Controls whether IMDSv2 session tokens are optional or required. + - `aws.ec2.metadata_options.http_put_response_hop_limit`: Network hop limit for Instance Metadata Service token responses. + - `aws.ec2.credit_specification`: CPU credit mode for burstable instance types, either `standard` or `unlimited`. + - `aws.ec2.cpu_options`: CPU topology and processor-feature configuration. + - `aws.ec2.cpu_options.core_count`: Number of CPU cores exposed to the runner instance. + - `aws.ec2.cpu_options.threads_per_core`: Number of hardware threads exposed per CPU core. + - `aws.ec2.cpu_options.amd_sev_snp`: Enables or disables AMD SEV-SNP on supported instance types. + - `aws.ec2.cpu_options.nested_virtualization`: Enables or disables nested virtualization on supported instance types. + - `aws.ec2.placement`: EC2 placement configuration for runner instances. + - `aws.ec2.placement.affinity`: Host affinity setting. + - `aws.ec2.placement.availability_zone`: Availability Zone in which the instance is placed. + - `aws.ec2.placement.group_id`: Placement-group ID. + - `aws.ec2.placement.group_name`: Placement-group name. + - `aws.ec2.placement.host_id`: Dedicated Host ID. + - `aws.ec2.placement.host_resource_group_arn`: ARN of the host resource group used for placement. + - `aws.ec2.placement.spread_domain`: Spread-domain placement value. + - `aws.ec2.placement.tenancy`: Instance tenancy, such as `default`, `dedicated`, or `host`. + - `aws.ec2.placement.partition_number`: Placement-group partition number. + - `aws.ec2.license_specifications`: License Manager configurations added to the launch template. + - `aws.ec2.license_specifications[].license_configuration_arn`: ARN of a License Manager license configuration. + - `aws.ec2.associate_public_ipv4_address`: Associates a public IPv4 address with runner network interfaces. + - `aws.ec2.on_demand_failover_for_errors`: EC2 error codes that trigger an on-demand fallback after a Spot launch failure. + - `aws.ec2.scale_errors`: EC2 error codes treated as retryable scale-up failures. + - `aws.ec2.use_dedicated_host`: Enables the dedicated-host launch path, required for macOS runners. + EOT + + type = object({ + aws = optional(object({ + ec2 = optional(object({ + ami = optional(object({ + filter = optional(map(list(string)), { state = ["available"] }) + owners = optional(list(string), ["amazon"]) + id_ssm_parameter = optional(object({ + arn = string + }), null) + kms_key = optional(object({ + arn = string + }), null) + }), null) + vpc_id = string + subnet_ids = list(string) + overrides = optional(object({ + name_runner = optional(string, "") + name_sg = optional(string, "") + }), {}) + instance_profile = optional(object({ + name = string + }), null) + instance_profile_path = optional(string, null) + binaries_syncer = optional(object({ + enabled = optional(bool, true) + s3 = optional(object({ + arn = string + id = string + key = string + }), null) + }), {}) + block_device_mappings = optional(list(object({ + delete_on_termination = optional(bool, true) + device_name = optional(string, "/dev/xvda") + encrypted = optional(bool, true) + iops = optional(number) + kms_key_id = optional(string) + snapshot_id = optional(string) + throughput = optional(number) + volume_initialization_rate = optional(number) + volume_size = number + volume_type = optional(string, "gp3") + })), [{ volume_size = 30 }]) + ebs_optimized = optional(bool, false) + instance_target_capacity_type = optional(string, "spot") + instance_allocation_strategy = optional(string, "lowest-price") + instance_type_priorities = optional(map(number), null) + instance_max_spot_price = optional(string, null) + instance_types = list(string) + user_data = optional(object({ + enabled = optional(bool, true) + template = optional(string, null) + content = optional(string, null) + pre_install = optional(string, "") + post_install = optional(string, "") + debug_logging_enabled = optional(bool, false) + }), {}) + ssm_enabled = optional(bool, false) + create_service_linked_role_spot = optional(bool, false) + cloudwatch_agent = optional(object({ + enabled = optional(bool, true) + config = optional(string, null) + }), {}) + managed_security_group_enabled = optional(bool, true) + log_files = optional(list(object({ + log_group_name = string + prefix_log_group = bool + file_path = string + log_stream_name = string + log_class = optional(string, "STANDARD") + })), null) + key_name = optional(string, null) + additional_security_group_ids = optional(list(string), []) + detailed_monitoring_enabled = optional(bool, false) + egress_rules = optional(list(object({ + cidr_blocks = list(string) + ipv6_cidr_blocks = list(string) + prefix_list_ids = list(string) + from_port = number + protocol = string + security_groups = list(string) + self = bool + to_port = number + description = string + })), [{ + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + prefix_list_ids = null + from_port = 0 + protocol = "-1" + security_groups = null + self = null + to_port = 0 + description = null + }]) + tags = optional(map(string), {}) + metadata_options = optional(object({ + instance_metadata_tags = optional(string, "enabled") + http_endpoint = optional(string, "enabled") + http_tokens = optional(string, "required") + http_put_response_hop_limit = optional(number, 1) + }), {}) + credit_specification = optional(string, null) + cpu_options = optional(object({ + core_count = optional(number) + threads_per_core = optional(number) + amd_sev_snp = optional(string) + nested_virtualization = optional(string) + }), null) + placement = optional(object({ + affinity = optional(string) + availability_zone = optional(string) + group_id = optional(string) + group_name = optional(string) + host_id = optional(string) + host_resource_group_arn = optional(string) + spread_domain = optional(string) + tenancy = optional(string) + partition_number = optional(number) + }), null) + license_specifications = optional(list(object({ + license_configuration_arn = string + })), []) + associate_public_ipv4_address = optional(bool, false) + on_demand_failover_for_errors = optional(list(string), []) + scale_errors = optional(list(string), [ + "UnfulfillableCapacity", + "MaxSpotInstanceCountExceeded", + "TargetCapacityLimitExceededException", + "RequestLimitExceeded", + "ResourceLimitExceeded", + "MaxSpotInstanceCountExceeded", + "MaxSpotFleetRequestCountExceeded", + "InsufficientInstanceCapacity", + "InsufficientCapacityOnHost", + ]) + use_dedicated_host = optional(bool, false) + }), null) + }), {}) + }) + +} diff --git a/modules/runner-config/variables.orchestration-provider.tf b/modules/runner-config/variables.orchestration-provider.tf new file mode 100644 index 0000000000..6d176ad6d7 --- /dev/null +++ b/modules/runner-config/variables.orchestration-provider.tf @@ -0,0 +1,141 @@ +# Typed orchestration-provider input boundary between the common runner configuration and demand controllers. +variable "orchestration_provider" { + description = <<-EOT + Runner demand-orchestration provider configuration. Exactly one provider block must be non-null. Wrapper presence selects the provider and must therefore be known during planning; values inside the selected provider may remain unknown until apply. + + - `webhook`: Selects the workflow-job webhook control plane. It owns runner lifecycle and capacity, the build queue reference, the runner-control artifact, scale-up, scale-down, scheduled pool, and optional job-retry controls. Future providers can be added as sibling blocks without moving this contract. + - `webhook.runner`: Runner lifecycle, boot timeout, and capacity settings owned by webhook orchestration. + - `webhook.runner.boot_time_in_minutes`: Expected runner boot duration used by scale-down and pool controls. The default is `5`. + - `webhook.runner.ephemeral`: Registers runners in ephemeral mode. The default is `false`. + - `webhook.runner.jit_config_enabled`: Explicitly enables or disables just-in-time configuration. The default is null, which follows `runner.ephemeral`. + - `webhook.runner.maximum_count`: Maximum number of runners managed for this runner configuration. The default is `3`. + - `webhook.github.organization_runners`: Registers runners at organization scope when true; otherwise registration is repository-scoped. + - `webhook.queue.build.arn`: ARN of the runner configuration's build queue. + - `webhook.queue.build.url`: URL of the runner configuration's build queue. + - `webhook.queue.kms_key_id`: Optional KMS key ARN encrypting the build queue. The default is null and is independent from the Parameter Store KMS key. + - `webhook.queue.tags`: Tags inherited by queue-related provider resources before component-specific overrides. The default is `{}`. + - `webhook.lambda.artifact`: Runner-control artifact shared by scale, pool, and job-retry components. Set at most one of `zip` or `s3`; no selection uses the packaged runner archive. + - `webhook.lambda.artifact.zip`: Optional local path to the runner-control Lambda archive. The default is null. + - `webhook.lambda.artifact.s3`: Optional S3 object selector in the common `lambda.artifact.s3.bucket`. Wrapper presence must be known during planning, selecting it requires a non-null common bucket, and the default is null. + - `webhook.lambda.artifact.s3.key`: Object key of the runner-control Lambda archive. + - `webhook.lambda.artifact.s3.object_version`: Optional object version of the runner-control Lambda archive. The default is null. + - `webhook.lambda.scale.up.memory_size`: Memory allocated to the scale-up Lambda in MB. The default is `512`. + - `webhook.lambda.scale.up.timeout`: Scale-up Lambda timeout in seconds. The default is `60`. + - `webhook.lambda.scale.up.reserved_concurrent_executions`: Reserved concurrency for scale-up. The default is `1`; use `-1` for unreserved concurrency. + - `webhook.lambda.scale.up.job_queued_check_enabled`: Enables queued-job verification before scaling. The default is null, which follows the resolved runner mode. + - `webhook.lambda.scale.up.event_source_mapping.batch_size`: Maximum build-queue records delivered per scale-up invocation. The default is `10`. + - `webhook.lambda.scale.up.event_source_mapping.maximum_batching_window_in_seconds`: Maximum batching window for build-queue records. The default is `0`. + - `webhook.lambda.scale.up.tags`: Tags applied within scale-up resource scopes after common provider tags. The default is `{}`. + - `webhook.lambda.scale.down.memory_size`: Memory allocated to the scale-down Lambda in MB. The default is `512`. + - `webhook.lambda.scale.down.timeout`: Scale-down Lambda timeout in seconds. The default is `60`. + - `webhook.lambda.scale.down.schedule_expression`: EventBridge schedule expression that invokes scale-down. The default is `cron(*/5 * * * ? *)`. + - `webhook.lambda.scale.down.minimum_running_time_in_minutes`: Optional minimum runner age before scale-down may terminate it. The default is null, which selects the operating-system default. + - `webhook.lambda.scale.down.idle_config`: Time-based desired idle-runner configurations. The default is `[]`. + - `webhook.lambda.scale.down.idle_config[].cron`: Cron expression identifying when the idle configuration applies. + - `webhook.lambda.scale.down.idle_config[].timeZone`: IANA time zone used to evaluate the cron expression. + - `webhook.lambda.scale.down.idle_config[].idleCount`: Number of idle runners retained during the matching period. + - `webhook.lambda.scale.down.idle_config[].evictionStrategy`: Selection strategy used when excess idle runners are removed. The default is `oldest_first`. + - `webhook.lambda.scale.down.tags`: Tags applied within scale-down resource scopes after common provider tags. The default is `{}`. + - `webhook.lambda.pool.memory_size`: Memory allocated to the pool Lambda in MB. The default is `512`. + - `webhook.lambda.pool.timeout`: Pool Lambda timeout in seconds. The default is `60`. + - `webhook.lambda.pool.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. The default is `1`; use `-1` for unreserved concurrency. + - `webhook.lambda.pool.config`: Scheduled target pool sizes. The default is `[]`, which disables the pool component. + - `webhook.lambda.pool.config[].schedule_expression`: Scheduler expression that activates the target size. + - `webhook.lambda.pool.config[].schedule_expression_timezone`: Optional IANA time zone used to evaluate the schedule. + - `webhook.lambda.pool.config[].size`: Desired number of runners for the schedule. + - `webhook.lambda.pool.include_busy_runners`: Includes busy runners when reconciling scheduled pool capacity. The default is `false`. + - `webhook.lambda.pool.runner_owner`: Optional GitHub organization or repository owner used for pooled runners. The default is null. + - `webhook.lambda.pool.tags`: Tags applied within pool resource scopes after common provider tags. The default is `{}`. + - `webhook.job_retry.enabled`: Creates the retry queue, Lambda function, event-source mapping, and related IAM resources. The default is `false`. + - `webhook.job_retry.delay_in_seconds`: Initial delay before a queued-job retry check. The default is `300`. + - `webhook.job_retry.delay_backoff`: Multiplier applied to the delay after each unsuccessful check. The default is `2`. + - `webhook.job_retry.max_attempts`: Maximum retry-check attempts before the message is no longer republished. The default is `1`. + - `webhook.job_retry.tags`: Tags applied within job-retry resource scopes after common provider tags. The default is `{}`. + - `webhook.job_retry.lambda.memory_size`: Memory allocated to the job-retry Lambda in MB. The default is `256`. + - `webhook.job_retry.lambda.reserved_concurrent_executions`: Reserved concurrency for job retry. The default is `1`; use `-1` for unreserved concurrency. + - `webhook.job_retry.lambda.timeout`: Job-retry Lambda timeout in seconds and visibility timeout for its retry queue. The default is `30`. + EOT + type = object({ + webhook = optional(object({ + runner = optional(object({ + boot_time_in_minutes = optional(number, 5) + ephemeral = optional(bool, false) + jit_config_enabled = optional(bool, null) + maximum_count = optional(number, 3) + }), {}) + github = object({ + organization_runners = bool + }) + queue = object({ + build = object({ + arn = string + url = string + }) + kms_key_id = optional(string, null) + tags = optional(map(string), {}) + }) + lambda = optional(object({ + artifact = optional(object({ + zip = optional(string, null) + s3 = optional(object({ + key = string + object_version = optional(string, null) + }), null) + }), {}) + scale = optional(object({ + up = optional(object({ + memory_size = optional(number, 512) + timeout = optional(number, 60) + reserved_concurrent_executions = optional(number, 1) + job_queued_check_enabled = optional(bool, null) + event_source_mapping = optional(object({ + batch_size = optional(number, 10) + maximum_batching_window_in_seconds = optional(number, 0) + }), {}) + tags = optional(map(string), {}) + }), {}) + down = optional(object({ + memory_size = optional(number, 512) + timeout = optional(number, 60) + schedule_expression = optional(string, "cron(*/5 * * * ? *)") + minimum_running_time_in_minutes = optional(number, null) + idle_config = optional(list(object({ + cron = string + timeZone = string + idleCount = number + evictionStrategy = optional(string, "oldest_first") + })), []) + tags = optional(map(string), {}) + }), {}) + }), {}) + pool = optional(object({ + memory_size = optional(number, 512) + timeout = optional(number, 60) + reserved_concurrent_executions = optional(number, 1) + config = optional(list(object({ + schedule_expression = string + schedule_expression_timezone = optional(string) + size = number + })), []) + include_busy_runners = optional(bool, false) + runner_owner = optional(string, null) + tags = optional(map(string), {}) + }), {}) + }), {}) + job_retry = optional(object({ + enabled = optional(bool, false) + delay_in_seconds = optional(number, 300) + delay_backoff = optional(number, 2) + max_attempts = optional(number, 1) + tags = optional(map(string), {}) + lambda = optional(object({ + memory_size = optional(number, 256) + reserved_concurrent_executions = optional(number, 1) + timeout = optional(number, 30) + }), {}) + }), {}) + }), null) + }) + nullable = false + +} diff --git a/modules/runner-config/variables.tf b/modules/runner-config/variables.tf new file mode 100644 index 0000000000..5928693ade --- /dev/null +++ b/modules/runner-config/variables.tf @@ -0,0 +1,246 @@ +variable "aws_region" { + description = "AWS region." + type = string +} + +variable "aws_partition" { + description = "AWS partition used to construct ARNs." + type = string + default = "aws" +} + +variable "prefix" { + description = "The prefix used for naming resources." + type = string + default = "github-actions" +} + +variable "tags" { + description = "Base tags added to taggable resources created by this runner configuration. Shared, component, and compute-provider tag maps override matching keys within their documented resource scopes." + type = map(string) + default = {} +} + +variable "runner" { + description = <<-EOT + Provider-neutral GitHub runner configuration. + + - `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`. + - `architecture`: Runner distribution architecture, such as `x64` or `arm64`. + - `disable_default_labels`: Prevents GitHub's default self-hosted, operating-system, and architecture labels from being registered. + - `labels`: Complete set of labels supplied to the control-plane functions. + - `group_name`: GitHub runner group used during registration. + - `name_prefix`: Prefix added to registered runner names. + - `run_as_root`: Runs the runner service as root when supported by the compute provider. + - `run_as`: Operating-system user used when `run_as_root` is false. + - `auto_update_disabled`: Disables the GitHub runner application's built-in updater. + - `tags`: Additional tags for common runner resources, currently the managed runner IAM role. These override module-level `tags` with the same key. + - `hooks.job_started`: Script content installed as the runner job-started hook. + - `hooks.job_completed`: Script content installed as the runner job-completed hook. + - `iam.role.arn`: ARN of an externally managed runner role. When set, this module does not create or modify that role. + - `iam.managed_policy_arns`: Named managed-policy ARNs attached to the module-managed runner role. + - `iam.additional_trust_policy_json`: Optional IAM policy document merged with the selected compute provider's default runner-role trust policy. + - `iam.path`: IAM path for the module-managed runner role. Defaults to a path derived from `prefix`. + - `iam.permissions_boundary`: Permissions-boundary ARN for the module-managed runner role. + EOT + type = object({ + os = optional(string, "linux") + architecture = optional(string, "x64") + disable_default_labels = optional(bool, false) + labels = list(string) + group_name = optional(string, "Default") + name_prefix = optional(string, "") + run_as_root = optional(bool, false) + run_as = optional(string, "ec2-user") + auto_update_disabled = optional(bool, false) + tags = optional(map(string), {}) + hooks = optional(object({ + job_started = optional(string, "") + job_completed = optional(string, "") + }), {}) + iam = optional(object({ + role = optional(object({ + arn = string + }), null) + managed_policy_arns = optional(map(string), {}) + additional_trust_policy_json = optional(string, null) + path = optional(string, null) + permissions_boundary = optional(string, null) + }), {}) + }) + +} + +variable "github" { + description = <<-EOT + GitHub API and runner-registration configuration. + + - `app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys. + - `app_parameters.id`: Ordered Parameter Store references for GitHub App IDs. + - `app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs. + - `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com. + - `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server requests. + - `user_agent`: Optional User-Agent value added to GitHub API requests. + EOT + type = object({ + app_parameters = object({ + key_base64 = list(map(string)) + id = list(map(string)) + installation_id = list(object({ name = string, arn = string })) + }) + enterprise_server = optional(object({ + url = optional(string, null) + ssl_verify = optional(bool, true) + }), {}) + user_agent = optional(string, null) + }) +} + +variable "lambda" { + description = <<-EOT + Common Lambda substrate independent of the selected runner orchestration provider. + + - `artifact.s3.bucket`: Optional shared S3 bucket containing component-owned Lambda artifacts. An orchestration provider selects its own object key and version; the bucket alone selects no artifact. + - `runtime`: Runtime used by the control-plane Lambda functions. + - `architecture`: Instruction-set architecture used by the control-plane Lambda functions. Supported values are `arm64` and `x86_64`. + - `subnet_ids`: Subnets used for Lambda VPC configuration. + - `security_group_ids`: Security groups used for Lambda VPC configuration. + - `tags`: Shared tags applied to Lambda function resources only. These override module-level `tags`; component `tags` override this map when keys conflict. + - `principals`: Additional principals allowed to assume the control-plane Lambda roles. + - `role.path`: IAM path for module-managed Lambda execution roles. Defaults to a path derived from `prefix`. + - `role.permissions_boundary`: Permissions-boundary ARN applied to module-managed Lambda execution roles. + EOT + type = object({ + artifact = optional(object({ + s3 = optional(object({ + bucket = optional(string, null) + }), {}) + }), {}) + runtime = optional(string, "nodejs24.x") + architecture = optional(string, "arm64") + subnet_ids = optional(list(string), []) + security_group_ids = optional(list(string), []) + tags = optional(map(string), {}) + principals = optional(list(object({ + type = string + identifiers = list(string) + })), []) + role = optional(object({ + path = optional(string, null) + permissions_boundary = optional(string, null) + }), {}) + }) + default = {} + +} + +variable "ssm" { + description = <<-EOT + Parameter Store paths, encryption, tag scopes, and housekeeper configuration. + + - `paths.root`: Root Parameter Store path for this runner configuration. + - `paths.tokens`: Path segment under `paths.root` used for registration tokens and just-in-time configuration. + - `paths.config`: Path segment under `paths.root` used for persistent runner configuration. + - `kms_key_id`: Optional customer-managed KMS key ARN used by control-plane IAM policies to decrypt shared GitHub App parameters. The ARN may be unknown until apply; null omits the provider-owned KMS statements. It does not select encryption for runtime-created runner parameters. + - `tags`: Shared tags for SSM-related resources. These override module-level `tags` and are inherited by parameter and housekeeper resources. + - `parameters.tags`: Tags for Terraform-managed runner configuration parameters and temporary parameters created by the scale-up and pool Lambdas. These override module-level and `ssm.tags` values with the same key. + - `housekeeper.schedule_expression`: EventBridge schedule expression that invokes the SSM housekeeper. + - `housekeeper.state`: EventBridge rule state, such as `ENABLED` or `DISABLED`. + - `housekeeper.tags`: Tags for housekeeper resources, including the Lambda function, log group, EventBridge rule, and IAM role. These override module-level, `ssm.tags`, shared Lambda, and shared log tags when keys conflict. + - `housekeeper.lambda.artifact`: Component-owned SSM-housekeeper artifact selection. Set at most one of `zip` or `s3`; when neither is selected, the module uses its packaged runner control-plane archive. This selector does not inherit an orchestration-provider artifact. + - `housekeeper.lambda.artifact.zip`: Optional local path to the SSM-housekeeper Lambda archive. + - `housekeeper.lambda.artifact.s3`: Optional object key and version in the shared `lambda.artifact.s3.bucket`. Selecting S3 requires that common bucket. + - `housekeeper.lambda.artifact.s3.key`: Object key of the SSM-housekeeper Lambda archive. + - `housekeeper.lambda.artifact.s3.object_version`: Optional object version of the SSM-housekeeper Lambda archive. + - `housekeeper.lambda.memory_size`: Memory allocated to the SSM housekeeper Lambda in MB. + - `housekeeper.lambda.timeout`: SSM housekeeper Lambda timeout in seconds. + - `housekeeper.config.tokenPath`: Parameter Store token path cleaned by the housekeeper. When omitted, the configured runner token path is used. + - `housekeeper.config.minimumDaysOld`: Minimum parameter age in days before deletion is allowed. + - `housekeeper.config.dryRun`: Reports eligible parameters without deleting them when true. + EOT + type = object({ + paths = object({ + root = string + tokens = string + config = string + }) + kms_key_id = optional(string, null) + tags = optional(map(string), {}) + parameters = optional(object({ + tags = optional(map(string), {}) + }), {}) + housekeeper = optional(object({ + schedule_expression = optional(string, "rate(1 day)") + state = optional(string, "ENABLED") + tags = optional(map(string), {}) + lambda = optional(object({ + artifact = optional(object({ + zip = optional(string, null) + s3 = optional(object({ + key = string + object_version = optional(string, null) + }), null) + }), {}) + memory_size = optional(number, 512) + timeout = optional(number, 60) + }), {}) + config = optional(object({ + tokenPath = optional(string) + minimumDaysOld = optional(number, 1) + dryRun = optional(bool, false) + }), {}) + }), {}) + }) + +} + +variable "observability" { + description = <<-EOT + Logging, tracing, and metrics configuration for control-plane and provider resources. + + - `logs.level`: Application log level supplied to the control-plane functions. + - `logs.retention_in_days`: CloudWatch Logs retention period. + - `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt CloudWatch log groups. + - `logs.class`: CloudWatch log-group class. Supported values are `STANDARD` and `INFREQUENT_ACCESS`. + - `logs.tags`: Shared tags for CloudWatch log groups. These override module-level `tags`; component `tags` override this map when keys conflict. + - `tracing.mode`: Optional Lambda active-tracing mode. Null disables X-Ray tracing configuration. + - `tracing.capture_http_requests`: Enables HTTP request capture in the tracing helper. + - `tracing.capture_error`: Enables error capture in the tracing helper. + - `metrics.enabled`: Enables module-emitted metrics. + - `metrics.namespace`: CloudWatch namespace used for emitted metrics. + - `metrics.metric.github_app_rate_limit.enabled`: Emits GitHub App rate-limit metrics. + - `metrics.metric.job_retry.enabled`: Emits job-retry metrics. + - `metrics.metric.spot_termination_warning.enabled`: Emits spot-termination warning metrics where supported. + EOT + type = object({ + logs = optional(object({ + level = optional(string, "info") + retention_in_days = optional(number, 180) + kms_key_id = optional(string, null) + class = optional(string, "STANDARD") + tags = optional(map(string), {}) + }), {}) + tracing = optional(object({ + mode = optional(string, null) + capture_http_requests = optional(bool, false) + capture_error = optional(bool, false) + }), {}) + metrics = optional(object({ + enabled = optional(bool, false) + namespace = optional(string, "GitHub Runners") + metric = optional(object({ + github_app_rate_limit = optional(object({ + enabled = optional(bool, true) + }), {}) + job_retry = optional(object({ + enabled = optional(bool, true) + }), {}) + spot_termination_warning = optional(object({ + enabled = optional(bool, true) + }), {}) + }), {}) + }), {}) + }) + default = {} + +} diff --git a/modules/runner-config/versions.tf b/modules/runner-config/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/runner-config/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +}